diff --git a/.github/workflows/build-if-tag.yml b/.github/workflows/build-if-tag.yml new file mode 100644 index 0000000..4fa0198 --- /dev/null +++ b/.github/workflows/build-if-tag.yml @@ -0,0 +1,41 @@ +name: build-if-tag + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +env: + APP: amneziawg-go + +jobs: + build: + runs-on: ubuntu-latest + name: build + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.ref_name }} + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Setup metadata + uses: docker/metadata-action@v5 + id: metadata + with: + images: amneziavpn/${{ env.APP }} + tags: type=semver,pattern={{version}} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ steps.metadata.outputs.tags }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..98a7e9e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.24.4 as awg +COPY . /awg +WORKDIR /awg +RUN go mod download && \ + go mod verify && \ + go build -ldflags '-linkmode external -extldflags "-fno-PIC -static"' -v -o /usr/bin + +FROM alpine:3.19 +ARG AWGTOOLS_RELEASE="1.0.20250901" + +RUN apk --no-cache add iproute2 iptables bash && \ + cd /usr/bin/ && \ + wget https://github.com/amnezia-vpn/amneziawg-tools/releases/download/v${AWGTOOLS_RELEASE}/alpine-3.19-amneziawg-tools.zip && \ + unzip -j alpine-3.19-amneziawg-tools.zip && \ + chmod +x /usr/bin/awg /usr/bin/awg-quick && \ + ln -s /usr/bin/awg /usr/bin/wg && \ + ln -s /usr/bin/awg-quick /usr/bin/wg-quick +COPY --from=awg /usr/bin/amneziawg-go /usr/bin/amneziawg-go diff --git a/LICENSE b/LICENSE index f85e365..ab45fb3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,5 @@ +Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to diff --git a/Makefile b/Makefile index 3f6e407..7a88647 100644 --- a/Makefile +++ b/Makefile @@ -9,23 +9,23 @@ MAKEFLAGS += --no-print-directory generate-version-and-build: @export GIT_CEILING_DIRECTORIES="$(realpath $(CURDIR)/..)" && \ - tag="$$(git describe --dirty 2>/dev/null)" && \ + tag="$$(git describe --tags --dirty 2>/dev/null)" && \ ver="$$(printf 'package main\n\nconst Version = "%s"\n' "$$tag")" && \ [ "$$(cat version.go 2>/dev/null)" != "$$ver" ] && \ echo "$$ver" > version.go && \ git update-index --assume-unchanged version.go || true - @$(MAKE) wireguard-go + @$(MAKE) amneziawg-go -wireguard-go: $(wildcard *.go) $(wildcard */*.go) +amneziawg-go: $(wildcard *.go) $(wildcard */*.go) go build -v -o "$@" -install: wireguard-go - @install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/wireguard-go" +install: amneziawg-go + @install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/amneziawg-go" test: go test ./... clean: - rm -f wireguard-go + rm -f amneziawg-go .PHONY: all clean test install generate-version-and-build diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd92535 --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +**English** · [Русский](README.ru.md) + +# wireguard-go (lx fork) — sagernet + AmneziaWG 2.0 + +The WireGuard-Go runtime used by **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: +**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (the fork sing-box builds on) **+ AmneziaWG 2.0 obfuscation**, merged together. + +This is **not** a general-purpose project. It exists for one reason — see below — and lives on the **`lx`** branch. + +--- + +## Why this fork exists + +sing-box's WireGuard endpoint needs **sagernet/wireguard-go**'s additions (the `conn.Bind.Send(…, offset)` contract, `device.InputPacket`, reserved/control). AmneziaWG's DPI-evasion obfuscation lives in **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, which is a fork of *upstream* wireguard-go and therefore **lacks** those sagernet additions. + +So neither fork alone works for sing-box-lx: + +| | sing-box-compat API | AmneziaWG obfuscation | +|---|:---:|:---:| +| `sagernet/wireguard-go` | ✅ | ❌ | +| `amnezia-vpn/amneziawg-go` | ❌ | ✅ | +| **this fork** | ✅ | ✅ | + +Each existing fork gives exactly **half** of what's needed: + +- **Take `sagernet/wireguard-go`** → sing-box-lx compiles and runs, but the AWG fields (`jc`/`h1`/`i1`…) do nothing → **no obfuscation**; AmneziaWG doesn't actually work. +- **Take `amnezia-vpn/amneziawg-go`** → the obfuscation is there, but sing-box-lx **won't even compile** (the sagernet functions are missing). + +We need **both** ✅ at once, and no ready-made fork has them — so we built one by **merging**: sagernet (for the API) + amnezia (for the obfuscation). That is exactly the **"this fork"** row above. + +The approach: **keep the sagernet base and graft the obfuscation onto it** — rather than the reverse (adding sagernet's APIs to amneziawg-go, which would route even plain WireGuard through a foreign device). This way sing-box compiles unchanged, the obfuscation is additive and off by default, and a config without AWG fields behaves exactly like plain WireGuard. + +## How the merge works + +Both `sagernet/wireguard-go` and `amneziawg-go` descend from the same upstream `git.zx2c4.com/wireguard-go`, so they share git history — which makes a real **3-way merge** possible (not a hand-port). + +- **Base:** `sagernet/wireguard-go` (the exact commit sing-box pins — currently `506b7631853c`). +- **Merged in:** `amnezia-vpn/amneziawg-go` (a tip with AWG2 / I1–I5 + the S4-keepalive fix). +- **Key trick:** `MessageEncapsulatingTransportSize` is set to **`0`** in `device/noise-protocol.go`. sing-box-lx does not use sagernet's 8-byte `Bind.Send` headroom, and zeroing it makes the AmneziaWG obfuscation compose cleanly with no weave conflicts in the packet send path. +- **Isolation:** the obfuscation is confined to `device/` — new files `device/obf*.go`, `device/magic-header.go`, plus grafts in `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` stay pure sagernet.** +- **Module path is unchanged** (`module github.com/sagernet/wireguard-go`) so the consumer plugs it in with a `replace` directive and needs **no import edits**. + +## Consumed by + +[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) wires this in as a git submodule + a `replace`: + +``` +# sing-box-lx/.gitmodules +[submodule "submodules/wireguard-go"] + url = https://github.com/Leadaxe/wireguard-go-awg2-lx + branch = lx + +# sing-box-lx/go.mod (// lx) +replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go +``` + +Built with the `with_awg` tag, it has been **live-validated** against a real AmneziaWG 2.0 server (handshake + keepalive + outbound traffic) and cross-compiles on linux/darwin/windows × amd64/arm64. + +## Maintaining it (rebase onto a new sagernet tag) + +When sing-box bumps `sagernet/wireguard-go`, redo the merge: + +```sh +git remote add origin https://github.com/sagernet/wireguard-go # base +git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # obfuscation source +git fetch --all +git checkout -b lx +git merge amnezia/master # real 3-way merge via the shared upstream ancestor +``` + +Conflict resolution recipe: + +1. New `device/obf*.go` + `device/magic-header.go` come in clean. +2. **Mechanical** conflicts (amnezia → sagernet import paths, `queueconstants*`, `sticky*`, `tun.go`) → take **ours** (sagernet). +3. Remove amnezia-added infra duplicates: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`. +4. `device/device.go` → **union** (sagernet `pauseManager` + amnezia obf fields). +5. `device/send.go` / `receive.go` → take amnezia's obfuscation, set `MessageEncapsulatingTransportSize = 0`, and keep the 3-arg `bind.Send(…, 0)` calls. +6. `conn/`, `tun/`, `ipc/`, `go.mod` module path → **ours** (sagernet). + +Then in sing-box-lx: bump the submodule, `make -f Makefile.lx lx-build`, and re-test against an AWG2 server. + +## Links + +| | | +|---|---| +| Consumer | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) | +| Base | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) | +| Obfuscation source | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) | +| Original | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) | + +## License + +MIT, inherited from WireGuard-Go (see [`LICENSE`](LICENSE)). The AmneziaWG obfuscation is likewise MIT (from amneziawg-go). This is an unofficial fork, not affiliated with WireGuard, SagerNet, or Amnezia. diff --git a/README.ru.md b/README.ru.md new file mode 100644 index 0000000..7482bd4 --- /dev/null +++ b/README.ru.md @@ -0,0 +1,93 @@ +[English](README.md) · **Русский** + +# wireguard-go (lx-форк) — sagernet + AmneziaWG 2.0 + +Рантайм WireGuard-Go для **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: +**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (форк, на котором собирается sing-box) **+ обфускация AmneziaWG 2.0**, слитые вместе. + +Это **не** универсальный проект. Он существует ради одной задачи (см. ниже) и живёт на ветке **`lx`**. + +--- + +## Зачем этот форк + +WireGuard-endpoint sing-box нуждается в добавках **sagernet/wireguard-go** (контракт `conn.Bind.Send(…, offset)`, `device.InputPacket`, reserved/control). Обфускация против DPI у AmneziaWG живёт в **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, который форкнут от *upstream* wireguard-go и потому этих sagernet-добавок **не имеет**. + +Значит для sing-box-lx ни один форк по отдельности не подходит: + +| | API под sing-box | обфускация AmneziaWG | +|---|:---:|:---:| +| `sagernet/wireguard-go` | ✅ | ❌ | +| `amnezia-vpn/amneziawg-go` | ❌ | ✅ | +| **этот форк** | ✅ | ✅ | + +Каждый существующий форк даёт ровно **половину** нужного: + +- **возьмёшь `sagernet/wireguard-go`** → sing-box-lx соберётся и запустится, но AWG-поля (`jc`/`h1`/`i1`…) ничего не сделают → **обфускации нет**; AmneziaWG не работает; +- **возьмёшь `amnezia-vpn/amneziawg-go`** → обфускация есть, но sing-box-lx **не скомпилируется** (нет sagernet-функций). + +Нужны **обе** ✅ сразу, а готового форка с двумя галочками не существует — поэтому мы собрали его **слиянием**: sagernet (за API) + amnezia (за обфускацию). Это ровно строка **«этот форк»** выше. + +Подход: **берём sagernet-базу и граффтим обфускацию на неё** — а не наоборот (не дотачиваем sagernet-API к amneziawg-go, иначе даже обычный WireGuard шёл бы через чужой device). Так sing-box компилируется без изменений, обфускация аддитивна и выключена по умолчанию, а конфиг без AWG-полей ведёт себя как обычный WireGuard. + +## Как устроен merge + +И `sagernet/wireguard-go`, и `amneziawg-go` происходят от одного upstream `git.zx2c4.com/wireguard-go`, поэтому делят git-историю — а значит возможен настоящий **3-way merge** (а не ручной перенос). + +- **База:** `sagernet/wireguard-go` (тот коммит, что пинит sing-box — сейчас `506b7631853c`). +- **Вливаем:** `amnezia-vpn/amneziawg-go` (тип с AWG2 / I1–I5 + фикс S4-keepalive). +- **Ключевой трюк:** `MessageEncapsulatingTransportSize` выставлен в **`0`** в `device/noise-protocol.go`. sing-box-lx не использует 8-байтный headroom sagernet для `Bind.Send`, и обнуление позволяет обфускации AmneziaWG встать чисто, без конфликтов в send-пути. +- **Изоляция:** обфускация замкнута в `device/` — новые файлы `device/obf*.go`, `device/magic-header.go` + графты в `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` остаются чистым sagernet.** +- **Module-path не меняется** (`module github.com/sagernet/wireguard-go`), поэтому потребитель подключает форк через `replace` без правки импортов. + +## Кто потребляет + +[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) подключает это как git submodule + `replace`: + +``` +# sing-box-lx/.gitmodules +[submodule "submodules/wireguard-go"] + url = https://github.com/Leadaxe/wireguard-go-awg2-lx + branch = lx + +# sing-box-lx/go.mod (// lx) +replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go +``` + +Собранный с тегом `with_awg`, он **проверен живым** сервером AmneziaWG 2.0 (handshake + keepalive + трафик наружу) и кросс-компилируется на linux/darwin/windows × amd64/arm64. + +## Сопровождение (ребейз на новый sagernet-тег) + +Когда sing-box бампит `sagernet/wireguard-go`, повторяем merge: + +```sh +git remote add origin https://github.com/sagernet/wireguard-go # база +git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # источник обфускации +git fetch --all +git checkout -b lx <новый-sagernet-коммит> +git merge amnezia/master # настоящий 3-way merge через общего upstream-предка +``` + +Рецепт разрешения конфликтов: + +1. Новые `device/obf*.go` + `device/magic-header.go` приходят чисто. +2. **Механические** конфликты (import-path amnezia → sagernet, `queueconstants*`, `sticky*`, `tun.go`) → берём **наши** (sagernet). +3. Удаляем amnezia-инфра-дубликаты: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`. +4. `device/device.go` → **union** (sagernet `pauseManager` + obf-поля amnezia). +5. `device/send.go` / `receive.go` → берём обфускацию amnezia, ставим `MessageEncapsulatingTransportSize = 0`, сохраняем 3-арг `bind.Send(…, 0)`. +6. `conn/`, `tun/`, `ipc/`, module-path в `go.mod` → **наши** (sagernet). + +Затем в sing-box-lx: бампим submodule, `make -f Makefile.lx lx-build` и пере-тест против AWG2-сервера. + +## Ссылки + +| | | +|---|---| +| Потребитель | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) | +| База | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) | +| Источник обфускации | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) | +| Оригинал | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) | + +## Лицензия + +MIT, унаследована от WireGuard-Go (см. [`LICENSE`](LICENSE)). Обфускация AmneziaWG — тоже MIT (из amneziawg-go). Это неофициальный форк, не аффилирован с WireGuard, SagerNet или Amnezia. diff --git a/conn/bind_std.go b/conn/bind_std.go index 4dadb0d..6bf5978 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn @@ -23,7 +23,10 @@ import ( "golang.org/x/net/ipv6" ) -var _ Bind = (*StdNetBind)(nil) +var ( + _ Bind = (*StdNetBind)(nil) + _ Endpoint = (*StdNetEndpoint)(nil) +) // StdNetBind implements Bind for all platforms. While Windows has its own Bind // (see bind_windows.go), it may fall back to StdNetBind. @@ -67,12 +70,10 @@ func NewStdNetBind(externalControl control.Func) Bind { msgsPool: sync.Pool{ New: func() any { - // ipv6.Message and ipv4.Message are interchangeable as they are - // both aliases for x/net/internal/socket.Message. msgs := make([]ipv6.Message, IdealBatchSize) for i := range msgs { msgs[i].Buffers = make(net.Buffers, 1) - msgs[i].OOB = make([]byte, 0, stickyControlSize+gsoControlSize) + msgs[i].OOB = make([]byte, controlSize) } return &msgs }, @@ -115,7 +116,7 @@ func (e *StdNetEndpoint) DstIP() netip.Addr { return e.AddrPort.Addr() } -// See control_default,linux, etc for implementations of SrcIP and SrcIfidx. +// See sticky_default,linux, etc for implementations of SrcIP and SrcIfidx. func (e *StdNetEndpoint) DstToBytes() []byte { b, _ := e.AddrPort.MarshalBinary() @@ -165,6 +166,11 @@ func listenNet(externalControl control.Func, network string, port int) (*net.UDP return conn.(*net.UDPConn), uaddr.Port, nil } +// errEADDRINUSE is syscall.EADDRINUSE, boxed into an interface once +// in erraddrinuse.go on almost all platforms. For other platforms, +// it's at least non-nil. +var errEADDRINUSE error = errors.New("") + func (s *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) { s.mu.Lock() defer s.mu.Unlock() @@ -191,7 +197,7 @@ again: // Listen on the same port as we're using for ipv4. v6conn, port, err = listenNet(s.externalControl, "udp6", port) - if uport == 0 && errors.Is(err, syscall.EADDRINUSE) && tries < 100 { + if uport == 0 && errors.Is(err, errEADDRINUSE) && tries < 100 { v4conn.Close() tries++ goto again @@ -203,7 +209,7 @@ again: var fns []ReceiveFunc if v4conn != nil { s.ipv4TxOffload, s.ipv4RxOffload = supportsUDPOffload(v4conn) - if runtime.GOOS == "linux" || runtime.GOOS == "android" { + if runtime.GOOS == "linux" { v4pc = ipv4.NewPacketConn(v4conn) s.ipv4PC = v4pc } @@ -212,7 +218,7 @@ again: } if v6conn != nil { s.ipv6TxOffload, s.ipv6RxOffload = supportsUDPOffload(v6conn) - if runtime.GOOS == "linux" || runtime.GOOS == "android" { + if runtime.GOOS == "linux" { v6pc = ipv6.NewPacketConn(v6conn) s.ipv6PC = v6pc } @@ -228,7 +234,6 @@ again: func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) { for i := range *msgs { - (*msgs)[i].OOB = (*msgs)[i].OOB[:0] (*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB} } s.msgsPool.Put(msgs) @@ -264,9 +269,9 @@ func (s *StdNetBind) receiveIP( } defer s.putMessages(msgs) var numMsgs int - if runtime.GOOS == "linux" || runtime.GOOS == "android" { + if runtime.GOOS == "linux" { if rxOffload { - readAt := len(*msgs) - (IdealBatchSize / udpSegmentMaxDatagrams) + readAt := len(*msgs) - 2 numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0) if err != nil { return 0, err @@ -320,7 +325,7 @@ func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxO // TODO: When all Binds handle IdealBatchSize, remove this dynamic function and // rename the IdealBatchSize constant to BatchSize. func (s *StdNetBind) BatchSize() int { - if runtime.GOOS == "linux" || runtime.GOOS == "android" { + if runtime.GOOS == "linux" { return IdealBatchSize } return 1 @@ -455,7 +460,7 @@ func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message err error start int ) - if runtime.GOOS == "linux" || runtime.GOOS == "android" { + if runtime.GOOS == "linux" { for { n, err = pc.WriteBatch(msgs[start:], 0) if err != nil || n == len(msgs[start:]) { diff --git a/conn/bind_windows.go b/conn/bind_windows.go index 121079f..51c0974 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn diff --git a/conn/sticky_default.go b/conn/control_default.go similarity index 59% rename from conn/sticky_default.go rename to conn/control_default.go index 15b65af..a8bc06a 100644 --- a/conn/sticky_default.go +++ b/conn/control_default.go @@ -1,8 +1,8 @@ -//go:build !linux || android +//go:build !(linux && !android) /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn @@ -22,7 +22,7 @@ func (e *StdNetEndpoint) SrcToString() string { } // TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets -// {get,set}srcControl feature set, but use alternatively named flags and need +// ({get,set}srcControl feature set, but use alternatively named flags and need // ports and require testing. // getSrcFromControl parses the control for PKTINFO and if found updates ep with @@ -35,8 +35,17 @@ func getSrcFromControl(control []byte, ep *StdNetEndpoint) { func setSrcControl(control *[]byte, ep *StdNetEndpoint) { } -// stickyControlSize returns the recommended buffer size for pooling sticky +// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. +func getGSOSize(control []byte) (int, error) { + return 0, nil +} + +// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. +func setGSOSize(control *[]byte, gsoSize uint16) { +} + +// controlSize returns the recommended buffer size for pooling sticky and UDP // offloading control data. -const stickyControlSize = 0 +const controlSize = 0 const StdNetSupportsStickySockets = false diff --git a/conn/sticky_linux.go b/conn/control_linux.go similarity index 64% rename from conn/sticky_linux.go rename to conn/control_linux.go index adfedc1..f32f26a 100644 --- a/conn/sticky_linux.go +++ b/conn/control_linux.go @@ -2,12 +2,13 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn import ( + "fmt" "net/netip" "unsafe" @@ -105,8 +106,54 @@ func setSrcControl(control *[]byte, ep *StdNetEndpoint) { *control = append(*control, ep.src...) } -// stickyControlSize returns the recommended buffer size for pooling sticky +const ( + sizeOfGSOData = 2 +) + +// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. +func getGSOSize(control []byte) (int, error) { + var ( + hdr unix.Cmsghdr + data []byte + rem = control + err error + ) + + for len(rem) > unix.SizeofCmsghdr { + hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem) + if err != nil { + return 0, fmt.Errorf("error parsing socket control message: %w", err) + } + if hdr.Level == socketOptionLevelUDP && hdr.Type == socketOptionUDPGRO && len(data) >= sizeOfGSOData { + var gso uint16 + copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData]) + return int(gso), nil + } + } + return 0, nil +} + +// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing +// data in control untouched. +func setGSOSize(control *[]byte, gsoSize uint16) { + existingLen := len(*control) + avail := cap(*control) - existingLen + space := unix.CmsgSpace(sizeOfGSOData) + if avail < space { + return + } + *control = (*control)[:cap(*control)] + gsoControl := (*control)[existingLen:] + hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0])) + hdr.Level = socketOptionLevelUDP + hdr.Type = socketOptionUDPSegment + hdr.SetLen(unix.CmsgLen(sizeOfGSOData)) + copy((gsoControl)[unix.SizeofCmsghdr:], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData)) + *control = (*control)[:existingLen+space] +} + +// controlSize returns the recommended buffer size for pooling sticky and UDP // offloading control data. -var stickyControlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) +var controlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) + unix.CmsgSpace(sizeOfGSOData) const StdNetSupportsStickySockets = true diff --git a/conn/controlfns_linux.go b/conn/controlfns_linux.go index f0deefa..ff591f1 100644 --- a/conn/controlfns_linux.go +++ b/conn/controlfns_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn @@ -13,35 +13,6 @@ import ( "golang.org/x/sys/unix" ) -// Taken from go/src/internal/syscall/unix/kernel_version_linux.go -func kernelVersion() (major, minor int) { - var uname unix.Utsname - if err := unix.Uname(&uname); err != nil { - return - } - - var ( - values [2]int - value, vi int - ) - for _, c := range uname.Release { - if '0' <= c && c <= '9' { - value = (value * 10) + int(c-'0') - } else { - // Note that we're assuming N.N.N here. - // If we see anything else, we are likely to mis-parse it. - values[vi] = value - vi++ - if vi >= len(values) { - break - } - value = 0 - } - } - - return values[0], values[1] -} - func init() { controlFns = append(controlFns, @@ -89,19 +60,17 @@ func init() { // Attempt to enable UDP_GRO func(network, address string, c syscall.RawConn) error { - // Kernels below 5.12 are missing 98184612aca0 ("net: - // udp: Add support for getsockopt(..., ..., UDP_GRO, - // ..., ...);"), which means we can't read this back - // later. We could pipe the return value through to - // the rest of the code, but UDP_GRO is kind of buggy - // anyway, so just gate this here. - major, minor := kernelVersion() - if major < 5 || (major == 5 && minor < 12) { + // lx(010): skip UDP_GRO on android. The GRO receive path in bind_std.go + // is gated on runtime.GOOS=="linux", which is false on android — so a + // coalesced super-packet is never split and corrupts the WG stream + // (download dies). Belt-and-suspenders with the rxOffload guard in + // features_linux.go. TX/GSO untouched. + // See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. + if runtime.GOOS == "android" { return nil } - c.Control(func(fd uintptr) { - _ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO, 1) + _ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO, 1) }) return nil }, diff --git a/conn/controlfns_unix.go b/conn/controlfns_unix.go index b2e7570..2bfccab 100644 --- a/conn/controlfns_unix.go +++ b/conn/controlfns_unix.go @@ -1,4 +1,4 @@ -//go:build !windows && !linux && !wasm +//go:build !windows && !linux && !wasm && !plan9 && !tamago /* SPDX-License-Identifier: MIT * diff --git a/conn/erraddrinuse.go b/conn/erraddrinuse.go new file mode 100644 index 0000000..a6563a4 --- /dev/null +++ b/conn/erraddrinuse.go @@ -0,0 +1,14 @@ +//go:build !plan9 + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + */ + +package conn + +import "syscall" + +func init() { + errEADDRINUSE = syscall.EADDRINUSE +} diff --git a/conn/errors_default.go b/conn/errors_default.go index 3c9b223..d967518 100644 --- a/conn/errors_default.go +++ b/conn/errors_default.go @@ -7,6 +7,6 @@ package conn -func errShouldDisableUDPGSO(_ error) bool { +func errShouldDisableUDPGSO(err error) bool { return false } diff --git a/conn/errors_linux.go b/conn/errors_linux.go index 037d820..8e61000 100644 --- a/conn/errors_linux.go +++ b/conn/errors_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn diff --git a/conn/features_default.go b/conn/features_default.go index 9fc5088..d53ff5f 100644 --- a/conn/features_default.go +++ b/conn/features_default.go @@ -3,13 +3,13 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn import "net" -func supportsUDPOffload(_ *net.UDPConn) (txOffload, rxOffload bool) { +func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { return } diff --git a/conn/features_linux.go b/conn/features_linux.go index 6386023..69dd2f4 100644 --- a/conn/features_linux.go +++ b/conn/features_linux.go @@ -1,26 +1,49 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package conn import ( "net" + "runtime" "golang.org/x/sys/unix" ) +const ( + // TODO: upstream to x/sys/unix + socketOptionLevelUDP = 17 + socketOptionUDPSegment = 103 + socketOptionUDPGRO = 104 +) + func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { rc, err := conn.SyscallConn() if err != nil { return } err = rc.Control(func(fd uintptr) { - _, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_SEGMENT) - txOffload = errSyscall == nil - opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO) - rxOffload = errSyscall == nil && opt == 1 + _, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPSegment) + if errSyscall != nil { + return + } + txOffload = true + // lx(010): never advertise RX offload on android. runtime.GOOS=="android" + // (not "linux"), so the GRO receive dispatcher in bind_std.go (gated on + // GOOS=="linux") is dead there — a coalesced GRO super-packet would be read + // as one datagram and corrupt the WG transport stream, killing download. + // Confirmed on device (CPH2411/Android-15: rxOffload=true, dispatch=single). + // TX is left untouched. See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. + if runtime.GOOS == "android" { + return + } + opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO) + if errSyscall != nil { + return + } + rxOffload = opt == 1 }) if err != nil { return false, false diff --git a/conn/gso_default.go b/conn/gso_default.go deleted file mode 100644 index a9a3e80..0000000 --- a/conn/gso_default.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !linux - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. - */ - -package conn - -// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. -func getGSOSize(control []byte) (int, error) { - return 0, nil -} - -// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. -func setGSOSize(control *[]byte, gsoSize uint16) { -} - -// gsoControlSize returns the recommended buffer size for pooling sticky and UDP -// offloading control data. -const gsoControlSize = 0 diff --git a/conn/gso_linux.go b/conn/gso_linux.go deleted file mode 100644 index 4ee31fa..0000000 --- a/conn/gso_linux.go +++ /dev/null @@ -1,65 +0,0 @@ -//go:build linux - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. - */ - -package conn - -import ( - "fmt" - "unsafe" - - "golang.org/x/sys/unix" -) - -const ( - sizeOfGSOData = 2 -) - -// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. -func getGSOSize(control []byte) (int, error) { - var ( - hdr unix.Cmsghdr - data []byte - rem = control - err error - ) - - for len(rem) > unix.SizeofCmsghdr { - hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem) - if err != nil { - return 0, fmt.Errorf("error parsing socket control message: %w", err) - } - if hdr.Level == unix.SOL_UDP && hdr.Type == unix.UDP_GRO && len(data) >= sizeOfGSOData { - var gso uint16 - copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData]) - return int(gso), nil - } - } - return 0, nil -} - -// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing -// data in control untouched. -func setGSOSize(control *[]byte, gsoSize uint16) { - existingLen := len(*control) - avail := cap(*control) - existingLen - space := unix.CmsgSpace(sizeOfGSOData) - if avail < space { - return - } - *control = (*control)[:cap(*control)] - gsoControl := (*control)[existingLen:] - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0])) - hdr.Level = unix.SOL_UDP - hdr.Type = unix.UDP_SEGMENT - hdr.SetLen(unix.CmsgLen(sizeOfGSOData)) - copy((gsoControl)[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData)) - *control = (*control)[:existingLen+space] -} - -// gsoControlSize returns the recommended buffer size for pooling UDP -// offloading control data. -var gsoControlSize = unix.CmsgSpace(sizeOfGSOData) diff --git a/device/keypair.go b/device/keypair.go index 0704748..fb3d392 100644 --- a/device/keypair.go +++ b/device/keypair.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package device diff --git a/device/queueconstants_android.go b/device/queueconstants_android.go index a3bee69..d0718d2 100644 --- a/device/queueconstants_android.go +++ b/device/queueconstants_android.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package device @@ -14,6 +14,6 @@ const ( QueueOutboundSize = 1024 QueueInboundSize = 1024 QueueHandshakeSize = 1024 - MaxSegmentSize = (1 << 16) - 1 // largest possible UDP datagram + MaxSegmentSize = 2200 PreallocatedBuffersPerPool = 4096 ) diff --git a/device/queueconstants_default.go b/device/queueconstants_default.go index 1d09285..79b74b0 100644 --- a/device/queueconstants_default.go +++ b/device/queueconstants_default.go @@ -2,7 +2,7 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package device diff --git a/device/receive.go b/device/receive.go index 93117c4..d9a0f69 100644 --- a/device/receive.go +++ b/device/receive.go @@ -96,13 +96,13 @@ func (device *Device) RoutineReceiveIncoming( elemsByPeer = make(map[*Peer]*QueueInboundElementsContainer, maxBatchSize) ) - for i := range bufsArrs { + for i := range maxBatchSize { bufsArrs[i] = device.GetMessageBuffer() bufs[i] = bufsArrs[i][:] } defer func() { - for i := 0; i < maxBatchSize; i++ { + for i := range maxBatchSize { if bufsArrs[i] != nil { device.PutMessageBuffer(bufsArrs[i]) } diff --git a/device/sticky_default.go b/device/sticky_default.go index cac7add..0d02174 100644 --- a/device/sticky_default.go +++ b/device/sticky_default.go @@ -7,6 +7,6 @@ import ( "github.com/sagernet/wireguard-go/rwcancel" ) -func (device *Device) startRouteListener(_ conn.Bind) (*rwcancel.RWCancel, error) { +func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { return nil, nil } diff --git a/device/sticky_linux.go b/device/sticky_linux.go index 9fcfeeb..b5b0e51 100644 --- a/device/sticky_linux.go +++ b/device/sticky_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. * * This implements userspace semantics of "sticky sockets", modeled after * WireGuard's kernelspace implementation. This is more or less a straight port @@ -9,7 +9,7 @@ * * Currently there is no way to achieve this within the net package: * See e.g. https://github.com/golang/go/issues/17930 - * So this code remains platform dependent. + * So this code is remains platform dependent. */ package device @@ -46,7 +46,7 @@ func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, er return netlinkCancel, nil } -func (device *Device) routineRouteListener(_ conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) { +func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) { type peerEndpointPtr struct { peer *Peer endpoint *conn.Endpoint diff --git a/device/tun.go b/device/tun.go index 01a92ed..891575d 100644 --- a/device/tun.go +++ b/device/tun.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package device diff --git a/go.mod b/go.mod index 773bf00..8679506 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/sagernet/wireguard-go -go 1.20 +go 1.24 require ( github.com/sagernet/sing v0.7.10 golang.org/x/crypto v0.13.0 golang.org/x/net v0.15.0 - golang.org/x/sys v0.21.0 + golang.org/x/sys v0.12.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 ) diff --git a/go.sum b/go.sum index 9ce9725..c405e0a 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,7 @@ golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= diff --git a/ipc/uapi_fake.go b/ipc/uapi_fake.go new file mode 100644 index 0000000..a2e0f85 --- /dev/null +++ b/ipc/uapi_fake.go @@ -0,0 +1,17 @@ +//go:build wasm || plan9 || aix || solaris || illumos + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package ipc + +// Made up sentinel error codes for {js,wasip1}/wasm, and plan9. +const ( + IpcErrorIO = 1 + IpcErrorInvalid = 2 + IpcErrorPortInUse = 3 + IpcErrorUnknown = 4 + IpcErrorProtocol = 5 +) diff --git a/ipc/uapi_linux.go b/ipc/uapi_linux.go index be59e58..c14d5d0 100644 --- a/ipc/uapi_linux.go +++ b/ipc/uapi_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package ipc diff --git a/ipc/uapi_wasm.go b/ipc/uapi_tamago.go similarity index 76% rename from ipc/uapi_wasm.go rename to ipc/uapi_tamago.go index 50ac091..3584a4d 100644 --- a/ipc/uapi_wasm.go +++ b/ipc/uapi_tamago.go @@ -1,3 +1,5 @@ +//go:build tamago + /* SPDX-License-Identifier: MIT * * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. @@ -5,7 +7,7 @@ package ipc -// Made up sentinel error codes for {js,wasip1}/wasm. +// Made up sentinel error codes for tamago platform. const ( IpcErrorIO = 1 IpcErrorInvalid = 2 diff --git a/ipc/uapi_unix.go b/ipc/uapi_unix.go index dcce167..79604ee 100644 --- a/ipc/uapi_unix.go +++ b/ipc/uapi_unix.go @@ -26,7 +26,7 @@ const ( // socketDirectory is variable because it is modified by a linker // flag in wireguard-android. -var socketDirectory = "/var/run/wireguard" +var socketDirectory = "/var/run/amneziawg" func sockPath(iface string) string { return fmt.Sprintf("%s/%s.sock", socketDirectory, iface) diff --git a/ipc/uapi_windows.go b/ipc/uapi_windows.go index a146f1a..5d236b3 100644 --- a/ipc/uapi_windows.go +++ b/ipc/uapi_windows.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package ipc diff --git a/module_rename.py b/module_rename.py new file mode 100644 index 0000000..bf03152 --- /dev/null +++ b/module_rename.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import argparse +import fileinput + + +PKG_ORIGINAL = "github.com/tailscale/wireguard-go" +PKG_NEW = "github.com/sagernet/wireguard-go" + +EXTENSIONS = [".go", ".md", ".mod", ".sh"] + +parser = argparse.ArgumentParser() +parser.add_argument("-r", "--reverse", action="store_true") +args = parser.parse_args() + + +def replace_line(line): + if args.reverse: + return line.replace(PKG_NEW, PKG_ORIGINAL) + return line.replace(PKG_ORIGINAL, PKG_NEW) + + +for dirpath, dirnames, filenames in os.walk("."): + # Skip hidden directories like .git + dirnames[:] = [d for d in dirnames if not d[0] == "."] + filenames = [f for f in filenames if os.path.splitext(f)[1] in EXTENSIONS] + for filename in filenames: + file_path = os.path.join(dirpath, filename) + with fileinput.FileInput(file_path, inplace=True) as file: + for line in file: + print(replace_line(line), end="") diff --git a/reformat.sh b/reformat.sh deleted file mode 100755 index 8d3ac05..0000000 --- a/reformat.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -e -o pipefail - -GO_FILES=$(find . -name "*.go" | grep -v .git) - -gofumpt -l -w $GO_FILES -gofmt -l -w $GO_FILES -gci write $GO_FILES diff --git a/rename-module.sh b/rename-module.sh deleted file mode 100755 index a6bd016..0000000 --- a/rename-module.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -e -o pipefail - -OLD_MODULE_NAME="github.com/tailscale/wireguard-go" -NEW_MODULE_NAME="github.com/sagernet/wireguard-go" - -rules=$(cat <.*) - by: $NEW_MODULE_NAME\$PATH -fix: \$NEW_IMPORT -EOF -) - -sg scan --inline-rules "$rules" -U - -sed -i "s|module $OLD_MODULE_NAME|module $NEW_MODULE_NAME|" go.mod - -go mod tidy - -./reformat.sh - -git commit -m "Rename module" -a diff --git a/rwcancel/rwcancel.go b/rwcancel/rwcancel.go index 4372453..a4668f0 100644 --- a/rwcancel/rwcancel.go +++ b/rwcancel/rwcancel.go @@ -1,4 +1,4 @@ -//go:build !windows && !wasm +//go:build !windows && !wasm && !plan9 && !tamago /* SPDX-License-Identifier: MIT * diff --git a/rwcancel/rwcancel_stub.go b/rwcancel/rwcancel_stub.go index 2a98b2b..60ae9af 100644 --- a/rwcancel/rwcancel_stub.go +++ b/rwcancel/rwcancel_stub.go @@ -1,4 +1,4 @@ -//go:build windows || wasm +//go:build windows || wasm || plan9 || tamago // SPDX-License-Identifier: MIT diff --git a/tun/checksum.go b/tun/checksum.go index ac16569..6634050 100644 --- a/tun/checksum.go +++ b/tun/checksum.go @@ -3,111 +3,710 @@ package tun import ( "encoding/binary" "math/bits" + "strconv" + + "golang.org/x/sys/cpu" ) -// TODO: Explore SIMD and/or other assembly optimizations. -func checksumNoFold(b []byte, initial uint64) uint64 { - tmp := make([]byte, 8) - binary.NativeEndian.PutUint64(tmp, initial) - ac := binary.BigEndian.Uint64(tmp) +// checksumGeneric64 is a reference implementation of checksum using 64 bit +// arithmetic for use in testing or when an architecture-specific implementation +// is not available. +func checksumGeneric64(b []byte, initial uint16) uint16 { + var ac uint64 var carry uint64 + if cpu.IsBigEndian { + ac = uint64(initial) + } else { + ac = uint64(bits.ReverseBytes16(initial)) + } + for len(b) >= 128 { - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[32:40]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[40:48]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[48:56]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[56:64]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[64:72]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[72:80]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[80:88]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[88:96]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[96:104]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[104:112]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[112:120]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[120:128]), carry) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[64:72]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[72:80]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[80:88]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[88:96]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[96:104]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[104:112]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[112:120]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[120:128]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[64:72]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[72:80]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[80:88]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[88:96]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[96:104]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[104:112]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[112:120]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[120:128]), carry) + } b = b[128:] } if len(b) >= 64 { - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[32:40]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[40:48]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[48:56]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[56:64]), carry) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry) + } b = b[64:] } if len(b) >= 32 { - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) + } b = b[32:] } if len(b) >= 16 { - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + } b = b[16:] } if len(b) >= 8 { - ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b), carry) + } b = b[8:] } if len(b) >= 4 { - ac, carry = bits.Add64(ac, uint64(binary.NativeEndian.Uint32(b[:4])), 0) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint32(b)), carry) + } else { + ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint32(b)), carry) + } b = b[4:] } if len(b) >= 2 { - ac, carry = bits.Add64(ac, uint64(binary.NativeEndian.Uint16(b[:2])), 0) - ac += carry + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint16(b)), carry) + } else { + ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint16(b)), carry) + } b = b[2:] } - if len(b) == 1 { - tmp := binary.NativeEndian.Uint16([]byte{b[0], 0}) - ac, carry = bits.Add64(ac, uint64(tmp), 0) - ac += carry + if len(b) >= 1 { + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, uint64(b[0])<<8, carry) + } else { + ac, carry = bits.Add64(ac, uint64(b[0]), carry) + } } - binary.NativeEndian.PutUint64(tmp, ac) - return binary.BigEndian.Uint64(tmp) + folded := ipChecksumFold64(ac, carry) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded } -func checksum(b []byte, initial uint64) uint16 { - ac := checksumNoFold(b, initial) - ac = (ac >> 16) + (ac & 0xffff) - ac = (ac >> 16) + (ac & 0xffff) - ac = (ac >> 16) + (ac & 0xffff) - ac = (ac >> 16) + (ac & 0xffff) - return uint16(ac) +// checksumGeneric32 is a reference implementation of checksum using 32 bit +// arithmetic for use in testing or when an architecture-specific implementation +// is not available. +func checksumGeneric32(b []byte, initial uint16) uint16 { + var ac uint32 + var carry uint32 + + if cpu.IsBigEndian { + ac = uint32(initial) + } else { + ac = uint32(bits.ReverseBytes16(initial)) + } + + for len(b) >= 64 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[32:36]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[36:40]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[40:44]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[44:48]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[48:52]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[52:56]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[56:60]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[60:64]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[32:36]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[36:40]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[40:44]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[44:48]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[48:52]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[52:56]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[56:60]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[60:64]), carry) + } + b = b[64:] + } + if len(b) >= 32 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry) + } + b = b[32:] + } + if len(b) >= 16 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) + } + b = b[16:] + } + if len(b) >= 8 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + } + b = b[8:] + } + if len(b) >= 4 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b), carry) + } + b = b[4:] + } + if len(b) >= 2 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, uint32(binary.BigEndian.Uint16(b)), carry) + } else { + ac, carry = bits.Add32(ac, uint32(binary.LittleEndian.Uint16(b)), carry) + } + b = b[2:] + } + if len(b) >= 1 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, uint32(b[0])<<8, carry) + } else { + ac, carry = bits.Add32(ac, uint32(b[0]), carry) + } + } + + folded := ipChecksumFold32(ac, carry) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded } -// Checksum computes an IP checksum starting with the provided initial value. -func Checksum(data []byte, initial uint16) uint16 { - return checksum(data, uint64(initial)) +// checksumGeneric32Alternate is an alternate reference implementation of +// checksum using 32 bit arithmetic for use in testing or when an +// architecture-specific implementation is not available. +func checksumGeneric32Alternate(b []byte, initial uint16) uint16 { + var ac uint32 + + if cpu.IsBigEndian { + ac = uint32(initial) + } else { + ac = uint32(bits.ReverseBytes16(initial)) + } + + for len(b) >= 64 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + ac += uint32(binary.BigEndian.Uint16(b[8:10])) + ac += uint32(binary.BigEndian.Uint16(b[10:12])) + ac += uint32(binary.BigEndian.Uint16(b[12:14])) + ac += uint32(binary.BigEndian.Uint16(b[14:16])) + ac += uint32(binary.BigEndian.Uint16(b[16:18])) + ac += uint32(binary.BigEndian.Uint16(b[18:20])) + ac += uint32(binary.BigEndian.Uint16(b[20:22])) + ac += uint32(binary.BigEndian.Uint16(b[22:24])) + ac += uint32(binary.BigEndian.Uint16(b[24:26])) + ac += uint32(binary.BigEndian.Uint16(b[26:28])) + ac += uint32(binary.BigEndian.Uint16(b[28:30])) + ac += uint32(binary.BigEndian.Uint16(b[30:32])) + ac += uint32(binary.BigEndian.Uint16(b[32:34])) + ac += uint32(binary.BigEndian.Uint16(b[34:36])) + ac += uint32(binary.BigEndian.Uint16(b[36:38])) + ac += uint32(binary.BigEndian.Uint16(b[38:40])) + ac += uint32(binary.BigEndian.Uint16(b[40:42])) + ac += uint32(binary.BigEndian.Uint16(b[42:44])) + ac += uint32(binary.BigEndian.Uint16(b[44:46])) + ac += uint32(binary.BigEndian.Uint16(b[46:48])) + ac += uint32(binary.BigEndian.Uint16(b[48:50])) + ac += uint32(binary.BigEndian.Uint16(b[50:52])) + ac += uint32(binary.BigEndian.Uint16(b[52:54])) + ac += uint32(binary.BigEndian.Uint16(b[54:56])) + ac += uint32(binary.BigEndian.Uint16(b[56:58])) + ac += uint32(binary.BigEndian.Uint16(b[58:60])) + ac += uint32(binary.BigEndian.Uint16(b[60:62])) + ac += uint32(binary.BigEndian.Uint16(b[62:64])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + ac += uint32(binary.LittleEndian.Uint16(b[8:10])) + ac += uint32(binary.LittleEndian.Uint16(b[10:12])) + ac += uint32(binary.LittleEndian.Uint16(b[12:14])) + ac += uint32(binary.LittleEndian.Uint16(b[14:16])) + ac += uint32(binary.LittleEndian.Uint16(b[16:18])) + ac += uint32(binary.LittleEndian.Uint16(b[18:20])) + ac += uint32(binary.LittleEndian.Uint16(b[20:22])) + ac += uint32(binary.LittleEndian.Uint16(b[22:24])) + ac += uint32(binary.LittleEndian.Uint16(b[24:26])) + ac += uint32(binary.LittleEndian.Uint16(b[26:28])) + ac += uint32(binary.LittleEndian.Uint16(b[28:30])) + ac += uint32(binary.LittleEndian.Uint16(b[30:32])) + ac += uint32(binary.LittleEndian.Uint16(b[32:34])) + ac += uint32(binary.LittleEndian.Uint16(b[34:36])) + ac += uint32(binary.LittleEndian.Uint16(b[36:38])) + ac += uint32(binary.LittleEndian.Uint16(b[38:40])) + ac += uint32(binary.LittleEndian.Uint16(b[40:42])) + ac += uint32(binary.LittleEndian.Uint16(b[42:44])) + ac += uint32(binary.LittleEndian.Uint16(b[44:46])) + ac += uint32(binary.LittleEndian.Uint16(b[46:48])) + ac += uint32(binary.LittleEndian.Uint16(b[48:50])) + ac += uint32(binary.LittleEndian.Uint16(b[50:52])) + ac += uint32(binary.LittleEndian.Uint16(b[52:54])) + ac += uint32(binary.LittleEndian.Uint16(b[54:56])) + ac += uint32(binary.LittleEndian.Uint16(b[56:58])) + ac += uint32(binary.LittleEndian.Uint16(b[58:60])) + ac += uint32(binary.LittleEndian.Uint16(b[60:62])) + ac += uint32(binary.LittleEndian.Uint16(b[62:64])) + } + b = b[64:] + } + if len(b) >= 32 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + ac += uint32(binary.BigEndian.Uint16(b[8:10])) + ac += uint32(binary.BigEndian.Uint16(b[10:12])) + ac += uint32(binary.BigEndian.Uint16(b[12:14])) + ac += uint32(binary.BigEndian.Uint16(b[14:16])) + ac += uint32(binary.BigEndian.Uint16(b[16:18])) + ac += uint32(binary.BigEndian.Uint16(b[18:20])) + ac += uint32(binary.BigEndian.Uint16(b[20:22])) + ac += uint32(binary.BigEndian.Uint16(b[22:24])) + ac += uint32(binary.BigEndian.Uint16(b[24:26])) + ac += uint32(binary.BigEndian.Uint16(b[26:28])) + ac += uint32(binary.BigEndian.Uint16(b[28:30])) + ac += uint32(binary.BigEndian.Uint16(b[30:32])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + ac += uint32(binary.LittleEndian.Uint16(b[8:10])) + ac += uint32(binary.LittleEndian.Uint16(b[10:12])) + ac += uint32(binary.LittleEndian.Uint16(b[12:14])) + ac += uint32(binary.LittleEndian.Uint16(b[14:16])) + ac += uint32(binary.LittleEndian.Uint16(b[16:18])) + ac += uint32(binary.LittleEndian.Uint16(b[18:20])) + ac += uint32(binary.LittleEndian.Uint16(b[20:22])) + ac += uint32(binary.LittleEndian.Uint16(b[22:24])) + ac += uint32(binary.LittleEndian.Uint16(b[24:26])) + ac += uint32(binary.LittleEndian.Uint16(b[26:28])) + ac += uint32(binary.LittleEndian.Uint16(b[28:30])) + ac += uint32(binary.LittleEndian.Uint16(b[30:32])) + } + b = b[32:] + } + if len(b) >= 16 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + ac += uint32(binary.BigEndian.Uint16(b[8:10])) + ac += uint32(binary.BigEndian.Uint16(b[10:12])) + ac += uint32(binary.BigEndian.Uint16(b[12:14])) + ac += uint32(binary.BigEndian.Uint16(b[14:16])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + ac += uint32(binary.LittleEndian.Uint16(b[8:10])) + ac += uint32(binary.LittleEndian.Uint16(b[10:12])) + ac += uint32(binary.LittleEndian.Uint16(b[12:14])) + ac += uint32(binary.LittleEndian.Uint16(b[14:16])) + } + b = b[16:] + } + if len(b) >= 8 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + } + b = b[8:] + } + if len(b) >= 4 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + } + b = b[4:] + } + if len(b) >= 2 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b)) + } else { + ac += uint32(binary.LittleEndian.Uint16(b)) + } + b = b[2:] + } + if len(b) >= 1 { + if cpu.IsBigEndian { + ac += uint32(b[0]) << 8 + } else { + ac += uint32(b[0]) + } + } + + folded := ipChecksumFold32(ac, 0) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded } -func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint64 { - sum := checksumNoFold(srcAddr, 0) - sum = checksumNoFold(dstAddr, sum) - sum = checksumNoFold([]byte{0, protocol}, sum) - tmp := make([]byte, 2) - binary.BigEndian.PutUint16(tmp, totalLen) - return checksumNoFold(tmp, sum) +// checksumGeneric64Alternate is an alternate reference implementation of +// checksum using 64 bit arithmetic for use in testing or when an +// architecture-specific implementation is not available. +func checksumGeneric64Alternate(b []byte, initial uint16) uint16 { + var ac uint64 + + if cpu.IsBigEndian { + ac = uint64(initial) + } else { + ac = uint64(bits.ReverseBytes16(initial)) + } + + for len(b) >= 64 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + ac += uint64(binary.BigEndian.Uint32(b[32:36])) + ac += uint64(binary.BigEndian.Uint32(b[36:40])) + ac += uint64(binary.BigEndian.Uint32(b[40:44])) + ac += uint64(binary.BigEndian.Uint32(b[44:48])) + ac += uint64(binary.BigEndian.Uint32(b[48:52])) + ac += uint64(binary.BigEndian.Uint32(b[52:56])) + ac += uint64(binary.BigEndian.Uint32(b[56:60])) + ac += uint64(binary.BigEndian.Uint32(b[60:64])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + ac += uint64(binary.LittleEndian.Uint32(b[8:12])) + ac += uint64(binary.LittleEndian.Uint32(b[12:16])) + ac += uint64(binary.LittleEndian.Uint32(b[16:20])) + ac += uint64(binary.LittleEndian.Uint32(b[20:24])) + ac += uint64(binary.LittleEndian.Uint32(b[24:28])) + ac += uint64(binary.LittleEndian.Uint32(b[28:32])) + ac += uint64(binary.LittleEndian.Uint32(b[32:36])) + ac += uint64(binary.LittleEndian.Uint32(b[36:40])) + ac += uint64(binary.LittleEndian.Uint32(b[40:44])) + ac += uint64(binary.LittleEndian.Uint32(b[44:48])) + ac += uint64(binary.LittleEndian.Uint32(b[48:52])) + ac += uint64(binary.LittleEndian.Uint32(b[52:56])) + ac += uint64(binary.LittleEndian.Uint32(b[56:60])) + ac += uint64(binary.LittleEndian.Uint32(b[60:64])) + } + b = b[64:] + } + if len(b) >= 32 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + ac += uint64(binary.LittleEndian.Uint32(b[8:12])) + ac += uint64(binary.LittleEndian.Uint32(b[12:16])) + ac += uint64(binary.LittleEndian.Uint32(b[16:20])) + ac += uint64(binary.LittleEndian.Uint32(b[20:24])) + ac += uint64(binary.LittleEndian.Uint32(b[24:28])) + ac += uint64(binary.LittleEndian.Uint32(b[28:32])) + } + b = b[32:] + } + if len(b) >= 16 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + ac += uint64(binary.LittleEndian.Uint32(b[8:12])) + ac += uint64(binary.LittleEndian.Uint32(b[12:16])) + } + b = b[16:] + } + if len(b) >= 8 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + } + b = b[8:] + } + if len(b) >= 4 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b)) + } else { + ac += uint64(binary.LittleEndian.Uint32(b)) + } + b = b[4:] + } + if len(b) >= 2 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint16(b)) + } else { + ac += uint64(binary.LittleEndian.Uint16(b)) + } + b = b[2:] + } + if len(b) >= 1 { + if cpu.IsBigEndian { + ac += uint64(b[0]) << 8 + } else { + ac += uint64(b[0]) + } + } + + folded := ipChecksumFold64(ac, 0) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded +} + +func ipChecksumFold64(unfolded uint64, initialCarry uint64) uint16 { + sum, carry := bits.Add32(uint32(unfolded>>32), uint32(unfolded&0xffff_ffff), uint32(initialCarry)) + // if carry != 0, sum <= 0xffff_fffe, otherwise sum <= 0xffff_ffff + // therefore (sum >> 16) + (sum & 0xffff) + carry <= 0x1_fffe; so there is + // no need to save the carry flag + sum = (sum >> 16) + (sum & 0xffff) + carry + // sum <= 0x1_fffe therefore this is the last fold needed: + // if (sum >> 16) > 0 then + // (sum >> 16) == 1 && (sum & 0xffff) <= 0xfffe and therefore + // the addition will not overflow + // otherwise (sum >> 16) == 0 and sum will be unchanged + sum = (sum >> 16) + (sum & 0xffff) + return uint16(sum) +} + +func ipChecksumFold32(unfolded uint32, initialCarry uint32) uint16 { + sum := (unfolded >> 16) + (unfolded & 0xffff) + initialCarry + // sum <= 0x1_ffff: + // 0xffff + 0xffff = 0x1_fffe + // initialCarry is 0 or 1, for a combined maximum of 0x1_ffff + sum = (sum >> 16) + (sum & 0xffff) + // sum <= 0x1_0000 therefore this is the last fold needed: + // if (sum >> 16) > 0 then + // (sum >> 16) == 1 && (sum & 0xffff) == 0 and therefore + // the addition will not overflow + // otherwise (sum >> 16) == 0 and sum will be unchanged + sum = (sum >> 16) + (sum & 0xffff) + return uint16(sum) +} + +func addrPartialChecksum64(addr []byte, initial, carryIn uint64) (sum, carry uint64) { + sum, carry = initial, carryIn + switch len(addr) { + case 4: // IPv4 + if cpu.IsBigEndian { + sum, carry = bits.Add64(sum, uint64(binary.BigEndian.Uint32(addr)), carry) + } else { + sum, carry = bits.Add64(sum, uint64(binary.LittleEndian.Uint32(addr)), carry) + } + case 16: // IPv6 + if cpu.IsBigEndian { + sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr), carry) + sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr[8:]), carry) + } else { + sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr), carry) + sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr[8:]), carry) + } + default: + panic("bad addr length") + } + return sum, carry +} + +func addrPartialChecksum32(addr []byte, initial, carryIn uint32) (sum, carry uint32) { + sum, carry = initial, carryIn + switch len(addr) { + case 4: // IPv4 + if cpu.IsBigEndian { + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry) + } else { + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry) + } + case 16: // IPv6 + if cpu.IsBigEndian { + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry) + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[4:8]), carry) + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[8:12]), carry) + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[12:16]), carry) + } else { + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry) + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[4:8]), carry) + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[8:12]), carry) + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[12:16]), carry) + } + default: + panic("bad addr length") + } + return sum, carry +} + +func pseudoHeaderChecksum64(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { + var sum uint64 + if cpu.IsBigEndian { + sum = uint64(totalLen) + uint64(protocol) + } else { + sum = uint64(bits.ReverseBytes16(totalLen)) + uint64(protocol)<<8 + } + sum, carry := addrPartialChecksum64(srcAddr, sum, 0) + sum, carry = addrPartialChecksum64(dstAddr, sum, carry) + + foldedSum := ipChecksumFold64(sum, carry) + if !cpu.IsBigEndian { + foldedSum = bits.ReverseBytes16(foldedSum) + } + return foldedSum +} + +func pseudoHeaderChecksum32(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { + var sum uint32 + if cpu.IsBigEndian { + sum = uint32(totalLen) + uint32(protocol) + } else { + sum = uint32(bits.ReverseBytes16(totalLen)) + uint32(protocol)<<8 + } + sum, carry := addrPartialChecksum32(srcAddr, sum, 0) + sum, carry = addrPartialChecksum32(dstAddr, sum, carry) + + foldedSum := ipChecksumFold32(sum, carry) + if !cpu.IsBigEndian { + foldedSum = bits.ReverseBytes16(foldedSum) + } + return foldedSum } // PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and // dstAddr must be 4 or 16 bytes in length. func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { - return checksum([]byte{}, pseudoHeaderChecksumNoFold(protocol, srcAddr, dstAddr, totalLen)) + if strconv.IntSize < 64 { + return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen) + } + return pseudoHeaderChecksum64(protocol, srcAddr, dstAddr, totalLen) } diff --git a/tun/checksum_amd64.go b/tun/checksum_amd64.go new file mode 100644 index 0000000..4fb684e --- /dev/null +++ b/tun/checksum_amd64.go @@ -0,0 +1,23 @@ +package tun + +import "golang.org/x/sys/cpu" + +var checksum = checksumAMD64 + +// Checksum computes an IP checksum starting with the provided initial value. +// The length of data should be at least 128 bytes for best performance. Smaller +// buffers will still compute a correct result. +func Checksum(data []byte, initial uint16) uint16 { + return checksum(data, initial) +} + +func init() { + if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { + checksum = checksumAVX2 + return + } + if cpu.X86.HasSSE2 { + checksum = checksumSSE2 + return + } +} diff --git a/tun/checksum_generated_amd64.go b/tun/checksum_generated_amd64.go new file mode 100644 index 0000000..b4a2941 --- /dev/null +++ b/tun/checksum_generated_amd64.go @@ -0,0 +1,18 @@ +// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT. + +package tun + +// checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2) +// +//go:noescape +func checksumAVX2(b []byte, initial uint16) uint16 + +// checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2) +// +//go:noescape +func checksumSSE2(b []byte, initial uint16) uint16 + +// checksumAMD64 computes an IP checksum using amd64 baseline instructions +// +//go:noescape +func checksumAMD64(b []byte, initial uint16) uint16 diff --git a/tun/checksum_generated_amd64.s b/tun/checksum_generated_amd64.s new file mode 100644 index 0000000..5f2e4c5 --- /dev/null +++ b/tun/checksum_generated_amd64.s @@ -0,0 +1,851 @@ +// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT. + +#include "textflag.h" + +DATA xmmLoadMasks<>+0(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff" +DATA xmmLoadMasks<>+16(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff" +DATA xmmLoadMasks<>+32(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+48(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+64(SB)/16, $"\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+80(SB)/16, $"\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+96(SB)/16, $"\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +GLOBL xmmLoadMasks<>(SB), RODATA|NOPTR, $112 + +// func checksumAVX2(b []byte, initial uint16) uint16 +// Requires: AVX, AVX2, BMI2 +TEXT ·checksumAVX2(SB), NOSPLIT|NOFRAME, $0-34 + MOVWQZX initial+24(FP), AX + XCHGB AH, AL + MOVQ b_base+0(FP), DX + MOVQ b_len+8(FP), BX + + // handle odd length buffers; they are difficult to handle in general + TESTQ $0x00000001, BX + JZ lengthIsEven + MOVBQZX -1(DX)(BX*1), CX + DECQ BX + ADDQ CX, AX + +lengthIsEven: + // handle tiny buffers (<=31 bytes) specially + CMPQ BX, $0x1f + JGT bufferIsNotTiny + XORQ CX, CX + XORQ SI, SI + XORQ DI, DI + + // shift twice to start because length is guaranteed to be even + // n = n >> 2; CF = originalN & 2 + SHRQ $0x02, BX + JNC handleTiny4 + + // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] + MOVWQZX (DX), CX + ADDQ $0x02, DX + +handleTiny4: + // n = n >> 1; CF = originalN & 4 + SHRQ $0x01, BX + JNC handleTiny8 + + // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] + MOVLQZX (DX), SI + ADDQ $0x04, DX + +handleTiny8: + // n = n >> 1; CF = originalN & 8 + SHRQ $0x01, BX + JNC handleTiny16 + + // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] + MOVQ (DX), DI + ADDQ $0x08, DX + +handleTiny16: + // n = n >> 1; CF = originalN & 16 + // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize + SHRQ $0x01, BX + JNC handleTinyFinish + ADDQ (DX), AX + ADCQ 8(DX), AX + +handleTinyFinish: + // CF should be included from the previous add, so we use ADCQ. + // If we arrived via the JNC above, then CF=0 due to the branch condition, + // so ADCQ will still produce the correct result. + ADCQ CX, AX + ADCQ SI, AX + ADCQ DI, AX + JMP foldAndReturn + +bufferIsNotTiny: + // skip all SIMD for small buffers + CMPQ BX, $0x00000100 + JGE startSIMD + + // Accumulate carries in this register. It is never expected to overflow. + XORQ SI, SI + + // We will perform an overlapped read for buffers with length not a multiple of 8. + // Overlapped in this context means some memory will be read twice, but a shift will + // eliminate the duplicated data. This extra read is performed at the end of the buffer to + // preserve any alignment that may exist for the start of the buffer. + MOVQ BX, CX + SHRQ $0x03, BX + ANDQ $0x07, CX + JZ handleRemaining8 + LEAQ (DX)(BX*8), DI + MOVQ -8(DI)(CX*1), DI + + // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) + SHLQ $0x03, CX + NEGQ CX + ADDQ $0x40, CX + SHRQ CL, DI + ADDQ DI, AX + ADCQ $0x00, SI + +handleRemaining8: + SHRQ $0x01, BX + JNC handleRemaining16 + ADDQ (DX), AX + ADCQ $0x00, SI + ADDQ $0x08, DX + +handleRemaining16: + SHRQ $0x01, BX + JNC handleRemaining32 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ $0x00, SI + ADDQ $0x10, DX + +handleRemaining32: + SHRQ $0x01, BX + JNC handleRemaining64 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ $0x20, DX + +handleRemaining64: + SHRQ $0x01, BX + JNC handleRemaining128 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ $0x00, SI + ADDQ $0x40, DX + +handleRemaining128: + SHRQ $0x01, BX + JNC handleRemainingComplete + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ 64(DX), AX + ADCQ 72(DX), AX + ADCQ 80(DX), AX + ADCQ 88(DX), AX + ADCQ 96(DX), AX + ADCQ 104(DX), AX + ADCQ 112(DX), AX + ADCQ 120(DX), AX + ADCQ $0x00, SI + ADDQ $0x80, DX + +handleRemainingComplete: + ADDQ SI, AX + JMP foldAndReturn + +startSIMD: + VPXOR Y0, Y0, Y0 + VPXOR Y1, Y1, Y1 + VPXOR Y2, Y2, Y2 + VPXOR Y3, Y3, Y3 + MOVQ BX, CX + + // Update number of bytes remaining after the loop completes + ANDQ $0xff, BX + + // Number of 256 byte iterations + SHRQ $0x08, CX + JZ smallLoop + +bigLoop: + VPMOVZXWD (DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 16(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 32(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 48(DX), Y4 + VPADDD Y4, Y3, Y3 + VPMOVZXWD 64(DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 80(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 96(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 112(DX), Y4 + VPADDD Y4, Y3, Y3 + VPMOVZXWD 128(DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 144(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 160(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 176(DX), Y4 + VPADDD Y4, Y3, Y3 + VPMOVZXWD 192(DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 208(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 224(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 240(DX), Y4 + VPADDD Y4, Y3, Y3 + ADDQ $0x00000100, DX + DECQ CX + JNZ bigLoop + CMPQ BX, $0x10 + JLT doneSmallLoop + + // now read a single 16 byte unit of data at a time +smallLoop: + VPMOVZXWD (DX), Y4 + VPADDD Y4, Y0, Y0 + ADDQ $0x10, DX + SUBQ $0x10, BX + CMPQ BX, $0x10 + JGE smallLoop + +doneSmallLoop: + CMPQ BX, $0x00 + JE doneSIMD + + // There are between 1 and 15 bytes remaining. Perform an overlapped read. + LEAQ xmmLoadMasks<>+0(SB), CX + VMOVDQU -16(DX)(BX*1), X4 + VPAND -16(CX)(BX*8), X4, X4 + VPMOVZXWD X4, Y4 + VPADDD Y4, Y0, Y0 + +doneSIMD: + // Multi-chain loop is done, combine the accumulators + VPADDD Y1, Y0, Y0 + VPADDD Y2, Y0, Y0 + VPADDD Y3, Y0, Y0 + + // extract the YMM into a pair of XMM and sum them + VEXTRACTI128 $0x01, Y0, X1 + VPADDD X0, X1, X0 + + // extract the XMM into GP64 + VPEXTRQ $0x00, X0, CX + VPEXTRQ $0x01, X0, DX + + // no more AVX code, clear upper registers to avoid SSE slowdowns + VZEROUPPER + ADDQ CX, AX + ADCQ DX, AX + +foldAndReturn: + // add CF and fold + RORXQ $0x20, AX, CX + ADCL CX, AX + RORXL $0x10, AX, CX + ADCW CX, AX + ADCW $0x00, AX + XCHGB AH, AL + MOVW AX, ret+32(FP) + RET + +// func checksumSSE2(b []byte, initial uint16) uint16 +// Requires: SSE2 +TEXT ·checksumSSE2(SB), NOSPLIT|NOFRAME, $0-34 + MOVWQZX initial+24(FP), AX + XCHGB AH, AL + MOVQ b_base+0(FP), DX + MOVQ b_len+8(FP), BX + + // handle odd length buffers; they are difficult to handle in general + TESTQ $0x00000001, BX + JZ lengthIsEven + MOVBQZX -1(DX)(BX*1), CX + DECQ BX + ADDQ CX, AX + +lengthIsEven: + // handle tiny buffers (<=31 bytes) specially + CMPQ BX, $0x1f + JGT bufferIsNotTiny + XORQ CX, CX + XORQ SI, SI + XORQ DI, DI + + // shift twice to start because length is guaranteed to be even + // n = n >> 2; CF = originalN & 2 + SHRQ $0x02, BX + JNC handleTiny4 + + // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] + MOVWQZX (DX), CX + ADDQ $0x02, DX + +handleTiny4: + // n = n >> 1; CF = originalN & 4 + SHRQ $0x01, BX + JNC handleTiny8 + + // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] + MOVLQZX (DX), SI + ADDQ $0x04, DX + +handleTiny8: + // n = n >> 1; CF = originalN & 8 + SHRQ $0x01, BX + JNC handleTiny16 + + // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] + MOVQ (DX), DI + ADDQ $0x08, DX + +handleTiny16: + // n = n >> 1; CF = originalN & 16 + // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize + SHRQ $0x01, BX + JNC handleTinyFinish + ADDQ (DX), AX + ADCQ 8(DX), AX + +handleTinyFinish: + // CF should be included from the previous add, so we use ADCQ. + // If we arrived via the JNC above, then CF=0 due to the branch condition, + // so ADCQ will still produce the correct result. + ADCQ CX, AX + ADCQ SI, AX + ADCQ DI, AX + JMP foldAndReturn + +bufferIsNotTiny: + // skip all SIMD for small buffers + CMPQ BX, $0x00000100 + JGE startSIMD + + // Accumulate carries in this register. It is never expected to overflow. + XORQ SI, SI + + // We will perform an overlapped read for buffers with length not a multiple of 8. + // Overlapped in this context means some memory will be read twice, but a shift will + // eliminate the duplicated data. This extra read is performed at the end of the buffer to + // preserve any alignment that may exist for the start of the buffer. + MOVQ BX, CX + SHRQ $0x03, BX + ANDQ $0x07, CX + JZ handleRemaining8 + LEAQ (DX)(BX*8), DI + MOVQ -8(DI)(CX*1), DI + + // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) + SHLQ $0x03, CX + NEGQ CX + ADDQ $0x40, CX + SHRQ CL, DI + ADDQ DI, AX + ADCQ $0x00, SI + +handleRemaining8: + SHRQ $0x01, BX + JNC handleRemaining16 + ADDQ (DX), AX + ADCQ $0x00, SI + ADDQ $0x08, DX + +handleRemaining16: + SHRQ $0x01, BX + JNC handleRemaining32 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ $0x00, SI + ADDQ $0x10, DX + +handleRemaining32: + SHRQ $0x01, BX + JNC handleRemaining64 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ $0x20, DX + +handleRemaining64: + SHRQ $0x01, BX + JNC handleRemaining128 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ $0x00, SI + ADDQ $0x40, DX + +handleRemaining128: + SHRQ $0x01, BX + JNC handleRemainingComplete + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ 64(DX), AX + ADCQ 72(DX), AX + ADCQ 80(DX), AX + ADCQ 88(DX), AX + ADCQ 96(DX), AX + ADCQ 104(DX), AX + ADCQ 112(DX), AX + ADCQ 120(DX), AX + ADCQ $0x00, SI + ADDQ $0x80, DX + +handleRemainingComplete: + ADDQ SI, AX + JMP foldAndReturn + +startSIMD: + PXOR X0, X0 + PXOR X1, X1 + PXOR X2, X2 + PXOR X3, X3 + PXOR X4, X4 + MOVQ BX, CX + + // Update number of bytes remaining after the loop completes + ANDQ $0xff, BX + + // Number of 256 byte iterations + SHRQ $0x08, CX + JZ smallLoop + +bigLoop: + MOVOU (DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 16(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 32(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 48(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + MOVOU 64(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 80(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 96(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 112(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + MOVOU 128(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 144(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 160(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 176(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + MOVOU 192(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 208(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 224(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 240(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + ADDQ $0x00000100, DX + DECQ CX + JNZ bigLoop + CMPQ BX, $0x10 + JLT doneSmallLoop + + // now read a single 16 byte unit of data at a time +smallLoop: + MOVOU (DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X1 + ADDQ $0x10, DX + SUBQ $0x10, BX + CMPQ BX, $0x10 + JGE smallLoop + +doneSmallLoop: + CMPQ BX, $0x00 + JE doneSIMD + + // There are between 1 and 15 bytes remaining. Perform an overlapped read. + LEAQ xmmLoadMasks<>+0(SB), CX + MOVOU -16(DX)(BX*1), X5 + PAND -16(CX)(BX*8), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X1 + +doneSIMD: + // Multi-chain loop is done, combine the accumulators + PADDD X1, X0 + PADDD X2, X0 + PADDD X3, X0 + + // extract the XMM into GP64 + MOVQ X0, CX + PSRLDQ $0x08, X0 + MOVQ X0, DX + ADDQ CX, AX + ADCQ DX, AX + +foldAndReturn: + // add CF and fold + MOVL AX, CX + ADCQ $0x00, CX + SHRQ $0x20, AX + ADDQ CX, AX + MOVWQZX AX, CX + SHRQ $0x10, AX + ADDQ CX, AX + MOVW AX, CX + SHRQ $0x10, AX + ADDW CX, AX + ADCW $0x00, AX + XCHGB AH, AL + MOVW AX, ret+32(FP) + RET + +// func checksumAMD64(b []byte, initial uint16) uint16 +TEXT ·checksumAMD64(SB), NOSPLIT|NOFRAME, $0-34 + MOVWQZX initial+24(FP), AX + XCHGB AH, AL + MOVQ b_base+0(FP), DX + MOVQ b_len+8(FP), BX + + // handle odd length buffers; they are difficult to handle in general + TESTQ $0x00000001, BX + JZ lengthIsEven + MOVBQZX -1(DX)(BX*1), CX + DECQ BX + ADDQ CX, AX + +lengthIsEven: + // handle tiny buffers (<=31 bytes) specially + CMPQ BX, $0x1f + JGT bufferIsNotTiny + XORQ CX, CX + XORQ SI, SI + XORQ DI, DI + + // shift twice to start because length is guaranteed to be even + // n = n >> 2; CF = originalN & 2 + SHRQ $0x02, BX + JNC handleTiny4 + + // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] + MOVWQZX (DX), CX + ADDQ $0x02, DX + +handleTiny4: + // n = n >> 1; CF = originalN & 4 + SHRQ $0x01, BX + JNC handleTiny8 + + // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] + MOVLQZX (DX), SI + ADDQ $0x04, DX + +handleTiny8: + // n = n >> 1; CF = originalN & 8 + SHRQ $0x01, BX + JNC handleTiny16 + + // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] + MOVQ (DX), DI + ADDQ $0x08, DX + +handleTiny16: + // n = n >> 1; CF = originalN & 16 + // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize + SHRQ $0x01, BX + JNC handleTinyFinish + ADDQ (DX), AX + ADCQ 8(DX), AX + +handleTinyFinish: + // CF should be included from the previous add, so we use ADCQ. + // If we arrived via the JNC above, then CF=0 due to the branch condition, + // so ADCQ will still produce the correct result. + ADCQ CX, AX + ADCQ SI, AX + ADCQ DI, AX + JMP foldAndReturn + +bufferIsNotTiny: + // Number of 256 byte iterations into loop counter + MOVQ BX, CX + + // Update number of bytes remaining after the loop completes + ANDQ $0xff, BX + SHRQ $0x08, CX + JZ startCleanup + CLC + XORQ SI, SI + XORQ DI, DI + XORQ R8, R8 + XORQ R9, R9 + XORQ R10, R10 + XORQ R11, R11 + XORQ R12, R12 + +bigLoop: + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ 32(DX), DI + ADCQ 40(DX), DI + ADCQ 48(DX), DI + ADCQ 56(DX), DI + ADCQ $0x00, R8 + ADDQ 64(DX), R9 + ADCQ 72(DX), R9 + ADCQ 80(DX), R9 + ADCQ 88(DX), R9 + ADCQ $0x00, R10 + ADDQ 96(DX), R11 + ADCQ 104(DX), R11 + ADCQ 112(DX), R11 + ADCQ 120(DX), R11 + ADCQ $0x00, R12 + ADDQ 128(DX), AX + ADCQ 136(DX), AX + ADCQ 144(DX), AX + ADCQ 152(DX), AX + ADCQ $0x00, SI + ADDQ 160(DX), DI + ADCQ 168(DX), DI + ADCQ 176(DX), DI + ADCQ 184(DX), DI + ADCQ $0x00, R8 + ADDQ 192(DX), R9 + ADCQ 200(DX), R9 + ADCQ 208(DX), R9 + ADCQ 216(DX), R9 + ADCQ $0x00, R10 + ADDQ 224(DX), R11 + ADCQ 232(DX), R11 + ADCQ 240(DX), R11 + ADCQ 248(DX), R11 + ADCQ $0x00, R12 + ADDQ $0x00000100, DX + SUBQ $0x01, CX + JNZ bigLoop + ADDQ SI, AX + ADCQ DI, AX + ADCQ R8, AX + ADCQ R9, AX + ADCQ R10, AX + ADCQ R11, AX + ADCQ R12, AX + + // accumulate CF (twice, in case the first time overflows) + ADCQ $0x00, AX + ADCQ $0x00, AX + +startCleanup: + // Accumulate carries in this register. It is never expected to overflow. + XORQ SI, SI + + // We will perform an overlapped read for buffers with length not a multiple of 8. + // Overlapped in this context means some memory will be read twice, but a shift will + // eliminate the duplicated data. This extra read is performed at the end of the buffer to + // preserve any alignment that may exist for the start of the buffer. + MOVQ BX, CX + SHRQ $0x03, BX + ANDQ $0x07, CX + JZ handleRemaining8 + LEAQ (DX)(BX*8), DI + MOVQ -8(DI)(CX*1), DI + + // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) + SHLQ $0x03, CX + NEGQ CX + ADDQ $0x40, CX + SHRQ CL, DI + ADDQ DI, AX + ADCQ $0x00, SI + +handleRemaining8: + SHRQ $0x01, BX + JNC handleRemaining16 + ADDQ (DX), AX + ADCQ $0x00, SI + ADDQ $0x08, DX + +handleRemaining16: + SHRQ $0x01, BX + JNC handleRemaining32 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ $0x00, SI + ADDQ $0x10, DX + +handleRemaining32: + SHRQ $0x01, BX + JNC handleRemaining64 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ $0x20, DX + +handleRemaining64: + SHRQ $0x01, BX + JNC handleRemaining128 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ $0x00, SI + ADDQ $0x40, DX + +handleRemaining128: + SHRQ $0x01, BX + JNC handleRemainingComplete + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ 64(DX), AX + ADCQ 72(DX), AX + ADCQ 80(DX), AX + ADCQ 88(DX), AX + ADCQ 96(DX), AX + ADCQ 104(DX), AX + ADCQ 112(DX), AX + ADCQ 120(DX), AX + ADCQ $0x00, SI + ADDQ $0x80, DX + +handleRemainingComplete: + ADDQ SI, AX + +foldAndReturn: + // add CF and fold + MOVL AX, CX + ADCQ $0x00, CX + SHRQ $0x20, AX + ADDQ CX, AX + MOVWQZX AX, CX + SHRQ $0x10, AX + ADDQ CX, AX + MOVW AX, CX + SHRQ $0x10, AX + ADDW CX, AX + ADCW $0x00, AX + XCHGB AH, AL + MOVW AX, ret+32(FP) + RET diff --git a/tun/checksum_generic.go b/tun/checksum_generic.go new file mode 100644 index 0000000..2ef201a --- /dev/null +++ b/tun/checksum_generic.go @@ -0,0 +1,15 @@ +// This file contains IP checksum algorithms that are not specific to any +// architecture and don't use hardware acceleration. + +//go:build !amd64 + +package tun + +import "strconv" + +func Checksum(data []byte, initial uint16) uint16 { + if strconv.IntSize < 64 { + return checksumGeneric32(data, initial) + } + return checksumGeneric64(data, initial) +} diff --git a/tun/generate_amd64.go b/tun/generate_amd64.go new file mode 100644 index 0000000..543e211 --- /dev/null +++ b/tun/generate_amd64.go @@ -0,0 +1,579 @@ +//go:build ignore + +//go:generate go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go + +package main + +import ( + "fmt" + "math" + "math/bits" + + . "github.com/mmcloughlin/avo/build" + "github.com/mmcloughlin/avo/operand" + "github.com/mmcloughlin/avo/reg" +) + +const checksumSignature = "func(b []byte, initial uint16) uint16" + +func loadParams() (accum, buf, n reg.GPVirtual) { + accum, buf, n = GP64(), GP64(), GP64() + Load(Param("initial"), accum) + XCHGB(accum.As8H(), accum.As8L()) + Load(Param("b").Base(), buf) + Load(Param("b").Len(), n) + return +} + +type simdStrategy int + +const ( + sse2 = iota + avx2 +) + +const tinyBufferSize = 31 // A buffer is tiny if it has at most 31 bytes. + +func generateSIMDChecksum(name, doc string, minSIMDSize, chains int, strategy simdStrategy) { + TEXT(name, NOSPLIT|NOFRAME, checksumSignature) + Pragma("noescape") + Doc(doc) + + accum64, buf, n := loadParams() + + handleOddLength(n, buf, accum64) + // no chance of overflow because accum64 was initialized by a uint16 and + // handleOddLength adds at most a uint8 + handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny")) + Label("bufferIsNotTiny") + + const simdReadSize = 16 + + if minSIMDSize > tinyBufferSize { + Comment("skip all SIMD for small buffers") + if minSIMDSize <= math.MaxUint8 { + CMPQ(n, operand.U8(minSIMDSize)) + } else { + CMPQ(n, operand.U32(minSIMDSize)) + } + JGE(operand.LabelRef("startSIMD")) + + handleRemaining(n, buf, accum64, minSIMDSize-1) + JMP(operand.LabelRef("foldAndReturn")) + } + + Label("startSIMD") + + // chains is the number of accumulators to use. This improves speed via + // reduced data dependency. We combine the accumulators once when the big + // loop is complete. + simdAccumulate := make([]reg.VecVirtual, chains) + for i := range simdAccumulate { + switch strategy { + case sse2: + simdAccumulate[i] = XMM() + PXOR(simdAccumulate[i], simdAccumulate[i]) + case avx2: + simdAccumulate[i] = YMM() + VPXOR(simdAccumulate[i], simdAccumulate[i], simdAccumulate[i]) + } + } + var zero reg.VecVirtual + if strategy == sse2 { + zero = XMM() + PXOR(zero, zero) + } + + // Number of loads per big loop + const unroll = 16 + // Number of bytes + loopSize := uint64(simdReadSize * unroll) + if bits.Len64(loopSize) != bits.Len64(loopSize-1)+1 { + panic("loopSize is not a power of 2") + } + loopCount := GP64() + + MOVQ(n, loopCount) + Comment("Update number of bytes remaining after the loop completes") + ANDQ(operand.Imm(loopSize-1), n) + Comment(fmt.Sprintf("Number of %d byte iterations", loopSize)) + SHRQ(operand.Imm(uint64(bits.Len64(loopSize-1))), loopCount) + JZ(operand.LabelRef("smallLoop")) + Label("bigLoop") + for i := 0; i < unroll; i++ { + chain := i % chains + switch strategy { + case sse2: + sse2AccumulateStep(i*simdReadSize, buf, zero, simdAccumulate[chain], simdAccumulate[(chain+chains/2)%chains]) + case avx2: + avx2AccumulateStep(i*simdReadSize, buf, simdAccumulate[chain]) + } + } + ADDQ(operand.U32(loopSize), buf) + DECQ(loopCount) + JNZ(operand.LabelRef("bigLoop")) + + Label("bigCleanup") + + CMPQ(n, operand.Imm(uint64(simdReadSize))) + JLT(operand.LabelRef("doneSmallLoop")) + + Commentf("now read a single %d byte unit of data at a time", simdReadSize) + Label("smallLoop") + + switch strategy { + case sse2: + sse2AccumulateStep(0, buf, zero, simdAccumulate[0], simdAccumulate[1]) + case avx2: + avx2AccumulateStep(0, buf, simdAccumulate[0]) + } + ADDQ(operand.Imm(uint64(simdReadSize)), buf) + SUBQ(operand.Imm(uint64(simdReadSize)), n) + CMPQ(n, operand.Imm(uint64(simdReadSize))) + JGE(operand.LabelRef("smallLoop")) + + Label("doneSmallLoop") + CMPQ(n, operand.Imm(0)) + JE(operand.LabelRef("doneSIMD")) + + Commentf("There are between 1 and %d bytes remaining. Perform an overlapped read.", simdReadSize-1) + + maskDataPtr := GP64() + LEAQ(operand.NewDataAddr(operand.NewStaticSymbol("xmmLoadMasks"), 0), maskDataPtr) + dataAddr := operand.Mem{Index: n, Scale: 1, Base: buf, Disp: -simdReadSize} + // scale 8 is only correct here because n is guaranteed to be even and we + // do not generate masks for odd lengths + maskAddr := operand.Mem{Base: maskDataPtr, Index: n, Scale: 8, Disp: -16} + remainder := XMM() + + switch strategy { + case sse2: + MOVOU(dataAddr, remainder) + PAND(maskAddr, remainder) + low := XMM() + MOVOA(remainder, low) + PUNPCKHWL(zero, remainder) + PUNPCKLWL(zero, low) + PADDD(remainder, simdAccumulate[0]) + PADDD(low, simdAccumulate[1]) + case avx2: + // Note: this is very similar to the sse2 path but MOVOU has a massive + // performance hit if used here, presumably due to switching between SSE + // and AVX2 modes. + VMOVDQU(dataAddr, remainder) + VPAND(maskAddr, remainder, remainder) + + temp := YMM() + VPMOVZXWD(remainder, temp) + VPADDD(temp, simdAccumulate[0], simdAccumulate[0]) + } + + Label("doneSIMD") + + Comment("Multi-chain loop is done, combine the accumulators") + for i := range simdAccumulate { + if i == 0 { + continue + } + switch strategy { + case sse2: + PADDD(simdAccumulate[i], simdAccumulate[0]) + case avx2: + VPADDD(simdAccumulate[i], simdAccumulate[0], simdAccumulate[0]) + } + } + + if strategy == avx2 { + Comment("extract the YMM into a pair of XMM and sum them") + tmp := YMM() + VEXTRACTI128(operand.Imm(1), simdAccumulate[0], tmp.AsX()) + + xAccumulate := XMM() + VPADDD(simdAccumulate[0].AsX(), tmp.AsX(), xAccumulate) + simdAccumulate = []reg.VecVirtual{xAccumulate} + } + + Comment("extract the XMM into GP64") + low, high := GP64(), GP64() + switch strategy { + case sse2: + MOVQ(simdAccumulate[0], low) + PSRLDQ(operand.Imm(8), simdAccumulate[0]) + MOVQ(simdAccumulate[0], high) + case avx2: + VPEXTRQ(operand.Imm(0), simdAccumulate[0], low) + VPEXTRQ(operand.Imm(1), simdAccumulate[0], high) + + Comment("no more AVX code, clear upper registers to avoid SSE slowdowns") + VZEROUPPER() + } + ADDQ(low, accum64) + ADCQ(high, accum64) + Label("foldAndReturn") + foldWithCF(accum64, strategy == avx2) + XCHGB(accum64.As8H(), accum64.As8L()) + Store(accum64.As16(), ReturnIndex(0)) + RET() +} + +// handleOddLength generates instructions to incorporate the last byte into +// accum64 if the length is odd. CF may be set if accum64 overflows; be sure to +// handle that if overflow is possible. +func handleOddLength(n, buf, accum64 reg.GPVirtual) { + Comment("handle odd length buffers; they are difficult to handle in general") + TESTQ(operand.U32(1), n) + JZ(operand.LabelRef("lengthIsEven")) + + tmp := GP64() + MOVBQZX(operand.Mem{Base: buf, Index: n, Scale: 1, Disp: -1}, tmp) + DECQ(n) + ADDQ(tmp, accum64) + + Label("lengthIsEven") +} + +func sse2AccumulateStep(offset int, buf reg.GPVirtual, zero, accumulate1, accumulate2 reg.VecVirtual) { + high, low := XMM(), XMM() + MOVOU(operand.Mem{Disp: offset, Base: buf}, high) + MOVOA(high, low) + PUNPCKHWL(zero, high) + PUNPCKLWL(zero, low) + PADDD(high, accumulate1) + PADDD(low, accumulate2) +} + +func avx2AccumulateStep(offset int, buf reg.GPVirtual, accumulate reg.VecVirtual) { + tmp := YMM() + VPMOVZXWD(operand.Mem{Disp: offset, Base: buf}, tmp) + VPADDD(tmp, accumulate, accumulate) +} + +func generateAMD64Checksum(name, doc string) { + TEXT(name, NOSPLIT|NOFRAME, checksumSignature) + Pragma("noescape") + Doc(doc) + + accum64, buf, n := loadParams() + + handleOddLength(n, buf, accum64) + // no chance of overflow because accum64 was initialized by a uint16 and + // handleOddLength adds at most a uint8 + handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny")) + Label("bufferIsNotTiny") + + const ( + // numChains is the number of accumulators and carry counters to use. + // This improves speed via reduced data dependency. We combine the + // accumulators and carry counters once when the loop is complete. + numChains = 4 + unroll = 32 // The number of 64-bit reads to perform per iteration of the loop. + loopSize = 8 * unroll // The number of bytes read per iteration of the loop. + ) + if bits.Len(loopSize) != bits.Len(loopSize-1)+1 { + panic("loopSize is not a power of 2") + } + loopCount := GP64() + + Comment(fmt.Sprintf("Number of %d byte iterations into loop counter", loopSize)) + MOVQ(n, loopCount) + Comment("Update number of bytes remaining after the loop completes") + ANDQ(operand.Imm(loopSize-1), n) + SHRQ(operand.Imm(uint64(bits.Len(loopSize-1))), loopCount) + JZ(operand.LabelRef("startCleanup")) + CLC() + + chains := make([]struct { + accum reg.GPVirtual + carries reg.GPVirtual + }, numChains) + for i := range chains { + if i == 0 { + chains[i].accum = accum64 + } else { + chains[i].accum = GP64() + XORQ(chains[i].accum, chains[i].accum) + } + chains[i].carries = GP64() + XORQ(chains[i].carries, chains[i].carries) + } + + Label("bigLoop") + + var curChain int + for i := 0; i < unroll; i++ { + // It is significantly faster to use a ADCX/ADOX pair instead of plain + // ADC, which results in two dependency chains, however those require + // ADX support, which was added after AVX2. If AVX2 is available, that's + // even better than ADCX/ADOX. + // + // However, multiple dependency chains using multiple accumulators and + // occasionally storing CF into temporary counters seems to work almost + // as well. + addr := operand.Mem{Disp: i * 8, Base: buf} + + if i%4 == 0 { + if i > 0 { + ADCQ(operand.Imm(0), chains[curChain].carries) + curChain = (curChain + 1) % len(chains) + } + ADDQ(addr, chains[curChain].accum) + } else { + ADCQ(addr, chains[curChain].accum) + } + } + ADCQ(operand.Imm(0), chains[curChain].carries) + ADDQ(operand.U32(loopSize), buf) + SUBQ(operand.Imm(1), loopCount) + JNZ(operand.LabelRef("bigLoop")) + for i := range chains { + if i == 0 { + ADDQ(chains[i].carries, accum64) + continue + } + ADCQ(chains[i].accum, accum64) + ADCQ(chains[i].carries, accum64) + } + + accumulateCF(accum64) + + Label("startCleanup") + handleRemaining(n, buf, accum64, loopSize-1) + Label("foldAndReturn") + foldWithCF(accum64, false) + + XCHGB(accum64.As8H(), accum64.As8L()) + Store(accum64.As16(), ReturnIndex(0)) + RET() +} + +// handleTinyBuffers computes checksums if the buffer length (the n parameter) +// is less than 32. After computing the checksum, a jump to returnLabel will +// be executed. Otherwise, if the buffer length is at least 32, nothing will be +// modified; a jump to continueLabel will be executed instead. +// +// When jumping to returnLabel, CF may be set and must be accommodated e.g. +// using foldWithCF or accumulateCF. +// +// Anecdotally, this appears to be faster than attempting to coordinate an +// overlapped read (which would also require special handling for buffers +// smaller than 8). +func handleTinyBuffers(n, buf, accum reg.GPVirtual, returnLabel, continueLabel operand.LabelRef) { + Comment("handle tiny buffers (<=31 bytes) specially") + CMPQ(n, operand.Imm(tinyBufferSize)) + JGT(continueLabel) + + tmp2, tmp4, tmp8 := GP64(), GP64(), GP64() + XORQ(tmp2, tmp2) + XORQ(tmp4, tmp4) + XORQ(tmp8, tmp8) + + Comment("shift twice to start because length is guaranteed to be even", + "n = n >> 2; CF = originalN & 2") + SHRQ(operand.Imm(2), n) + JNC(operand.LabelRef("handleTiny4")) + Comment("tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]") + MOVWQZX(operand.Mem{Base: buf}, tmp2) + ADDQ(operand.Imm(2), buf) + + Label("handleTiny4") + Comment("n = n >> 1; CF = originalN & 4") + SHRQ(operand.Imm(1), n) + JNC(operand.LabelRef("handleTiny8")) + Comment("tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]") + MOVLQZX(operand.Mem{Base: buf}, tmp4) + ADDQ(operand.Imm(4), buf) + + Label("handleTiny8") + Comment("n = n >> 1; CF = originalN & 8") + SHRQ(operand.Imm(1), n) + JNC(operand.LabelRef("handleTiny16")) + Comment("tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]") + MOVQ(operand.Mem{Base: buf}, tmp8) + ADDQ(operand.Imm(8), buf) + + Label("handleTiny16") + Comment("n = n >> 1; CF = originalN & 16", + "n == 0 now, otherwise we would have branched after comparing with tinyBufferSize") + SHRQ(operand.Imm(1), n) + JNC(operand.LabelRef("handleTinyFinish")) + ADDQ(operand.Mem{Base: buf}, accum) + ADCQ(operand.Mem{Base: buf, Disp: 8}, accum) + + Label("handleTinyFinish") + Comment("CF should be included from the previous add, so we use ADCQ.", + "If we arrived via the JNC above, then CF=0 due to the branch condition,", + "so ADCQ will still produce the correct result.") + ADCQ(tmp2, accum) + ADCQ(tmp4, accum) + ADCQ(tmp8, accum) + + JMP(returnLabel) +} + +// handleRemaining generates a series of conditional unrolled additions, +// starting with 8 bytes long and doubling each time until the length reaches +// max. This is the reverse order of what may be intuitive, but makes the branch +// conditions convenient to compute: perform one right shift each time and test +// against CF. +// +// When done, CF may be set and must be accommodated e.g., using foldWithCF or +// accumulateCF. +// +// If n is not a multiple of 8, an extra 64 bit read at the end of the buffer +// will be performed, overlapping with data that will be read later. The +// duplicate data will be shifted off. +// +// The original buffer length must have been at least 8 bytes long, even if +// n < 8, otherwise this will access memory before the start of the buffer, +// which may be unsafe. +func handleRemaining(n, buf, accum64 reg.GPVirtual, max int) { + Comment("Accumulate carries in this register. It is never expected to overflow.") + carries := GP64() + XORQ(carries, carries) + + Comment("We will perform an overlapped read for buffers with length not a multiple of 8.", + "Overlapped in this context means some memory will be read twice, but a shift will", + "eliminate the duplicated data. This extra read is performed at the end of the buffer to", + "preserve any alignment that may exist for the start of the buffer.") + leftover := reg.RCX + MOVQ(n, leftover) + SHRQ(operand.Imm(3), n) // n is now the number of 64 bit reads remaining + ANDQ(operand.Imm(0x7), leftover) // leftover is now the number of bytes to read from the end + JZ(operand.LabelRef("handleRemaining8")) + endBuf := GP64() + // endBuf is the position near the end of the buffer that is just past the + // last multiple of 8: (buf + len(buf)) & ^0x7 + LEAQ(operand.Mem{Base: buf, Index: n, Scale: 8}, endBuf) + + overlapRead := GP64() + // equivalent to overlapRead = binary.LittleEndian.Uint64(buf[len(buf)-8:len(buf)]) + MOVQ(operand.Mem{Base: endBuf, Index: leftover, Scale: 1, Disp: -8}, overlapRead) + + Comment("Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)") + SHLQ(operand.Imm(3), leftover) // leftover = leftover * 8 + NEGQ(leftover) // leftover = -leftover; this completes the (-leftoverBytes*8) part of the expression + ADDQ(operand.Imm(64), leftover) // now we have (64 - leftoverBytes*8) + SHRQ(reg.CL, overlapRead) // shift right by (64 - leftoverBytes*8); CL is the low 8 bits of leftover (set to RCX above) and variable shift only accepts CL + + ADDQ(overlapRead, accum64) + ADCQ(operand.Imm(0), carries) + + for curBytes := 8; curBytes <= max; curBytes *= 2 { + Label(fmt.Sprintf("handleRemaining%d", curBytes)) + SHRQ(operand.Imm(1), n) + if curBytes*2 <= max { + JNC(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2))) + } else { + JNC(operand.LabelRef("handleRemainingComplete")) + } + + numLoads := curBytes / 8 + for i := 0; i < numLoads; i++ { + addr := operand.Mem{Base: buf, Disp: i * 8} + // It is possible to add the multiple dependency chains trick here + // that generateAMD64Checksum uses but anecdotally it does not + // appear to outweigh the cost. + if i == 0 { + ADDQ(addr, accum64) + continue + } + ADCQ(addr, accum64) + } + ADCQ(operand.Imm(0), carries) + + if curBytes > math.MaxUint8 { + ADDQ(operand.U32(uint64(curBytes)), buf) + } else { + ADDQ(operand.U8(uint64(curBytes)), buf) + } + if curBytes*2 >= max { + continue + } + JMP(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2))) + } + Label("handleRemainingComplete") + ADDQ(carries, accum64) +} + +func accumulateCF(accum64 reg.GPVirtual) { + Comment("accumulate CF (twice, in case the first time overflows)") + // accum64 += CF + ADCQ(operand.Imm(0), accum64) + // accum64 += CF again if the previous add overflowed. The previous add was + // 0 or 1. If it overflowed, then accum64 == 0, so adding another 1 can + // never overflow. + ADCQ(operand.Imm(0), accum64) +} + +// foldWithCF generates instructions to fold accum (a GP64) into a 16-bit value +// according to ones-complement arithmetic. BMI2 instructions will be used if +// allowBMI2 is true (requires fewer instructions). +func foldWithCF(accum reg.GPVirtual, allowBMI2 bool) { + Comment("add CF and fold") + + // CF|accum max value starts as 0x1_ffff_ffff_ffff_ffff + + tmp := GP64() + if allowBMI2 { + // effectively, tmp = accum >> 32 (technically, this is a rotate) + RORXQ(operand.Imm(32), accum, tmp) + // accum as uint32 = uint32(accum) + uint32(tmp64) + CF; max value 0xffff_ffff + CF set + ADCL(tmp.As32(), accum.As32()) + // effectively, tmp64 as uint32 = uint32(accum) >> 16 (also a rotate) + RORXL(operand.Imm(16), accum.As32(), tmp.As32()) + // accum as uint16 = uint16(accum) + uint16(tmp) + CF; max value 0xffff + CF unset or 0xfffe + CF set + ADCW(tmp.As16(), accum.As16()) + } else { + // tmp = uint32(accum); max value 0xffff_ffff + // MOVL clears the upper 32 bits of a GP64 so this is equivalent to the + // non-existent MOVLQZX. + MOVL(accum.As32(), tmp.As32()) + // tmp += CF; max value 0x1_0000_0000, CF unset + ADCQ(operand.Imm(0), tmp) + // accum = accum >> 32; max value 0xffff_ffff + SHRQ(operand.Imm(32), accum) + // accum = accum + tmp; max value 0x1_ffff_ffff + CF unset + ADDQ(tmp, accum) + // tmp = uint16(accum); max value 0xffff + MOVWQZX(accum.As16(), tmp) + // accum = accum >> 16; max value 0x1_ffff + SHRQ(operand.Imm(16), accum) + // accum = accum + tmp; max value 0x2_fffe + CF unset + ADDQ(tmp, accum) + // tmp as uint16 = uint16(accum); max value 0xffff + MOVW(accum.As16(), tmp.As16()) + // accum = accum >> 16; max value 0x2 + SHRQ(operand.Imm(16), accum) + // accum as uint16 = uint16(accum) + uint16(tmp); max value 0xffff + CF unset or 0x2 + CF set + ADDW(tmp.As16(), accum.As16()) + } + // accum as uint16 += CF; will not overflow: either CF was 0 or accum <= 0xfffe + ADCW(operand.Imm(0), accum.As16()) +} + +func generateLoadMasks() { + var offset int + // xmmLoadMasks is a table of masks that can be used with PAND to zero all but the last N bytes in an XMM, N=2,4,6,8,10,12,14 + GLOBL("xmmLoadMasks", RODATA|NOPTR) + + for n := 2; n < 16; n += 2 { + var pattern [16]byte + for i := 0; i < len(pattern); i++ { + if i < len(pattern)-n { + pattern[i] = 0 + continue + } + pattern[i] = 0xff + } + DATA(offset, operand.String(pattern[:])) + offset += len(pattern) + } +} + +func main() { + generateLoadMasks() + generateSIMDChecksum("checksumAVX2", "checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2)", 256, 4, avx2) + generateSIMDChecksum("checksumSSE2", "checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2)", 256, 4, sse2) + generateAMD64Checksum("checksumAMD64", "checksumAMD64 computes an IP checksum using amd64 baseline instructions") + Generate() +} diff --git a/tun/offload.go b/tun/offload.go index 4e84db4..6db437c 100644 --- a/tun/offload.go +++ b/tun/offload.go @@ -55,25 +55,32 @@ type GSOOptions struct { } const ( - gsoIPv4SrcAddrOffset = 12 - gsoIPv6SrcAddrOffset = 8 - gsoTCPFlagsOffset = 13 - gsoIPProtoTCP = 6 - gsoIPProtoUDP = 17 + ipv4SrcAddrOffset = 12 + ipv6SrcAddrOffset = 8 +) + +const tcpFlagsOffset = 13 + +const ( + tcpFlagFIN uint8 = 0x01 + tcpFlagPSH uint8 = 0x08 + tcpFlagACK uint8 = 0x10 ) const ( - gsoTCPFlagFIN uint8 = 0x01 - gsoTCPFlagPSH uint8 = 0x08 + // defined here in order to avoid importation of any platform-specific pkgs + ipProtoTCP = 6 + ipProtoUDP = 17 ) -// GSOSplit splits packets from in into outBufs[][outOffset:], writing +// GSOSplit splits packets from 'in' into outBufs[][outOffset:], writing // the size of each element into sizes. It returns the number of buffers -// populated, and/or an error. Callers may pass an in slice that overlaps with -// the first element of outBufs, i.e. &in[0] may be equal to +// populated, and/or an error. Callers may pass an 'in' slice that overlaps with +// the first element of outBuffers, i.e. &in[0] may be equal to // &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the // value of options.NeedsCsum. Length of each outBufs element must be greater -// than or equal to the length of in, otherwise output may be silently truncated. +// than or equal to the length of 'in', otherwise output may be silently +// truncated. func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) { cSumAt := int(options.CsumStart) + int(options.CsumOffset) if cSumAt+1 >= len(in) { @@ -84,12 +91,15 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen) } + // Handle the conditions where we are copying a single element to outBuffs. payloadLen := len(in) - int(options.HdrLen) if options.GSOType == GSONone || payloadLen < int(options.GSOSize) { if len(in) > len(outBufs[0][outOffset:]) { return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:])) } if options.NeedsCsum { + // The initial value at the checksum offset should be summed with + // the checksum we compute. This is typically the pseudo-header sum. initial := binary.BigEndian.Uint16(in[cSumAt:]) in[cSumAt], in[cSumAt+1] = 0, 0 binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial)) @@ -123,24 +133,24 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO } iphLen := int(options.CsumStart) - srcAddrOffset := gsoIPv6SrcAddrOffset + srcAddrOffset := ipv6SrcAddrOffset addrLen := 16 if ipVersion == 4 { - srcAddrOffset = gsoIPv4SrcAddrOffset + srcAddrOffset = ipv4SrcAddrOffset addrLen = 4 } transportCsumAt := int(options.CsumStart + options.CsumOffset) var firstTCPSeqNum uint32 var protocol uint8 if options.GSOType == GSOTCPv4 || options.GSOType == GSOTCPv6 { - protocol = gsoIPProtoTCP + protocol = ipProtoTCP if len(in) < int(options.CsumStart)+20 { return 0, fmt.Errorf("length of packet (%d) < GSO CsumStart (%d) + minimum TCP header size (%d)", len(in), options.CsumStart, 20) } firstTCPSeqNum = binary.BigEndian.Uint32(in[options.CsumStart+4:]) } else { - protocol = gsoIPProtoUDP + protocol = ipProtoUDP } nextSegmentDataAt := int(options.HdrLen) i := 0 @@ -159,35 +169,45 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO copy(out, in[:iphLen]) if ipVersion == 4 { + // For IPv4 we are responsible for incrementing the ID field, + // updating the total len field, and recalculating the header + // checksum. if i > 0 { id := binary.BigEndian.Uint16(out[4:]) id += uint16(i) binary.BigEndian.PutUint16(out[4:], id) } - out[10], out[11] = 0, 0 + out[10], out[11] = 0, 0 // clear ipv4 header checksum binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) ipv4CSum := ^Checksum(out[:iphLen], 0) binary.BigEndian.PutUint16(out[10:], ipv4CSum) } else { + // For IPv6 we are responsible for updating the payload length field. binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) } + // copy transport header copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen]) - if protocol == gsoIPProtoTCP { + if protocol == ipProtoTCP { + // set TCP seq and adjust TCP flags tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i)) binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq) if nextSegmentEnd != len(in) { - clearFlags := gsoTCPFlagFIN | gsoTCPFlagPSH - out[options.CsumStart+gsoTCPFlagsOffset] &^= clearFlags + // FIN and PSH should only be set on last segment + clearFlags := tcpFlagFIN | tcpFlagPSH + out[options.CsumStart+tcpFlagsOffset] &^= clearFlags } } else { + // set UDP header len binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart)) } + // payload copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) - out[transportCsumAt], out[transportCsumAt+1] = 0, 0 + // transport checksum + out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum transportHeaderLen := int(options.HdrLen - options.CsumStart) lenForPseudo := uint16(transportHeaderLen + segmentDataLen) transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index 6825bbc..6ba8547 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package tun @@ -9,6 +9,7 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "io" "unsafe" @@ -16,14 +17,6 @@ import ( "golang.org/x/sys/unix" ) -const tcpFlagsOffset = 13 - -const ( - tcpFlagFIN uint8 = 0x01 - tcpFlagPSH uint8 = 0x08 - tcpFlagACK uint8 = 0x10 -) - // virtioNetHdr is defined in the kernel in include/uapi/linux/virtio_net.h. The // kernel symbol is virtio_net_hdr. type virtioNetHdr struct { @@ -35,6 +28,30 @@ type virtioNetHdr struct { csumOffset uint16 } +func (v *virtioNetHdr) toGSOOptions() (GSOOptions, error) { + var gsoType GSOType + switch v.gsoType { + case unix.VIRTIO_NET_HDR_GSO_NONE: + gsoType = GSONone + case unix.VIRTIO_NET_HDR_GSO_TCPV4: + gsoType = GSOTCPv4 + case unix.VIRTIO_NET_HDR_GSO_TCPV6: + gsoType = GSOTCPv6 + case unix.VIRTIO_NET_HDR_GSO_UDP_L4: + gsoType = GSOUDPL4 + default: + return GSOOptions{}, fmt.Errorf("unsupported virtio gsoType: %d", v.gsoType) + } + return GSOOptions{ + GSOType: gsoType, + HdrLen: v.hdrLen, + CsumStart: v.csumStart, + CsumOffset: v.csumOffset, + GSOSize: v.gsoSize, + NeedsCsum: v.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0, + }, nil +} + func (v *virtioNetHdr) decode(b []byte) error { if len(b) < virtioNetHdrLen { return io.ErrShortBuffer @@ -393,8 +410,8 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool { addrSize = 16 } lenForPseudo := uint16(len(pkt) - int(iphLen)) - cSum := pseudoHeaderChecksumNoFold(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) - return ^checksum(pkt[iphLen:], cSum) == 0 + cSum := PseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) + return ^Checksum(pkt[iphLen:], cSum) == 0 } // coalesceResult represents the result of attempting to coalesce two TCP @@ -510,9 +527,7 @@ const ( ) const ( - ipv4SrcAddrOffset = 12 - ipv6SrcAddrOffset = 8 - maxUint16 = 1<<16 - 1 + maxUint16 = 1<<16 - 1 ) type groResult int @@ -644,7 +659,7 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 pkt[10], pkt[11] = 0, 0 binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length - iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field } err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -664,8 +679,8 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) - binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) + psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -701,7 +716,7 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e } else { pkt[10], pkt[11] = 0, 0 binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length - iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field } err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -724,8 +739,8 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) - binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) + psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -894,100 +909,3 @@ func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGR errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable) return errors.Join(errTCP, errUDP) } - -// gsoSplit splits packets from in into outBuffs, writing the size of each -// element into sizes. It returns the number of buffers populated, and/or an -// error. -func gsoSplit(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffset int, isV6 bool) (int, error) { - iphLen := int(hdr.csumStart) - srcAddrOffset := ipv6SrcAddrOffset - addrLen := 16 - if !isV6 { - in[10], in[11] = 0, 0 // clear ipv4 header checksum - srcAddrOffset = ipv4SrcAddrOffset - addrLen = 4 - } - transportCsumAt := int(hdr.csumStart + hdr.csumOffset) - in[transportCsumAt], in[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum - var firstTCPSeqNum uint32 - var protocol uint8 - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 || hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV6 { - protocol = unix.IPPROTO_TCP - firstTCPSeqNum = binary.BigEndian.Uint32(in[hdr.csumStart+4:]) - } else { - protocol = unix.IPPROTO_UDP - } - nextSegmentDataAt := int(hdr.hdrLen) - i := 0 - for ; nextSegmentDataAt < len(in); i++ { - if i == len(outBuffs) { - return i - 1, ErrTooManySegments - } - nextSegmentEnd := nextSegmentDataAt + int(hdr.gsoSize) - if nextSegmentEnd > len(in) { - nextSegmentEnd = len(in) - } - segmentDataLen := nextSegmentEnd - nextSegmentDataAt - totalLen := int(hdr.hdrLen) + segmentDataLen - sizes[i] = totalLen - out := outBuffs[i][outOffset:] - - copy(out, in[:iphLen]) - if !isV6 { - // For IPv4 we are responsible for incrementing the ID field, - // updating the total len field, and recalculating the header - // checksum. - if i > 0 { - id := binary.BigEndian.Uint16(out[4:]) - id += uint16(i) - binary.BigEndian.PutUint16(out[4:], id) - } - binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) - ipv4CSum := ^checksum(out[:iphLen], 0) - binary.BigEndian.PutUint16(out[10:], ipv4CSum) - } else { - // For IPv6 we are responsible for updating the payload length field. - binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) - } - - // copy transport header - copy(out[hdr.csumStart:hdr.hdrLen], in[hdr.csumStart:hdr.hdrLen]) - - if protocol == unix.IPPROTO_TCP { - // set TCP seq and adjust TCP flags - tcpSeq := firstTCPSeqNum + uint32(hdr.gsoSize*uint16(i)) - binary.BigEndian.PutUint32(out[hdr.csumStart+4:], tcpSeq) - if nextSegmentEnd != len(in) { - // FIN and PSH should only be set on last segment - clearFlags := tcpFlagFIN | tcpFlagPSH - out[hdr.csumStart+tcpFlagsOffset] &^= clearFlags - } - } else { - // set UDP header len - binary.BigEndian.PutUint16(out[hdr.csumStart+4:], uint16(segmentDataLen)+(hdr.hdrLen-hdr.csumStart)) - } - - // payload - copy(out[hdr.hdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) - - // transport checksum - transportHeaderLen := int(hdr.hdrLen - hdr.csumStart) - lenForPseudo := uint16(transportHeaderLen + segmentDataLen) - transportCSumNoFold := pseudoHeaderChecksumNoFold(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) - transportCSum := ^checksum(out[hdr.csumStart:totalLen], transportCSumNoFold) - binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], transportCSum) - - nextSegmentDataAt += int(hdr.gsoSize) - } - return i, nil -} - -func gsoNoneChecksum(in []byte, cSumStart, cSumOffset uint16) error { - cSumAt := cSumStart + cSumOffset - // The initial value at the checksum offset should be summed with the - // checksum we compute. This is typically the pseudo-header checksum. - initial := binary.BigEndian.Uint16(in[cSumAt:]) - in[cSumAt], in[cSumAt+1] = 0, 0 - binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[cSumStart:], uint64(initial))) - return nil -} diff --git a/tun/tun.go b/tun/tun.go index 6fb5c56..733b24c 100644 --- a/tun/tun.go +++ b/tun/tun.go @@ -56,12 +56,21 @@ type Device interface { // versions may have offload bugs. Where these bugs negatively impact throughput // or break connectivity entirely we can use these methods to disable the // related offload. +// +// Linux has the following known, GRO bugs. +// +// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net +// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then +// resolved later in +// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending +// commit was pulled into various LTS releases. +// +// UDP GRO writes end up blackholing/dropping packets destined for a +// vxlan/geneve interface on kernel versions prior to 6.8.5. type GRODevice interface { Device - // DisableUDPGRO disables UDP GRO if it is enabled. DisableUDPGRO() - // DisableTCPGRO disables TCP GRO if it is enabled. DisableTCPGRO() } diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 4b7866d..8f4aed1 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. */ package tun @@ -48,7 +48,7 @@ type NativeTun struct { readOpMu sync.Mutex // readOpMu guards readBuff readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr - writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable, udpGROTable, gro + writeOpMu sync.Mutex // writeOpMu guards the following fields toWrite []int tcpGROTable *tcpGROTable udpGROTable *udpGROTable @@ -269,21 +269,15 @@ func (tun *NativeTun) setMTU(n int) error { defer unix.Close(fd) - // do ioctl call - var ifr [ifReqSize]byte - copy(ifr[:], name) - *(*uint32)(unsafe.Pointer(&ifr[unix.IFNAMSIZ])) = uint32(n) - _, _, errno := unix.Syscall( - unix.SYS_IOCTL, - uintptr(fd), - uintptr(unix.SIOCSIFMTU), - uintptr(unsafe.Pointer(&ifr[0])), - ) - - if errno != 0 { - return fmt.Errorf("failed to set MTU of TUN device: %w", errno) + req, err := unix.NewIfreq(name) + if err != nil { + return fmt.Errorf("unix.NewIfreq(%q): %w", name, err) + } + req.SetUint32(uint32(n)) + err = unix.IoctlIfreq(fd, unix.SIOCSIFMTU, req) + if err != nil { + return fmt.Errorf("failed to set MTU of TUN device %q: %w", name, err) } - return nil } @@ -402,73 +396,32 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e return 0, err } in = in[virtioNetHdrLen:] - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_NONE { - if hdr.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 { - // This means CHECKSUM_PARTIAL in skb context. We are responsible - // for computing the checksum starting at hdr.csumStart and placing - // at hdr.csumOffset. - err = gsoNoneChecksum(in, hdr.csumStart, hdr.csumOffset) - if err != nil { - return 0, err - } - } - if len(in) > len(bufs[0][offset:]) { - return 0, fmt.Errorf("read len %d overflows bufs element len %d", len(in), len(bufs[0][offset:])) - } - n := copy(bufs[0][offset:], in) - sizes[0] = n - return 1, nil - } - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - return 0, fmt.Errorf("unsupported virtio GSO type: %d", hdr.gsoType) + + options, err := hdr.toGSOOptions() + if err != nil { + return 0, err } - ipVersion := in[0] >> 4 - switch ipVersion { - case 4: - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) - } - case 6: - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) - } - default: - return 0, fmt.Errorf("invalid ip header version: %d", ipVersion) - } - - // Don't trust hdr.hdrLen from the kernel as it can be equal to the length + // Don't trust HdrLen from the kernel as it can be equal to the length // of the entire first packet when the kernel is handling it as part of a // FORWARD path. Instead, parse the transport header length and add it onto - // csumStart, which is synonymous for IP header length. - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - hdr.hdrLen = hdr.csumStart + 8 - } else { - if len(in) <= int(hdr.csumStart+12) { + // CsumStart, which is synonymous for IP header length. + if options.GSOType == GSOUDPL4 { + options.HdrLen = options.CsumStart + 8 + } else if options.GSOType != GSONone { + if len(in) <= int(options.CsumStart+12) { return 0, errors.New("packet is too short") } - tcpHLen := uint16(in[hdr.csumStart+12] >> 4 * 4) + tcpHLen := uint16(in[options.CsumStart+12] >> 4 * 4) if tcpHLen < 20 || tcpHLen > 60 { // A TCP header must be between 20 and 60 bytes in length. return 0, fmt.Errorf("tcp header len is invalid: %d", tcpHLen) } - hdr.hdrLen = hdr.csumStart + tcpHLen + options.HdrLen = options.CsumStart + tcpHLen } - if len(in) < int(hdr.hdrLen) { - return 0, fmt.Errorf("length of packet (%d) < virtioNetHdr.hdrLen (%d)", len(in), hdr.hdrLen) - } - - if hdr.hdrLen < hdr.csumStart { - return 0, fmt.Errorf("virtioNetHdr.hdrLen (%d) < virtioNetHdr.csumStart (%d)", hdr.hdrLen, hdr.csumStart) - } - cSumAt := int(hdr.csumStart + hdr.csumOffset) - if cSumAt+1 >= len(in) { - return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in)) - } - - return gsoSplit(in, hdr, bufs, sizes, offset, ipVersion == 6) + return GSOSplit(in, options, bufs, sizes, offset) } func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { @@ -525,14 +478,16 @@ func (tun *NativeTun) BatchSize() int { return tun.batchSize } -// DisableUDPGRO disables UDP GRO if it is enabled. +// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface +// for cases where it should be called. func (tun *NativeTun) DisableUDPGRO() { tun.writeOpMu.Lock() tun.gro.disableUDPGRO() tun.writeOpMu.Unlock() } -// DisableTCPGRO disables TCP GRO if it is enabled. +// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface +// for cases where it should be called. func (tun *NativeTun) DisableTCPGRO() { tun.writeOpMu.Lock() tun.gro.disableTCPGRO() diff --git a/tun/tun_plan9.go b/tun/tun_plan9.go new file mode 100644 index 0000000..7b66ead --- /dev/null +++ b/tun/tun_plan9.go @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package tun + +import ( + "fmt" + "io" + "os" + "strconv" + "strings" + "sync" +) + +type NativeTun struct { + name string // "/net/ipifc/2" + ctlFile *os.File + dataFile *os.File + events chan Event + errors chan error + closeOnce sync.Once +} + +func CreateTUN(_ string, mtu int) (Device, error) { + ctl, err := os.OpenFile("/net/ipifc/clone", os.O_RDWR, 0) + if err != nil { + return nil, err + } + nbuf := make([]byte, 5) + n, err := ctl.Read(nbuf) + if err != nil { + ctl.Close() + return nil, fmt.Errorf("error reading from clone file: %w", err) + } + ifn, err := strconv.Atoi(strings.TrimSpace(string(nbuf[:n]))) + if err != nil { + ctl.Close() + return nil, fmt.Errorf("error converting clone result %q to int: %w", nbuf[:n], err) + } + + if _, err := fmt.Fprintf(ctl, "bind pkt\n"); err != nil { + ctl.Close() + return nil, fmt.Errorf("error binding to pkt: %w", err) + } + if mtu > 0 { + if _, err := fmt.Fprintf(ctl, "mtu %d\n", mtu); err != nil { + ctl.Close() + return nil, fmt.Errorf("error setting MTU: %w", err) + } + } + + dataFile, err := os.OpenFile(fmt.Sprintf("/net/ipifc/%d/data", ifn), os.O_RDWR, 0) + if err != nil { + ctl.Close() + return nil, err + } + + tun := &NativeTun{ + ctlFile: ctl, + dataFile: dataFile, + name: fmt.Sprintf("/net/ipifc/%d", ifn), + events: make(chan Event, 10), + errors: make(chan error, 5), + } + tun.events <- EventUp + + return tun, nil +} + +func (tun *NativeTun) Name() (string, error) { + return tun.name, nil +} + +func (tun *NativeTun) File() *os.File { + return tun.ctlFile +} + +func (tun *NativeTun) Events() <-chan Event { + return tun.events +} + +func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { + select { + case err := <-tun.errors: + return 0, err + default: + n, err := tun.dataFile.Read(bufs[0][offset:]) + if n == 1 && bufs[0][offset] == 0 { + // EOF + err = io.EOF + n = 0 + } + sizes[0] = n + return 1, err + } +} + +func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { + for i, buf := range bufs { + if _, err := tun.dataFile.Write(buf[offset:]); err != nil { + return i, err + } + } + return len(bufs), nil +} + +func (tun *NativeTun) Close() error { + var err1, err2 error + tun.closeOnce.Do(func() { + _, err1 := fmt.Fprintf(tun.ctlFile, "unbind\n") + if err := tun.ctlFile.Close(); err != nil && err1 == nil { + err1 = err + } + err2 = tun.dataFile.Close() + }) + if err1 != nil { + return err1 + } + return err2 +} + +func (tun *NativeTun) MTU() (int, error) { + var buf [100]byte + f, err := os.Open(tun.name + "/status") + if err != nil { + return 0, err + } + defer f.Close() + n, err := f.Read(buf[:]) + _, res, ok := strings.Cut(string(buf[:n]), " maxtu ") + if ok { + if mtus, _, ok := strings.Cut(res, " "); ok { + mtu, err := strconv.Atoi(mtus) + if err != nil { + return 0, fmt.Errorf("error converting mtu %q to int: %w", mtus, err) + } + return mtu, nil + } + } + return 0, fmt.Errorf("no 'maxtu' field found in %s/status", tun.name) +} + +func (tun *NativeTun) BatchSize() int { + return 1 +}