eburet-oss/gvisor-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

snapshot: sagernet/gvisor v0.0.0-20250811.0-sing-box-mod.1

Browse source 
Содержимое пина, зафиксированного в go.mod sing-box-lx, одним коммитом
без истории. Полная история SagerNet/gvisor — 1.45 ГБ и клонируется в
каждой CI-джобе; наша дельта — одна вставка в одну функцию, история для
неё не нужна.

Module path github.com/sagernet/gvisor сохранён намеренно: на него
опирается replace-директива суперпроекта.

Патч поверх — отдельным коммитом, чтобы дельта читалась одним git show
и переносилась на новый пин копированием.

SPECS/TASKS/048-GVISOR_HANDSHAKE_NIL_CRASH
This commit is contained in:
Leadaxe 2026-08-04 15:50:08 +03:00
commit 2c4ae3b0a4
712 changed files with 185689 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

119
pkg/xdp/completionqueue.go Normal file
View file

@ -0,0 +1,119 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64 || arm64
// +build amd64 arm64
package xdp
import (
"github.com/sagernet/gvisor/pkg/atomicbitops"
)
// The CompletionQueue is how the kernel tells a process which buffers have
// been transmitted and can be reused.
//
// CompletionQueue is not thread-safe and requires external synchronization
type CompletionQueue struct {
// mem is the mmap'd area shared with the kernel. Many other fields of
// this struct point into mem.
mem []byte
// ring is the actual ring buffer. It is a list of frame addresses
// ready to be reused.
//
// len(ring) must be a power of 2.
ring []uint64
// mask is used whenever indexing into ring. It is always len(ring)-1.
// It prevents index out of bounds errors while allowing the producer
// and consumer pointers to repeatedly "overflow" and loop back around
// the ring.
mask uint32
// producer points to the shared atomic value that indicates the last
// produced descriptor. Only the kernel updates this value.
producer *atomicbitops.Uint32
// consumer points to the shared atomic value that indicates the last
// consumed descriptor. Only we update this value.
consumer *atomicbitops.Uint32
// flags points to the shared atomic value that holds flags for the
// queue.
flags *atomicbitops.Uint32
// Cached values are used to avoid relatively expensive atomic
// operations. They are used, incremented, and decremented multiple
// times with non-atomic operations, and then "batch-updated" by
// reading or writing atomically to synchronize with the kernel.
// cachedProducer is updated when we atomically read *producer.
cachedProducer uint32
// cachedConsumer is used to atomically write *consumer.
cachedConsumer uint32
}
// Peek returns the number of buffers available to reuse as well as the index
// at which they start. Peek will only return a buffer once, so callers must
// process any received buffers.
func (cq *CompletionQueue) Peek() (nAvailable, index uint32) {
// Get the number of available buffers and update cachedConsumer to
// reflect that we're going to consume them.
entries := cq.free()
index = cq.cachedConsumer
cq.cachedConsumer += entries
return entries, index
}
func (cq *CompletionQueue) free() uint32 {
// Return any buffers we know about without incurring an atomic
// operation if possible.
entries := cq.cachedProducer - cq.cachedConsumer
// If we're not aware of any completed packets, refresh the producer
// pointer to see whether the kernel enqueued anything.
if entries == 0 {
cq.cachedProducer = cq.producer.Load()
entries = cq.cachedProducer - cq.cachedConsumer
}
return entries
}
// Release notifies the kernel that we have consumed nDone packets.
func (cq *CompletionQueue) Release(nDone uint32) {
// We don't have to use an atomic add because only we update this; the
// kernel just reads it.
cq.consumer.Store(cq.consumer.RacyLoad() + nDone)
}
// Get gets the descriptor at index.
func (cq *CompletionQueue) Get(index uint32) uint64 {
// Use mask to avoid overflowing and loop back around the ring.
return cq.ring[index&cq.mask]
}
// FreeAll dequeues as many buffers as possible from the queue and returns them
// to the UMEM.
//
// +checklocks:umem.mu
func (cq *CompletionQueue) FreeAll(umem *UMEM) {
available, index := cq.Peek()
if available < 1 {
return
}
for i := uint32(0); i < available; i++ {
umem.FreeFrame(cq.Get(index + i))
}
cq.Release(available)
}

121
pkg/xdp/fillqueue.go Normal file
View file

@ -0,0 +1,121 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64 || arm64
// +build amd64 arm64
package xdp
import (
"github.com/sagernet/gvisor/pkg/atomicbitops"
)
// The FillQueue is how a process tells the kernel which buffers are available
// to be filled by incoming packets.
//
// FillQueue is not thread-safe and requires external synchronization
type FillQueue struct {
// mem is the mmap'd area shared with the kernel. Many other fields of
// this struct point into mem.
mem []byte
// ring is the actual ring buffer. It is a list of frame addresses
// ready for incoming packets.
//
// len(ring) must be a power of 2.
ring []uint64
// mask is used whenever indexing into ring. It is always len(ring)-1.
// It prevents index out of bounds errors while allowing the producer
// and consumer pointers to repeatedly "overflow" and loop back around
// the ring.
mask uint32
// producer points to the shared atomic value that indicates the last
// produced descriptor. Only we update this value.
producer *atomicbitops.Uint32
// consumer points to the shared atomic value that indicates the last
// consumed descriptor. Only the kernel updates this value.
consumer *atomicbitops.Uint32
// flags points to the shared atomic value that holds flags for the
// queue.
flags *atomicbitops.Uint32
// Cached values are used to avoid relatively expensive atomic
// operations. They are used, incremented, and decremented multiple
// times with non-atomic operations, and then "batch-updated" by
// reading or writing atomically to synchronize with the kernel.
// cachedProducer is used to atomically write *producer.
cachedProducer uint32
// cachedConsumer is updated when we atomically read *consumer.
// cachedConsumer is actually len(ring) larger than the real consumer
// value. See free() for details.
cachedConsumer uint32
}
// free returns the number of free descriptors in the fill queue.
func (fq *FillQueue) free(toReserve uint32) uint32 {
// Try to find free descriptors without incurring an atomic operation.
//
// cachedConsumer is always len(fq.ring) larger than the real consumer
// value. This lets us, in the common case, compute the number of free
// descriptors simply via fq.cachedConsumer - fq.cachedProducer without
// also adding len(fq.ring).
if available := fq.cachedConsumer - fq.cachedProducer; available >= toReserve {
return available
}
// If we didn't already have enough descriptors available, check
// whether the kernel has returned some to us.
fq.cachedConsumer = fq.consumer.Load()
fq.cachedConsumer += uint32(len(fq.ring))
return fq.cachedConsumer - fq.cachedProducer
}
// Notify updates the producer such that it is visible to the kernel.
func (fq *FillQueue) Notify() {
fq.producer.Store(fq.cachedProducer)
}
// Set sets the fill queue's descriptor at index to addr.
func (fq *FillQueue) Set(index uint32, addr uint64) {
// Use mask to avoid overflowing and loop back around the ring.
fq.ring[index&fq.mask] = addr
}
// FillAll posts as many empty buffers as possible for the kernel to fill, then
// notifies the kernel.
//
// +checklocks:umem.mu
func (fq *FillQueue) FillAll(umem *UMEM) {
// Figure out how many buffers and queue slots are available.
available := fq.free(umem.nFreeFrames)
if available == 0 {
return
}
if available > umem.nFreeFrames {
available = umem.nFreeFrames
}
// Fill the queue as much as possible and notify the kernel.
index := fq.cachedProducer
fq.cachedProducer += available
for i := uint32(0); i < available; i++ {
fq.Set(index+i, umem.AllocFrame())
}
fq.Notify()
}

105
pkg/xdp/rxqueue.go Normal file
View file

@ -0,0 +1,105 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64 || arm64
// +build amd64 arm64
package xdp
import (
"github.com/sagernet/gvisor/pkg/atomicbitops"
"golang.org/x/sys/unix"
)
// The RXQueue is how the kernel tells a process which buffers are full with
// incoming packets.
//
// RXQueue is not thread-safe and requires external synchronization
type RXQueue struct {
// mem is the mmap'd area shared with the kernel. Many other fields of
// this struct point into mem.
mem []byte
// ring is the actual ring buffer. It is a list of XDP descriptors
// pointing to incoming packets.
//
// len(ring) must be a power of 2.
ring []unix.XDPDesc
// mask is used whenever indexing into ring. It is always len(ring)-1.
// It prevents index out of bounds errors while allowing the producer
// and consumer pointers to repeatedly "overflow" and loop back around
// the ring.
mask uint32
// producer points to the shared atomic value that indicates the last
// produced descriptor. Only the kernel updates this value.
producer *atomicbitops.Uint32
// consumer points to the shared atomic value that indicates the last
// consumed descriptor. Only we update this value.
consumer *atomicbitops.Uint32
// flags points to the shared atomic value that holds flags for the
// queue.
flags *atomicbitops.Uint32
// Cached values are used to avoid relatively expensive atomic
// operations. They are used, incremented, and decremented multiple
// times with non-atomic operations, and then "batch-updated" by
// reading or writing atomically to synchronize with the kernel.
// cachedProducer is updated when we atomically read *producer.
cachedProducer uint32
// cachedConsumer is used to atomically write *consumer.
cachedConsumer uint32
}
// Peek returns the number of packets available to read as well as the index at
// which they start. Peek will only return a packet once, so callers must
// process any received packets.
func (rq *RXQueue) Peek() (nReceived, index uint32) {
// Get the number of available buffers and update cachedConsumer to
// reflect that we're going to consume them.
entries := rq.free()
index = rq.cachedConsumer
rq.cachedConsumer += entries
return entries, index
}
func (rq *RXQueue) free() uint32 {
// Return any buffers we know about without incurring an atomic
// operation if possible.
entries := rq.cachedProducer - rq.cachedConsumer
// If we're not aware of any RX'd packets, refresh the producer pointer
// to see whether the kernel enqueued anything.
if entries == 0 {
rq.cachedProducer = rq.producer.Load()
entries = rq.cachedProducer - rq.cachedConsumer
}
return entries
}
// Release notifies the kernel that we have consumed nDone packets.
func (rq *RXQueue) Release(nDone uint32) {
// We don't have to use an atomic add because only we update this; the
// kernel just reads it.
rq.consumer.Store(rq.consumer.RacyLoad() + nDone)
}
// Get gets the descriptor at index.
func (rq *RXQueue) Get(index uint32) unix.XDPDesc {
// Use mask to avoid overflowing and loop back around the ring.
return rq.ring[index&rq.mask]
}

116
pkg/xdp/txqueue.go Normal file
View file

@ -0,0 +1,116 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64 || arm64
// +build amd64 arm64
package xdp
import (
"github.com/sagernet/gvisor/pkg/atomicbitops"
"golang.org/x/sys/unix"
)
// The TXQueue is how a process tells the kernel which buffers are available to
// be sent via the NIC.
//
// TXQueue is not thread-safe and requires external synchronization
type TXQueue struct {
// sockfd is the underlying AF_XDP socket.
sockfd uint32
// mem is the mmap'd area shared with the kernel. Many other fields of
// this struct point into mem.
mem []byte
// ring is the actual ring buffer. It is a list of XDP descriptors
// pointing to ready-to-transmit packets.
//
// len(ring) must be a power of 2.
ring []unix.XDPDesc
// mask is used whenever indexing into ring. It is always len(ring)-1.
// It prevents index out of bounds errors while allowing the producer
// and consumer pointers to repeatedly "overflow" and loop back around
// the ring.
mask uint32
// producer points to the shared atomic value that indicates the last
// produced descriptor. Only we update this value.
producer *atomicbitops.Uint32
// consumer points to the shared atomic value that indicates the last
// consumed descriptor. Only the kernel updates this value.
consumer *atomicbitops.Uint32
// flags points to the shared atomic value that holds flags for the
// queue.
flags *atomicbitops.Uint32
// Cached values are used to avoid relatively expensive atomic
// operations. They are used, incremented, and decremented multiple
// times with non-atomic operations, and then "batch-updated" by
// reading or writing atomically to synchronize with the kernel.
// cachedProducer is used to atomically write *producer.
cachedProducer uint32
// cachedConsumer is updated when we atomically read *consumer.
// cachedConsumer is actually len(ring) larger than the real consumer
// value. See free() for details.
cachedConsumer uint32
}
// Reserve reserves descriptors in the queue. If toReserve descriptors cannot
// be reserved, none are reserved.
//
// +checklocks:umem.mu
func (tq *TXQueue) Reserve(umem *UMEM, toReserve uint32) (nReserved, index uint32) {
if umem.nFreeFrames < toReserve || tq.free(toReserve) < toReserve {
return 0, 0
}
idx := tq.cachedProducer
tq.cachedProducer += toReserve
return toReserve, idx
}
// free returns the number of free descriptors in the TX queue.
func (tq *TXQueue) free(toReserve uint32) uint32 {
// Try to find free descriptors without incurring an atomic operation.
//
// cachedConsumer is always len(tq.ring) larger than the real consumer
// value. This lets us, in the common case, compute the number of free
// descriptors simply via tq.cachedConsumer - tq.cachedProducer without
// also addign len(tq.ring).
if available := tq.cachedConsumer - tq.cachedProducer; available >= toReserve {
return available
}
// If we didn't already have enough descriptors available, check
// whether the kernel has returned some to us.
tq.cachedConsumer = tq.consumer.Load()
tq.cachedConsumer += uint32(len(tq.ring))
return tq.cachedConsumer - tq.cachedProducer
}
// Notify updates the producer such that it is visible to the kernel.
func (tq *TXQueue) Notify() {
tq.producer.Store(tq.cachedProducer)
tq.kick()
}
// Set sets the TX queue's descriptor at index to addr.
func (tq *TXQueue) Set(index uint32, desc unix.XDPDesc) {
// Use mask to avoid overflowing and loop back around the ring.
tq.ring[index&tq.mask] = desc
}

107
pkg/xdp/umem.go Normal file
View file

@ -0,0 +1,107 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64 || arm64
// +build amd64 arm64
package xdp
import (
"fmt"
"github.com/sagernet/gvisor/pkg/sync"
"golang.org/x/sys/unix"
)
// TODO(b/240191988): There's some kind of memory corruption bug that occurs
// occasionally. This occurred even before TX was supported.
// TODO(b/240191988): We can hold locks for less time if we accept a more
// obtuse API. For example, CompletionQueue.FreeAll doesn't need to hold a
// mutex for its entire duration.
// UMEM is the shared memory area that the kernel and userspace put packets in.
type UMEM struct {
// mem is the mmap'd area shared with the kernel.
mem []byte
// sockfd is the underlying AF_XDP socket.
sockfd uint32
// frameMask masks the lower bits of an address to get the frame's
// address.
frameMask uint64
// mu protects frameAddresses and nFreeFrames.
mu sync.Mutex
// frameAddresses is a stack of available frame addresses.
// +checklocks:mu
frameAddresses []uint64
// nFreeFrames is the number of frames available and is used to index
// into frameAddresses.
// +checklocks:mu
nFreeFrames uint32
}
// SockFD returns the underlying AF_XDP socket FD.
func (um *UMEM) SockFD() uint32 {
return um.sockfd
}
// Lock locks the UMEM.
//
// +checklocksacquire:um.mu
func (um *UMEM) Lock() {
um.mu.Lock()
}
// Unlock unlocks the UMEM.
//
// +checklocksrelease:um.mu
func (um *UMEM) Unlock() {
um.mu.Unlock()
}
// FreeFrame returns the frame containing addr to the set of free frames.
//
// The UMEM must be locked during the call to FreeFrame.
//
// +checklocks:um.mu
func (um *UMEM) FreeFrame(addr uint64) {
um.frameAddresses[um.nFreeFrames] = addr
um.nFreeFrames++
}
// AllocFrame returns the address of a frame that can be enqueued to the fill
// or TX queue. It will panic if there are no frames left, so callers must call
// it no more than the number of buffers reserved via TXQueue.Reserve().
//
// The UMEM must be locked during the call to AllocFrame.
//
// +checklocks:um.mu
func (um *UMEM) AllocFrame() uint64 {
um.nFreeFrames--
return um.frameAddresses[um.nFreeFrames] & um.frameMask
}
// Get gets the bytes of the packet pointed to by desc.
func (um *UMEM) Get(desc unix.XDPDesc) []byte {
end := desc.Addr + uint64(desc.Len)
if desc.Addr&um.frameMask != (end-1)&um.frameMask {
panic(fmt.Sprintf("UMEM (%+v) access crosses frame boundaries: %+v", um, desc))
}
return um.mem[desc.Addr:end]
}

323
pkg/xdp/xdp.go Normal file
View file

@ -0,0 +1,323 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64 || arm64
// +build amd64 arm64
// Package xdp provides tools for working with AF_XDP sockets.
//
// AF_XDP shares a memory area (UMEM) with the kernel to pass packets
// back and forth. Communication is done via a number of queues.
// Briefly, the queues work as follows:
//
// - Receive: Userspace adds a descriptor to the fill queue. The
// descriptor points to an area of the UMEM that the kernel should fill
// with an incoming packet. The packet is filled by the kernel, which
// places a descriptor to the same UMEM area in the RX queue, signifying
// that userspace may read the packet.
// - Transmit: Userspace adds a descriptor to TX queue. The kernel
// sends the packet (stored in UMEM) pointed to by the descriptor.
// Upon completion, the kernel places a descriptor in the completion
// queue to notify userspace that the packet is sent and the UMEM
// area can be reused.
//
// So in short: RX packets move from the fill to RX queue, and TX
// packets move from the TX to completion queue.
//
// Note that the shared UMEM for RX and TX means that packet forwarding
// can be done without copying; only the queues need to be updated to point to
// the packet in UMEM.
package xdp
import (
"fmt"
"math/bits"
"github.com/sagernet/gvisor/pkg/cleanup"
"github.com/sagernet/gvisor/pkg/log"
"github.com/sagernet/gvisor/pkg/memutil"
"golang.org/x/sys/unix"
)
// A ControlBlock contains all the control structures necessary to use an
// AF_XDP socket.
//
// The ControlBlock and the structures it contains are meant to be used with a
// single RX goroutine and a single TX goroutine.
type ControlBlock struct {
UMEM UMEM
Fill FillQueue
RX RXQueue
TX TXQueue
Completion CompletionQueue
}
// Opts configure an AF_XDP socket.
type Opts struct {
NFrames uint32
FrameSize uint32
NDescriptors uint32
Bind bool
UseNeedWakeup bool
}
// DefaultOpts provides recommended default options for initializing an AF_XDP
// socket. AF_XDP setup is extremely finnicky and can fail if incorrect values
// are used.
func DefaultOpts() Opts {
return Opts{
NFrames: 4096,
// Frames must be 2048 or 4096 bytes, although not all drivers support
// both.
FrameSize: 4096,
NDescriptors: 2048,
}
}
// New returns an initialized AF_XDP socket bound to a particular interface and
// queue.
func New(ifaceIdx, queueID uint32, opts Opts) (*ControlBlock, error) {
sockfd, err := unix.Socket(unix.AF_XDP, unix.SOCK_RAW, 0)
if err != nil {
return nil, fmt.Errorf("failed to create AF_XDP socket: %v", err)
}
return NewFromSocket(sockfd, ifaceIdx, queueID, opts)
}
// NewFromSocket takes an AF_XDP socket, initializes it, and binds it to a
// particular interface and queue.
func NewFromSocket(sockfd int, ifaceIdx, queueID uint32, opts Opts) (*ControlBlock, error) {
if opts.FrameSize != 2048 && opts.FrameSize != 4096 {
return nil, fmt.Errorf("invalid frame size %d: must be either 2048 or 4096", opts.FrameSize)
}
if bits.OnesCount32(opts.NDescriptors) != 1 {
return nil, fmt.Errorf("invalid number of descriptors %d: must be a power of 2", opts.NDescriptors)
}
var cb ControlBlock
// Create the UMEM area. Use mmap instead of make([[]byte) to ensure
// that the UMEM is page-aligned. Aligning the UMEM keeps individual
// packets from spilling over between pages.
var zerofd uintptr
umemMemory, err := memutil.MapSlice(
0,
uintptr(opts.NFrames*opts.FrameSize),
unix.PROT_READ|unix.PROT_WRITE,
unix.MAP_PRIVATE|unix.MAP_ANONYMOUS,
zerofd-1,
0,
)
if err != nil {
return nil, fmt.Errorf("failed to mmap umem: %v", err)
}
cleanup := cleanup.Make(func() {
memutil.UnmapSlice(umemMemory)
})
if sliceBackingPointer(umemMemory)%uintptr(unix.Getpagesize()) != 0 {
return nil, fmt.Errorf("UMEM is not page aligned (address 0x%x)", sliceBackingPointer(umemMemory))
}
cb.UMEM = UMEM{
mem: umemMemory,
sockfd: uint32(sockfd),
frameAddresses: make([]uint64, opts.NFrames),
nFreeFrames: opts.NFrames,
frameMask: ^(uint64(opts.FrameSize) - 1),
}
// Fill in each frame address.
for i := range cb.UMEM.frameAddresses {
cb.UMEM.frameAddresses[i] = uint64(i) * uint64(opts.FrameSize)
}
// Check whether we're likely to fail due to RLIMIT_MEMLOCK.
var rlimit unix.Rlimit
if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlimit); err != nil {
return nil, fmt.Errorf("failed to get rlimit for memlock: %v", err)
}
if rlimit.Cur < uint64(len(cb.UMEM.mem)) {
log.Infof("UMEM size (%d) may exceed RLIMIT_MEMLOCK (%+v) and cause registration to fail", len(cb.UMEM.mem), rlimit)
}
reg := unix.XDPUmemReg{
Addr: uint64(sliceBackingPointer(umemMemory)),
Len: uint64(len(umemMemory)),
Size: opts.FrameSize,
// Not useful in the RX path.
Headroom: 0,
// TODO(b/240191988): Investigate use of SHARED flag.
Flags: 0,
}
if err := registerUMEM(sockfd, reg); err != nil {
return nil, fmt.Errorf("failed to register UMEM: %v", err)
}
// Set the number of descriptors in the fill queue.
if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_FILL_RING, int(opts.NDescriptors)); err != nil {
return nil, fmt.Errorf("failed to register fill ring: %v", err)
}
// Set the number of descriptors in the completion queue.
if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_COMPLETION_RING, int(opts.NDescriptors)); err != nil {
return nil, fmt.Errorf("failed to register completion ring: %v", err)
}
// Set the number of descriptors in the RX queue.
if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_RX_RING, int(opts.NDescriptors)); err != nil {
return nil, fmt.Errorf("failed to register RX queue: %v", err)
}
// Set the number of descriptors in the TX queue.
if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_TX_RING, int(opts.NDescriptors)); err != nil {
return nil, fmt.Errorf("failed to register TX queue: %v", err)
}
// Get offset information for the queues. Offsets indicate where, once
// we mmap space for each queue, values in the queue are. They give
// offsets for the shared pointers, a shared flags value, and the
// beginning of the ring of descriptors.
off, err := getOffsets(sockfd)
if err != nil {
return nil, fmt.Errorf("failed to get offsets: %v", err)
}
// Allocate space for the fill queue.
fillQueueMem, err := memutil.MapSlice(
0,
uintptr(off.Fr.Desc+uint64(opts.NDescriptors)*sizeOfFillQueueDesc()),
unix.PROT_READ|unix.PROT_WRITE,
unix.MAP_SHARED|unix.MAP_POPULATE,
uintptr(sockfd),
unix.XDP_UMEM_PGOFF_FILL_RING,
)
if err != nil {
return nil, fmt.Errorf("failed to mmap fill queue: %v", err)
}
cleanup.Add(func() {
memutil.UnmapSlice(fillQueueMem)
})
// Setup the fillQueue with offsets into allocated memory.
cb.Fill = FillQueue{
mem: fillQueueMem,
mask: opts.NDescriptors - 1,
cachedConsumer: opts.NDescriptors,
}
cb.Fill.init(off, opts)
// Allocate space for the completion queue.
completionQueueMem, err := memutil.MapSlice(
0,
uintptr(off.Cr.Desc+uint64(opts.NDescriptors)*sizeOfCompletionQueueDesc()),
unix.PROT_READ|unix.PROT_WRITE,
unix.MAP_SHARED|unix.MAP_POPULATE,
uintptr(sockfd),
unix.XDP_UMEM_PGOFF_COMPLETION_RING,
)
if err != nil {
return nil, fmt.Errorf("failed to mmap completion queue: %v", err)
}
cleanup.Add(func() {
memutil.UnmapSlice(completionQueueMem)
})
// Setup the completionQueue with offsets into allocated memory.
cb.Completion = CompletionQueue{
mem: completionQueueMem,
mask: opts.NDescriptors - 1,
}
cb.Completion.init(off, opts)
// Allocate space for the RX queue.
rxQueueMem, err := memutil.MapSlice(
0,
uintptr(off.Rx.Desc+uint64(opts.NDescriptors)*sizeOfRXQueueDesc()),
unix.PROT_READ|unix.PROT_WRITE,
unix.MAP_SHARED|unix.MAP_POPULATE,
uintptr(sockfd),
unix.XDP_PGOFF_RX_RING,
)
if err != nil {
return nil, fmt.Errorf("failed to mmap RX queue: %v", err)
}
cleanup.Add(func() {
memutil.UnmapSlice(rxQueueMem)
})
// Setup the rxQueue with offsets into allocated memory.
cb.RX = RXQueue{
mem: rxQueueMem,
mask: opts.NDescriptors - 1,
}
cb.RX.init(off, opts)
// Allocate space for the TX queue.
txQueueMem, err := memutil.MapSlice(
0,
uintptr(off.Tx.Desc+uint64(opts.NDescriptors)*sizeOfTXQueueDesc()),
unix.PROT_READ|unix.PROT_WRITE,
unix.MAP_SHARED|unix.MAP_POPULATE,
uintptr(sockfd),
unix.XDP_PGOFF_TX_RING,
)
if err != nil {
return nil, fmt.Errorf("failed to mmap tx queue: %v", err)
}
cleanup.Add(func() {
memutil.UnmapSlice(txQueueMem)
})
// Setup the txQueue with offsets into allocated memory.
cb.TX = TXQueue{
sockfd: uint32(sockfd),
mem: txQueueMem,
mask: opts.NDescriptors - 1,
cachedConsumer: opts.NDescriptors,
}
cb.TX.init(off, opts)
// In some cases we don't call bind, as we're not in the netns with the
// device. In those cases, another process with the same socket will
// bind for us.
if opts.Bind {
if err := Bind(sockfd, ifaceIdx, queueID, opts.UseNeedWakeup); err != nil {
return nil, fmt.Errorf("failed to bind to interface %d: %v", ifaceIdx, err)
}
}
cleanup.Release()
return &cb, nil
}
// Bind binds a socket to a particular network interface and queue.
func Bind(sockfd int, ifindex, queueID uint32, useNeedWakeup bool) error {
var flags uint16
if useNeedWakeup {
flags |= unix.XDP_USE_NEED_WAKEUP
}
addr := unix.SockaddrXDP{
// XDP_USE_NEED_WAKEUP lets the driver sleep if there is no
// work to do. It will need to be woken by poll. It is expected
// that this improves performance by preventing the driver from
// burning cycles.
//
// By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct
// the kernel to use zerocopy if available and then fallback to
// copy mode.
Flags: flags,
Ifindex: ifindex,
// AF_XDP sockets are per device RX queue, although multiple
// sockets on multiple queues (or devices) can share a single
// UMEM.
QueueID: queueID,
// We're not using shared mode, so the value here is irrelevant.
SharedUmemFD: 0,
}
return unix.Bind(sockfd, &addr)
}

11
pkg/xdp/xdp_state_autogen.go Normal file
View file

@ -0,0 +1,11 @@
// automatically generated by stateify.
//go:build (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64)
// +build amd64 arm64
// +build amd64 arm64
// +build amd64 arm64
// +build amd64 arm64
// +build amd64 arm64
// +build amd64 arm64
package xdp

123
pkg/xdp/xdp_unsafe.go Normal file
View file

@ -0,0 +1,123 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package xdp
import (
"fmt"
"reflect"
"unsafe"
"github.com/sagernet/gvisor/pkg/atomicbitops"
"golang.org/x/sys/unix"
)
func registerUMEM(fd int, reg unix.XDPUmemReg) error {
if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_UMEM_REG, uintptr(unsafe.Pointer(&reg)), unsafe.Sizeof(reg), 0); errno != 0 {
return fmt.Errorf("failed to setsockopt(XDP_UMEM_REG): errno %d", errno)
}
return nil
}
func getOffsets(fd int) (unix.XDPMmapOffsets, error) {
var off unix.XDPMmapOffsets
size := unsafe.Sizeof(off)
if _, _, errno := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_MMAP_OFFSETS, uintptr(unsafe.Pointer(&off)), uintptr(unsafe.Pointer(&size)), 0); errno != 0 {
return unix.XDPMmapOffsets{}, fmt.Errorf("failed to get offsets: %v", errno)
} else if unsafe.Sizeof(off) != size {
return unix.XDPMmapOffsets{}, fmt.Errorf("expected optlen of %d, but found %d", unsafe.Sizeof(off), size)
}
return off, nil
}
func sliceBackingPointer(slice []byte) uintptr {
return uintptr(unsafe.Pointer(&slice[0]))
}
func sizeOfFillQueueDesc() uint64 {
return uint64(unsafe.Sizeof(uint64(0)))
}
func sizeOfRXQueueDesc() uint64 {
return uint64(unsafe.Sizeof(unix.XDPDesc{}))
}
func sizeOfCompletionQueueDesc() uint64 {
return uint64(unsafe.Sizeof(uint64(0)))
}
func sizeOfTXQueueDesc() uint64 {
return uint64(unsafe.Sizeof(unix.XDPDesc{}))
}
func (fq *FillQueue) init(off unix.XDPMmapOffsets, opts Opts) {
fillQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&fq.ring))
fillQueueRingHdr.Data = uintptr(unsafe.Pointer(&fq.mem[off.Fr.Desc]))
fillQueueRingHdr.Len = int(opts.NDescriptors)
fillQueueRingHdr.Cap = fillQueueRingHdr.Len
fq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Producer]))
fq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Consumer]))
fq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Flags]))
}
func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts Opts) {
rxQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&rq.ring))
rxQueueRingHdr.Data = uintptr(unsafe.Pointer(&rq.mem[off.Rx.Desc]))
rxQueueRingHdr.Len = int(opts.NDescriptors)
rxQueueRingHdr.Cap = rxQueueRingHdr.Len
rq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Producer]))
rq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Consumer]))
rq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Flags]))
// These probably don't have to be atomic, but we're only loading once
// so better safe than sorry.
rq.cachedProducer = rq.producer.Load()
rq.cachedConsumer = rq.consumer.Load()
}
func (cq *CompletionQueue) init(off unix.XDPMmapOffsets, opts Opts) {
completionQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cq.ring))
completionQueueRingHdr.Data = uintptr(unsafe.Pointer(&cq.mem[off.Cr.Desc]))
completionQueueRingHdr.Len = int(opts.NDescriptors)
completionQueueRingHdr.Cap = completionQueueRingHdr.Len
cq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Producer]))
cq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Consumer]))
cq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Flags]))
// These probably don't have to be atomic, but we're only loading once
// so better safe than sorry.
cq.cachedProducer = cq.producer.Load()
cq.cachedConsumer = cq.consumer.Load()
}
func (tq *TXQueue) init(off unix.XDPMmapOffsets, opts Opts) {
txQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&tq.ring))
txQueueRingHdr.Data = uintptr(unsafe.Pointer(&tq.mem[off.Tx.Desc]))
txQueueRingHdr.Len = int(opts.NDescriptors)
txQueueRingHdr.Cap = txQueueRingHdr.Len
tq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Producer]))
tq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Consumer]))
tq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Flags]))
}
// kick notifies the kernel that there are packets to transmit.
func (tq *TXQueue) kick() error {
if tq.flags.RacyLoad()&unix.XDP_RING_NEED_WAKEUP == 0 {
return nil
}
var msg unix.Msghdr
if _, _, errno := unix.Syscall6(unix.SYS_SENDMSG, uintptr(tq.sockfd), uintptr(unsafe.Pointer(&msg)), unix.MSG_DONTWAIT|unix.MSG_NOSIGNAL, 0, 0, 0); errno != 0 {
return fmt.Errorf("failed to kick TX queue via sendmsg: errno %d", errno)
}
return nil
}

3
pkg/xdp/xdp_unsafe_state_autogen.go Normal file
View file

@ -0,0 +1,3 @@
// automatically generated by stateify.
package xdp