// Copyright 2022 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package xdp import ( "fmt" "reflect" "unsafe" "github.com/sagernet/gvisor/pkg/atomicbitops" "golang.org/x/sys/unix" ) func registerUMEM(fd int, reg unix.XDPUmemReg) error { if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_UMEM_REG, uintptr(unsafe.Pointer(®)), unsafe.Sizeof(reg), 0); errno != 0 { return fmt.Errorf("failed to setsockopt(XDP_UMEM_REG): errno %d", errno) } return nil } func getOffsets(fd int) (unix.XDPMmapOffsets, error) { var off unix.XDPMmapOffsets size := unsafe.Sizeof(off) if _, _, errno := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_MMAP_OFFSETS, uintptr(unsafe.Pointer(&off)), uintptr(unsafe.Pointer(&size)), 0); errno != 0 { return unix.XDPMmapOffsets{}, fmt.Errorf("failed to get offsets: %v", errno) } else if unsafe.Sizeof(off) != size { return unix.XDPMmapOffsets{}, fmt.Errorf("expected optlen of %d, but found %d", unsafe.Sizeof(off), size) } return off, nil } func sliceBackingPointer(slice []byte) uintptr { return uintptr(unsafe.Pointer(&slice[0])) } func sizeOfFillQueueDesc() uint64 { return uint64(unsafe.Sizeof(uint64(0))) } func sizeOfRXQueueDesc() uint64 { return uint64(unsafe.Sizeof(unix.XDPDesc{})) } func sizeOfCompletionQueueDesc() uint64 { return uint64(unsafe.Sizeof(uint64(0))) } func sizeOfTXQueueDesc() uint64 { return uint64(unsafe.Sizeof(unix.XDPDesc{})) } func (fq *FillQueue) init(off unix.XDPMmapOffsets, opts Opts) { fillQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&fq.ring)) fillQueueRingHdr.Data = uintptr(unsafe.Pointer(&fq.mem[off.Fr.Desc])) fillQueueRingHdr.Len = int(opts.NDescriptors) fillQueueRingHdr.Cap = fillQueueRingHdr.Len fq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Producer])) fq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Consumer])) fq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Flags])) } func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts Opts) { rxQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&rq.ring)) rxQueueRingHdr.Data = uintptr(unsafe.Pointer(&rq.mem[off.Rx.Desc])) rxQueueRingHdr.Len = int(opts.NDescriptors) rxQueueRingHdr.Cap = rxQueueRingHdr.Len rq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Producer])) rq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Consumer])) rq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Flags])) // These probably don't have to be atomic, but we're only loading once // so better safe than sorry. rq.cachedProducer = rq.producer.Load() rq.cachedConsumer = rq.consumer.Load() } func (cq *CompletionQueue) init(off unix.XDPMmapOffsets, opts Opts) { completionQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cq.ring)) completionQueueRingHdr.Data = uintptr(unsafe.Pointer(&cq.mem[off.Cr.Desc])) completionQueueRingHdr.Len = int(opts.NDescriptors) completionQueueRingHdr.Cap = completionQueueRingHdr.Len cq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Producer])) cq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Consumer])) cq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Flags])) // These probably don't have to be atomic, but we're only loading once // so better safe than sorry. cq.cachedProducer = cq.producer.Load() cq.cachedConsumer = cq.consumer.Load() } func (tq *TXQueue) init(off unix.XDPMmapOffsets, opts Opts) { txQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&tq.ring)) txQueueRingHdr.Data = uintptr(unsafe.Pointer(&tq.mem[off.Tx.Desc])) txQueueRingHdr.Len = int(opts.NDescriptors) txQueueRingHdr.Cap = txQueueRingHdr.Len tq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Producer])) tq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Consumer])) tq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Flags])) } // kick notifies the kernel that there are packets to transmit. func (tq *TXQueue) kick() error { if tq.flags.RacyLoad()&unix.XDP_RING_NEED_WAKEUP == 0 { return nil } var msg unix.Msghdr if _, _, errno := unix.Syscall6(unix.SYS_SENDMSG, uintptr(tq.sockfd), uintptr(unsafe.Pointer(&msg)), unix.MSG_DONTWAIT|unix.MSG_NOSIGNAL, 0, 0, 0); errno != 0 { return fmt.Errorf("failed to kick TX queue via sendmsg: errno %d", errno) } return nil }