From 2c4ae3b0a40e5248963dc8150cb9f07e0e1f2034 Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:50:08 +0300 Subject: [PATCH] snapshot: sagernet/gvisor v0.0.0-20250811.0-sing-box-mod.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Содержимое пина, зафиксированного в go.mod sing-box-lx, одним коммитом без истории. Полная история SagerNet/gvisor — 1.45 ГБ и клонируется в каждой CI-джобе; наша дельта — одна вставка в одну функцию, история для неё не нужна. Module path github.com/sagernet/gvisor сохранён намеренно: на него опирается replace-директива суперпроекта. Патч поверх — отдельным коммитом, чтобы дельта читалась одним git show и переносилась на новый пин копированием. SPECS/TASKS/048-GVISOR_HANDSHAKE_NIL_CRASH --- .gitignore | 2 + AUTHORS | 8 + LICENSE | 254 + README.md | 5 + go.mod | 9 + go.sum | 6 + pkg/abi/abi.go | 44 + pkg/abi/abi_linux.go | 21 + pkg/abi/abi_linux_state_autogen.go | 6 + pkg/abi/abi_state_autogen.go | 3 + pkg/abi/flag.go | 85 + pkg/abi/gasket/gasket.go | 127 + pkg/abi/gasket/gasket_abi_autogen_unsafe.go | 428 + pkg/abi/gasket/gasket_state_autogen.go | 3 + pkg/abi/linux/aio.go | 96 + pkg/abi/linux/arch_amd64.go | 24 + pkg/abi/linux/audit.go | 23 + pkg/abi/linux/bpf.go | 35 + pkg/abi/linux/capability.go | 337 + pkg/abi/linux/clone.go | 71 + pkg/abi/linux/context.go | 36 + pkg/abi/linux/dev.go | 72 + pkg/abi/linux/elf.go | 158 + pkg/abi/linux/epoll.go | 58 + pkg/abi/linux/epoll_amd64.go | 30 + pkg/abi/linux/epoll_arm64.go | 29 + pkg/abi/linux/errno/errno.go | 187 + pkg/abi/linux/errno/errno_state_autogen.go | 3 + pkg/abi/linux/errqueue.go | 93 + pkg/abi/linux/eventfd.go | 22 + pkg/abi/linux/exec.go | 18 + pkg/abi/linux/fadvise.go | 25 + pkg/abi/linux/fcntl.go | 79 + pkg/abi/linux/file.go | 437 + pkg/abi/linux/file_amd64.go | 47 + pkg/abi/linux/file_arm64.go | 48 + pkg/abi/linux/fs.go | 129 + pkg/abi/linux/fuse.go | 1139 + pkg/abi/linux/futex.go | 80 + pkg/abi/linux/inotify.go | 97 + pkg/abi/linux/ioctl.go | 186 + pkg/abi/linux/ioctl_tun.go | 33 + pkg/abi/linux/iouring.go | 238 + pkg/abi/linux/ip.go | 171 + pkg/abi/linux/ipc.go | 61 + pkg/abi/linux/keyctl.go | 29 + pkg/abi/linux/limits.go | 88 + pkg/abi/linux/linux.go | 42 + pkg/abi/linux/linux_abi_autogen_unsafe.go | 22507 ++++++++++++++++ .../linux/linux_amd64_abi_autogen_unsafe.go | 760 + pkg/abi/linux/linux_amd64_state_autogen.go | 119 + .../linux/linux_arm64_abi_autogen_unsafe.go | 673 + pkg/abi/linux/linux_arm64_state_autogen.go | 50 + pkg/abi/linux/linux_state_autogen.go | 599 + .../linux/linux_unsafe_abi_autogen_unsafe.go | 3 + pkg/abi/linux/linux_unsafe_state_autogen.go | 3 + pkg/abi/linux/membarrier.go | 34 + pkg/abi/linux/mm.go | 160 + pkg/abi/linux/mm_amd64.go | 33 + pkg/abi/linux/mm_arm64.go | 25 + pkg/abi/linux/mqueue.go | 55 + pkg/abi/linux/msgqueue.go | 108 + pkg/abi/linux/netdevice.go | 128 + pkg/abi/linux/netfilter.go | 796 + pkg/abi/linux/netfilter_arp.go | 24 + pkg/abi/linux/netfilter_bridge.go | 41 + pkg/abi/linux/netfilter_ipv4.go | 36 + pkg/abi/linux/netfilter_ipv6.go | 314 + pkg/abi/linux/netlink.go | 159 + pkg/abi/linux/netlink_netfilter.go | 92 + pkg/abi/linux/netlink_route.go | 377 + pkg/abi/linux/nf_tables.go | 464 + pkg/abi/linux/poll.go | 44 + pkg/abi/linux/prctl.go | 179 + pkg/abi/linux/ptrace.go | 95 + pkg/abi/linux/ptrace_amd64.go | 69 + pkg/abi/linux/ptrace_arm64.go | 77 + pkg/abi/linux/rseq.go | 130 + pkg/abi/linux/rusage.go | 48 + pkg/abi/linux/sched.go | 37 + pkg/abi/linux/seccomp.go | 173 + pkg/abi/linux/sem.go | 82 + pkg/abi/linux/sem_amd64.go | 34 + pkg/abi/linux/sem_arm64.go | 32 + pkg/abi/linux/shm.go | 92 + pkg/abi/linux/signal.go | 552 + pkg/abi/linux/signalfd.go | 47 + pkg/abi/linux/socket.go | 740 + pkg/abi/linux/splice.go | 23 + pkg/abi/linux/tcp.go | 71 + pkg/abi/linux/time.go | 288 + pkg/abi/linux/timer.go | 23 + pkg/abi/linux/tty.go | 349 + pkg/abi/linux/uio.go | 18 + pkg/abi/linux/utsname.go | 51 + pkg/abi/linux/vfio.go | 215 + pkg/abi/linux/vfio_unsafe.go | 22 + pkg/abi/linux/wait.go | 161 + pkg/abi/linux/xattr.go | 39 + pkg/abi/nvgpu/classes.go | 590 + pkg/abi/nvgpu/ctrl.go | 719 + pkg/abi/nvgpu/frontend.go | 805 + pkg/abi/nvgpu/frontend_unsafe.go | 27 + pkg/abi/nvgpu/nvgpu.go | 92 + pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go | 17491 ++++++++++++ pkg/abi/nvgpu/nvgpu_state_autogen.go | 114 + .../nvgpu/nvgpu_unsafe_abi_autogen_unsafe.go | 3 + pkg/abi/nvgpu/nvgpu_unsafe_state_autogen.go | 3 + pkg/abi/nvgpu/status.go | 26 + pkg/abi/nvgpu/uvm.go | 778 + pkg/abi/sentry/sentry.go | 16 + pkg/abi/sentry/sentry_state_autogen.go | 3 + pkg/abi/sentry/syscall.go | 20 + pkg/abi/tpu/tpu.go | 155 + pkg/abi/tpu/tpu_state_autogen.go | 3 + pkg/atomicbitops/32b_32bit.go | 289 + pkg/atomicbitops/32b_64bit.go | 289 + pkg/atomicbitops/aligned_32bit_unsafe.go | 231 + pkg/atomicbitops/aligned_64bit.go | 212 + pkg/atomicbitops/atomicbitops.go | 82 + .../atomicbitops_32bit_state_autogen.go | 93 + ...atomicbitops_32bit_unsafe_state_autogen.go | 73 + .../atomicbitops_64bit_state_autogen.go | 145 + pkg/atomicbitops/atomicbitops_amd64.s | 77 + pkg/atomicbitops/atomicbitops_arm64.go | 40 + pkg/atomicbitops/atomicbitops_arm64.s | 141 + .../atomicbitops_arm64_state_autogen.go | 6 + pkg/atomicbitops/atomicbitops_float64.go | 105 + pkg/atomicbitops/atomicbitops_noasm.go | 112 + .../atomicbitops_state_autogen.go | 43 + pkg/binary/binary.go | 266 + pkg/binary/binary_state_autogen.go | 3 + pkg/bits/bits.go | 26 + pkg/bits/bits32.go | 33 + pkg/bits/bits64.go | 33 + pkg/bits/bits_state_autogen.go | 8 + pkg/bits/uint64_arch.go | 37 + pkg/bits/uint64_arch_amd64_asm.s | 32 + pkg/bits/uint64_arch_arm64_asm.s | 34 + pkg/bits/uint64_arch_generic.go | 56 + pkg/buffer/buffer.go | 660 + pkg/buffer/buffer_state.go | 29 + pkg/buffer/buffer_state_autogen.go | 187 + pkg/buffer/buffer_unsafe_state_autogen.go | 3 + pkg/buffer/chunk.go | 113 + pkg/buffer/chunk_refs.go | 141 + pkg/buffer/view.go | 366 + pkg/buffer/view_list.go | 239 + pkg/buffer/view_unsafe.go | 26 + pkg/cleanup/cleanup.go | 61 + pkg/cleanup/cleanup_state_autogen.go | 3 + pkg/compressio/compressio.go | 788 + pkg/compressio/compressio_state_autogen.go | 3 + pkg/compressio/nocompressio.go | 352 + pkg/context/context.go | 238 + pkg/context/context_state_autogen.go | 3 + pkg/coretag/coretag.go | 99 + pkg/coretag/coretag_state_autogen.go | 3 + pkg/coretag/coretag_unsafe.go | 34 + pkg/coretag/coretag_unsafe_state_autogen.go | 3 + pkg/cpuid/cpuid.go | 264 + pkg/cpuid/cpuid_amd64.go | 496 + pkg/cpuid/cpuid_amd64_state_autogen.go | 110 + pkg/cpuid/cpuid_arm64.go | 154 + pkg/cpuid/cpuid_arm64_state_autogen.go | 59 + pkg/cpuid/cpuid_state_autogen.go | 41 + pkg/cpuid/features_amd64.go | 664 + pkg/cpuid/features_arm64.go | 147 + pkg/cpuid/hwcap_amd64.go | 24 + pkg/cpuid/hwcap_arm64.go | 79 + pkg/cpuid/native_amd64.go | 250 + pkg/cpuid/native_amd64.s | 38 + pkg/cpuid/native_arm64.go | 165 + pkg/cpuid/static_amd64.go | 135 + pkg/errors/errors.go | 40 + pkg/errors/errors_state_autogen.go | 3 + pkg/errors/linuxerr/internal.go | 129 + pkg/errors/linuxerr/linuxerr.go | 368 + pkg/errors/linuxerr/linuxerr_state_autogen.go | 3 + pkg/eventfd/eventfd.go | 179 + pkg/eventfd/eventfd_state_autogen.go | 3 + pkg/eventfd/eventfd_unsafe.go | 40 + pkg/eventfd/eventfd_unsafe_state_autogen.go | 3 + pkg/fd/fd.go | 273 + pkg/fd/fd_state_autogen.go | 3 + pkg/fdchannel/fdchannel_unsafe.go | 143 + .../fdchannel_unsafe_state_autogen.go | 6 + pkg/fdnotifier/fdnotifier.go | 215 + pkg/fdnotifier/fdnotifier_state_autogen.go | 6 + .../fdnotifier_unsafe_state_autogen.go | 6 + pkg/fdnotifier/poll_unsafe.go | 82 + pkg/flipcall/ctrl_futex.go | 215 + pkg/flipcall/flipcall.go | 282 + pkg/flipcall/flipcall_linux_state_autogen.go | 6 + pkg/flipcall/flipcall_state_autogen.go | 6 + pkg/flipcall/flipcall_unsafe.go | 87 + pkg/flipcall/flipcall_unsafe_state_autogen.go | 3 + pkg/flipcall/futex_linux.go | 88 + pkg/flipcall/io.go | 113 + pkg/flipcall/packet_window.go | 166 + pkg/fspath/builder.go | 112 + pkg/fspath/fspath.go | 187 + pkg/fspath/fspath_state_autogen.go | 3 + pkg/fsutil/fsutil.go | 48 + pkg/fsutil/fsutil_amd64_unsafe.go | 49 + .../fsutil_amd64_unsafe_state_autogen.go | 6 + pkg/fsutil/fsutil_arm64_unsafe.go | 49 + .../fsutil_arm64_unsafe_state_autogen.go | 6 + pkg/fsutil/fsutil_state_autogen.go | 3 + pkg/fsutil/fsutil_unsafe.go | 121 + pkg/fsutil/fsutil_unsafe_state_autogen.go | 3 + pkg/gohacks/linkname_go113_unsafe.go | 51 + pkg/gohacks/noescape_unsafe.go | 34 + pkg/gohacks/slice_go113_unsafe.go | 45 + pkg/gohacks/slice_go120_unsafe.go | 30 + pkg/gohacks/string_go113_unsafe.go | 51 + pkg/gohacks/string_go120_unsafe.go | 39 + pkg/goid/goid.go | 28 + pkg/goid/goid_122_amd64.s | 26 + pkg/goid/goid_122_arm64.s | 26 + pkg/goid/goid_123_amd64.s | 26 + pkg/goid/goid_123_arm64.s | 26 + pkg/goid/goid_125_amd64.s | 26 + pkg/goid/goid_125_arm64.s | 26 + pkg/gomaxprocs/gomaxprocs.go | 82 + pkg/gomaxprocs/gomaxprocs_mutex.go | 64 + pkg/gomaxprocs/gomaxprocs_state_autogen.go | 3 + pkg/hostarch/access_type.go | 127 + pkg/hostarch/addr.go | 119 + pkg/hostarch/addr_range.go | 76 + pkg/hostarch/addr_range_seq_unsafe.go | 277 + pkg/hostarch/addr_unsafe.go | 22 + pkg/hostarch/hostarch.go | 8 + pkg/hostarch/hostarch_arm64.go | 98 + pkg/hostarch/hostarch_arm64_state_autogen.go | 6 + pkg/hostarch/hostarch_state_autogen.go | 82 + pkg/hostarch/hostarch_unsafe_state_autogen.go | 3 + pkg/hostarch/hostarch_x86.go | 48 + pkg/hostarch/hostarch_x86_state_autogen.go | 6 + pkg/hostarch/memory_type.go | 84 + pkg/hostarch/sizes_util.go | 114 + pkg/hostsyscall/hostsyscall.go | 45 + pkg/hostsyscall/hostsyscall_amd64.s | 96 + pkg/hostsyscall/hostsyscall_arm64.s | 97 + pkg/hostsyscall/hostsyscall_state_autogen.go | 6 + pkg/hosttid/hosttid.go | 30 + pkg/hosttid/hosttid_124_amd64.s | 29 + pkg/hosttid/hosttid_124_arm64.s | 28 + pkg/hosttid/hosttid_125_amd64.s | 29 + pkg/hosttid/hosttid_125_arm64.s | 28 + pkg/hosttid/hosttid_state_autogen.go | 3 + pkg/ilist/ilist_state_autogen.go | 70 + pkg/ilist/interface_list.go | 258 + pkg/linewriter/linewriter.go | 79 + pkg/log/bug.go | 161 + pkg/log/glog.go | 86 + pkg/log/json.go | 85 + pkg/log/json_k8s.go | 56 + pkg/log/log.go | 401 + pkg/log/rate_limited.go | 63 + pkg/marshal/marshal.go | 226 + pkg/marshal/marshal_state_autogen.go | 3 + pkg/marshal/primitive/primitive.go | 405 + .../primitive/primitive_abi_autogen_unsafe.go | 1586 ++ .../primitive/primitive_state_autogen.go | 3 + pkg/marshal/util.go | 44 + pkg/memutil/memfd_linux_unsafe.go | 41 + .../memutil_linux_unsafe_state_autogen.go | 6 + pkg/memutil/memutil_state_autogen.go | 6 + pkg/memutil/memutil_unsafe.go | 44 + pkg/memutil/memutil_unsafe_state_autogen.go | 3 + pkg/memutil/mmap.go | 32 + pkg/pool/pool.go | 67 + pkg/pool/pool_state_autogen.go | 3 + pkg/rand/rand.go | 28 + pkg/rand/rand_linux.go | 82 + pkg/rand/rand_linux_state_autogen.go | 3 + pkg/rand/rand_state_autogen.go | 6 + pkg/rand/rng.go | 131 + pkg/rawfile/blockingpoll_amd64.s | 41 + pkg/rawfile/blockingpoll_arm64.s | 42 + pkg/rawfile/blockingpoll_noyield_unsafe.go | 33 + pkg/rawfile/blockingpoll_yield_unsafe.go | 69 + pkg/rawfile/rawfile_unsafe.go | 233 + pkg/rawfile/rawfile_unsafe_state_autogen.go | 11 + pkg/refs/refcounter.go | 196 + pkg/refs/refs_map.go | 179 + pkg/refs/refs_state_autogen.go | 3 + pkg/ringdeque/ringdeque.go | 154 + pkg/safecopy/atomic_amd64.s | 160 + pkg/safecopy/atomic_arm64.s | 152 + pkg/safecopy/memclr_amd64.s | 155 + pkg/safecopy/memclr_arm64.s | 82 + pkg/safecopy/memcpy_amd64.s | 225 + pkg/safecopy/memcpy_arm64.s | 86 + pkg/safecopy/safecopy.go | 151 + pkg/safecopy/safecopy_amd64_unsafe.go | 42 + .../safecopy_amd64_unsafe_state_autogen.go | 6 + pkg/safecopy/safecopy_arm64.go | 21 + pkg/safecopy/safecopy_arm64_state_autogen.go | 6 + pkg/safecopy/safecopy_state_autogen.go | 3 + pkg/safecopy/safecopy_unsafe.go | 333 + pkg/safecopy/safecopy_unsafe_state_autogen.go | 3 + pkg/safecopy/sighandler_amd64.s | 147 + pkg/safecopy/sighandler_arm64.s | 149 + pkg/safecopy/xrstor_amd64.s | 59 + pkg/secio/full_reader.go | 34 + pkg/secio/secio.go | 105 + pkg/secio/secio_state_autogen.go | 3 + pkg/sighandling/sighandling.go | 113 + pkg/sighandling/sighandling_darwin.go | 40 + pkg/sighandling/sighandling_linux_unsafe.go | 103 + .../sighandling_linux_unsafe_state_autogen.go | 6 + pkg/sighandling/sighandling_state_autogen.go | 6 + pkg/sleep/sleep_unsafe.go | 476 + pkg/sleep/sleep_unsafe_state_autogen.go | 109 + pkg/state/addr_range.go | 76 + pkg/state/addr_set.go | 2017 ++ pkg/state/decode.go | 742 + pkg/state/decode_unsafe.go | 76 + pkg/state/deferred_list.go | 239 + pkg/state/encode.go | 873 + pkg/state/encode_unsafe.go | 32 + pkg/state/ods_list.go | 239 + pkg/state/pretty/pretty.go | 307 + pkg/state/pretty/pretty_state_autogen.go | 3 + pkg/state/state.go | 324 + pkg/state/state_norace.go | 20 + pkg/state/state_race.go | 20 + pkg/state/statefile/statefile.go | 338 + .../statefile/statefile_state_autogen.go | 3 + pkg/state/stats.go | 145 + pkg/state/types.go | 384 + pkg/state/wire/wire.go | 983 + pkg/sync/aliases.go | 52 + pkg/sync/checklocks_off_unsafe.go | 20 + pkg/sync/checklocks_on_unsafe.go | 110 + pkg/sync/fence.go | 19 + pkg/sync/fence_amd64.s | 26 + pkg/sync/fence_arm64.s | 23 + pkg/sync/gate_unsafe.go | 151 + pkg/sync/goyield_go113_unsafe.go | 19 + pkg/sync/goyield_unsafe.go | 21 + .../locking/atomicptrmap_ancestors_unsafe.go | 439 + .../locking/atomicptrmap_goroutine_unsafe.go | 439 + pkg/sync/locking/lockdep.go | 191 + pkg/sync/locking/lockdep_norace.go | 42 + pkg/sync/locking/locking.go | 28 + pkg/sync/mutex_unsafe.go | 80 + pkg/sync/nocopy.go | 28 + pkg/sync/norace_unsafe.go | 48 + pkg/sync/race_amd64.s | 33 + pkg/sync/race_arm64.s | 35 + pkg/sync/race_unsafe.go | 49 + pkg/sync/runtime.go | 22 + pkg/sync/runtime_amd64.go | 31 + pkg/sync/runtime_constants_go125.go | 25 + pkg/sync/runtime_constants_not_go125.go | 25 + pkg/sync/runtime_exectracer2.go | 21 + pkg/sync/runtime_go121_unsafe.go | 17 + pkg/sync/runtime_go124_unsafe.go | 16 + pkg/sync/runtime_go126_unsafe.go | 16 + pkg/sync/runtime_not_go121_unsafe.go | 19 + pkg/sync/runtime_other.go | 15 + pkg/sync/runtime_spinning_amd64.s | 26 + pkg/sync/runtime_spinning_go125_amd64.s | 26 + pkg/sync/runtime_spinning_other.s | 18 + pkg/sync/runtime_unsafe.go | 141 + pkg/sync/rwmutex_unsafe.go | 316 + pkg/sync/seqcount.go | 120 + pkg/sync/sync.go | 10 + pkg/syncevent/broadcaster.go | 220 + pkg/syncevent/receiver.go | 101 + pkg/syncevent/source.go | 61 + pkg/syncevent/syncevent.go | 32 + pkg/syncevent/syncevent_state_autogen.go | 3 + .../syncevent_unsafe_state_autogen.go | 3 + pkg/syncevent/waiter_unsafe.go | 197 + pkg/syserr/host_darwin.go | 44 + pkg/syserr/host_linux.go | 95 + pkg/syserr/netstack.go | 160 + pkg/syserr/syserr.go | 282 + pkg/syserr/syserr_linux_state_autogen.go | 6 + pkg/syserr/syserr_state_autogen.go | 6 + pkg/tcpip/adapters/gonet/err.go | 103 + pkg/tcpip/adapters/gonet/err_linuxwindows.go | 7 + pkg/tcpip/adapters/gonet/err_stub.go | 7 + pkg/tcpip/adapters/gonet/gonet.go | 714 + .../adapters/gonet/gonet_state_autogen.go | 3 + pkg/tcpip/checksum/checksum.go | 68 + pkg/tcpip/checksum/checksum_state_autogen.go | 3 + pkg/tcpip/checksum/checksum_unsafe.go | 182 + .../checksum/checksum_unsafe_state_autogen.go | 3 + pkg/tcpip/errors.go | 643 + pkg/tcpip/errors_linux.go | 74 + pkg/tcpip/faketime/faketime.go | 392 + pkg/tcpip/faketime/faketime_state_autogen.go | 172 + pkg/tcpip/hash/jenkins/jenkins.go | 79 + .../hash/jenkins/jenkins_state_autogen.go | 3 + pkg/tcpip/header/arp.go | 127 + pkg/tcpip/header/checksum.go | 107 + pkg/tcpip/header/datagram.go | 18 + pkg/tcpip/header/eth.go | 192 + pkg/tcpip/header/gue.go | 73 + pkg/tcpip/header/header_state_autogen.go | 120 + pkg/tcpip/header/icmpv4.go | 228 + pkg/tcpip/header/icmpv6.go | 304 + pkg/tcpip/header/igmp.go | 185 + pkg/tcpip/header/igmpv3.go | 500 + pkg/tcpip/header/interfaces.go | 130 + pkg/tcpip/header/ipv4.go | 1274 + pkg/tcpip/header/ipv6.go | 597 + pkg/tcpip/header/ipv6_extension_headers.go | 1010 + pkg/tcpip/header/ipv6_fragment.go | 158 + pkg/tcpip/header/mld.go | 103 + pkg/tcpip/header/mldv2.go | 539 + pkg/tcpip/header/mldv2_igmpv3_common.go | 124 + pkg/tcpip/header/ndp_neighbor_advert.go | 110 + pkg/tcpip/header/ndp_neighbor_solicit.go | 52 + pkg/tcpip/header/ndp_options.go | 1072 + pkg/tcpip/header/ndp_router_advert.go | 204 + pkg/tcpip/header/ndp_router_solicit.go | 36 + .../header/ndpoptionidentifier_string.go | 56 + pkg/tcpip/header/parse/parse.go | 243 + pkg/tcpip/header/parse/parse_state_autogen.go | 3 + pkg/tcpip/header/tcp.go | 730 + pkg/tcpip/header/udp.go | 199 + pkg/tcpip/header/virtionet.go | 94 + pkg/tcpip/internal/tcp/tcp.go | 48 + pkg/tcpip/internal/tcp/tcp_state_autogen.go | 38 + pkg/tcpip/link/channel/channel.go | 321 + .../link/channel/channel_state_autogen.go | 79 + pkg/tcpip/link/channel/endpoint_mutex.go | 96 + pkg/tcpip/link/channel/queue_mutex.go | 96 + pkg/tcpip/link/ethernet/ethernet.go | 121 + .../link/ethernet/ethernet_state_autogen.go | 38 + pkg/tcpip/link/fdbased/endpoint.go | 906 + pkg/tcpip/link/fdbased/endpoint_mutex.go | 96 + pkg/tcpip/link/fdbased/endpoint_unsafe.go | 24 + .../fdbased/fdbased_nonlinux_state_autogen.go | 6 + .../link/fdbased/fdbased_state_autogen.go | 438 + .../fdbased/fdbased_unsafe_state_autogen.go | 7 + .../link/fdbased/injectable_endpoint_mutex.go | 96 + pkg/tcpip/link/fdbased/mmap.go | 199 + pkg/tcpip/link/fdbased/mmap_nonlinux.go | 24 + pkg/tcpip/link/fdbased/mmap_unsafe.go | 92 + pkg/tcpip/link/fdbased/packet_dispatchers.go | 330 + pkg/tcpip/link/fdbased/processor_mutex.go | 64 + pkg/tcpip/link/fdbased/processors.go | 278 + pkg/tcpip/link/fdbased/save_restore.go | 26 + pkg/tcpip/link/loopback/endpoint_mutex.go | 96 + pkg/tcpip/link/loopback/loopback.go | 146 + .../link/loopback/loopback_state_autogen.go | 44 + pkg/tcpip/link/muxed/endpoint_mutex.go | 96 + pkg/tcpip/link/muxed/injectable.go | 174 + pkg/tcpip/link/muxed/muxed_state_autogen.go | 41 + pkg/tcpip/link/nested/nested.go | 185 + pkg/tcpip/link/nested/nested_state_autogen.go | 44 + pkg/tcpip/link/packetsocket/packetsocket.go | 62 + .../packetsocket_state_autogen.go | 38 + pkg/tcpip/link/pipe/endpoint_mutex.go | 96 + pkg/tcpip/link/pipe/pipe.go | 154 + pkg/tcpip/link/pipe/pipe_state_autogen.go | 47 + pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go | 64 + pkg/tcpip/link/qdisc/fifo/fifo.go | 158 + .../link/qdisc/fifo/fifo_state_autogen.go | 102 + .../qdisc/fifo/packet_buffer_circular_list.go | 93 + pkg/tcpip/link/sharedmem/endpoint_mutex.go | 96 + pkg/tcpip/link/sharedmem/pipe/pipe.go | 80 + .../link/sharedmem/pipe/pipe_state_autogen.go | 111 + pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go | 36 + .../pipe/pipe_unsafe_state_autogen.go | 3 + pkg/tcpip/link/sharedmem/pipe/rx.go | 108 + pkg/tcpip/link/sharedmem/pipe/tx.go | 164 + .../sharedmem/queue/queue_state_autogen.go | 3 + pkg/tcpip/link/sharedmem/queue/rx.go | 226 + pkg/tcpip/link/sharedmem/queue/tx.go | 161 + pkg/tcpip/link/sharedmem/queuepair.go | 220 + pkg/tcpip/link/sharedmem/rx.go | 152 + .../link/sharedmem/server_endpoint_mutex.go | 96 + pkg/tcpip/link/sharedmem/server_rx.go | 162 + pkg/tcpip/link/sharedmem/server_tx.go | 194 + pkg/tcpip/link/sharedmem/sharedmem.go | 559 + pkg/tcpip/link/sharedmem/sharedmem_server.go | 399 + .../link/sharedmem/sharedmem_state_autogen.go | 309 + pkg/tcpip/link/sharedmem/sharedmem_unsafe.go | 59 + .../sharedmem_unsafe_state_autogen.go | 3 + pkg/tcpip/link/sharedmem/tx.go | 279 + pkg/tcpip/link/sniffer/pcap.go | 85 + pkg/tcpip/link/sniffer/sniffer.go | 399 + .../link/sniffer/sniffer_state_autogen.go | 47 + pkg/tcpip/link/stopfd/stopfd.go | 54 + pkg/tcpip/link/stopfd/stopfd_state_autogen.go | 41 + pkg/tcpip/link/tun/device.go | 447 + pkg/tcpip/link/tun/device_mutex.go | 96 + pkg/tcpip/link/tun/endpoint_mutex.go | 64 + pkg/tcpip/link/tun/protocol.go | 56 + pkg/tcpip/link/tun/tun_endpoint_refs.go | 141 + pkg/tcpip/link/tun/tun_state_autogen.go | 152 + pkg/tcpip/link/tun/tun_unsafe.go | 65 + .../link/tun/tun_unsafe_state_autogen.go | 6 + pkg/tcpip/link/veth/endpoint_mutex.go | 96 + pkg/tcpip/link/veth/veth.go | 258 + pkg/tcpip/link/veth/veth_mutex.go | 96 + pkg/tcpip/link/veth/veth_state_autogen.go | 111 + pkg/tcpip/link/waitable/endpoint_mutex.go | 96 + pkg/tcpip/link/waitable/waitable.go | 196 + .../link/waitable/waitable_state_autogen.go | 47 + pkg/tcpip/link/xdp/endpoint.go | 421 + pkg/tcpip/link/xdp/endpoint_mutex.go | 96 + pkg/tcpip/link/xdp/xdp_state_autogen.go | 56 + pkg/tcpip/network/arp/arp.go | 416 + pkg/tcpip/network/arp/arp_state_autogen.go | 219 + pkg/tcpip/network/arp/stats.go | 74 + pkg/tcpip/network/hash/hash.go | 93 + pkg/tcpip/network/hash/hash_state_autogen.go | 3 + .../internal/fragmentation/fragmentation.go | 375 + .../fragmentation_state_autogen.go | 246 + .../internal/fragmentation/reassembler.go | 185 + .../fragmentation/reassembler_list.go | 239 + .../ip/duplicate_address_detection.go | 304 + pkg/tcpip/network/internal/ip/errors.go | 129 + .../internal/ip/generic_multicast_protocol.go | 1192 + .../network/internal/ip/ip_state_autogen.go | 435 + pkg/tcpip/network/internal/ip/stats.go | 219 + .../multicast/multicast_state_autogen.go | 137 + .../network/internal/multicast/route_table.go | 446 + pkg/tcpip/network/ipv4/icmp.go | 709 + pkg/tcpip/network/ipv4/igmp.go | 654 + pkg/tcpip/network/ipv4/ipv4.go | 2405 ++ pkg/tcpip/network/ipv4/ipv4_export.go | 14 + pkg/tcpip/network/ipv4/ipv4_state_autogen.go | 785 + pkg/tcpip/network/ipv4/stats.go | 203 + .../dhcpv6configurationfromndpra_string.go | 40 + pkg/tcpip/network/ipv6/icmp.go | 1184 + pkg/tcpip/network/ipv6/ipv6.go | 2875 ++ pkg/tcpip/network/ipv6/ipv6_export.go | 14 + pkg/tcpip/network/ipv6/ipv6_state_autogen.go | 1049 + pkg/tcpip/network/ipv6/mld.go | 478 + pkg/tcpip/network/ipv6/ndp.go | 2033 ++ pkg/tcpip/network/ipv6/stats.go | 145 + pkg/tcpip/ports/flags.go | 152 + pkg/tcpip/ports/ports.go | 496 + pkg/tcpip/ports/ports_state_autogen.go | 163 + pkg/tcpip/route_list.go | 239 + pkg/tcpip/seqnum/seqnum.go | 62 + pkg/tcpip/seqnum/seqnum_state_autogen.go | 3 + pkg/tcpip/sock_err_list.go | 239 + pkg/tcpip/socketops.go | 773 + pkg/tcpip/stack/address_state_mutex.go | 96 + pkg/tcpip/stack/address_state_refs.go | 141 + pkg/tcpip/stack/addressable_endpoint_state.go | 950 + .../stack/addressable_endpoint_state_mutex.go | 96 + pkg/tcpip/stack/bridge.go | 306 + pkg/tcpip/stack/bridge_mutex.go | 96 + pkg/tcpip/stack/bucket_mutex.go | 97 + pkg/tcpip/stack/cleanup_endpoints_mutex.go | 64 + pkg/tcpip/stack/conn_mutex.go | 96 + pkg/tcpip/stack/conn_track_mutex.go | 96 + pkg/tcpip/stack/conntrack.go | 1169 + pkg/tcpip/stack/endpoints_by_nic_mutex.go | 96 + pkg/tcpip/stack/gro/gro.go | 603 + pkg/tcpip/stack/gro/gro_packet_list.go | 239 + pkg/tcpip/stack/gro/gro_state_autogen.go | 178 + pkg/tcpip/stack/headertype_string.go | 40 + pkg/tcpip/stack/hook_string.go | 41 + pkg/tcpip/stack/icmp_rate_limit.go | 75 + pkg/tcpip/stack/iptables.go | 716 + pkg/tcpip/stack/iptables_mutex.go | 96 + pkg/tcpip/stack/iptables_targets.go | 492 + pkg/tcpip/stack/iptables_types.go | 385 + pkg/tcpip/stack/multi_port_endpoint_mutex.go | 96 + pkg/tcpip/stack/neighbor_cache.go | 314 + pkg/tcpip/stack/neighbor_cache_mutex.go | 96 + pkg/tcpip/stack/neighbor_entry.go | 653 + pkg/tcpip/stack/neighbor_entry_list.go | 239 + pkg/tcpip/stack/neighbor_entry_mutex.go | 96 + pkg/tcpip/stack/neighborstate_string.go | 44 + pkg/tcpip/stack/nftables_types.go | 170 + pkg/tcpip/stack/nic.go | 1124 + pkg/tcpip/stack/nic_mutex.go | 96 + pkg/tcpip/stack/nic_stats.go | 84 + pkg/tcpip/stack/nud.go | 429 + pkg/tcpip/stack/packet_buffer.go | 770 + pkg/tcpip/stack/packet_buffer_list.go | 87 + pkg/tcpip/stack/packet_buffer_refs.go | 141 + pkg/tcpip/stack/packet_buffer_unsafe.go | 28 + pkg/tcpip/stack/packet_endpoint_list_mutex.go | 96 + pkg/tcpip/stack/packet_eps_mutex.go | 96 + .../packets_pending_link_resolution_mutex.go | 64 + pkg/tcpip/stack/pending_packets.go | 223 + pkg/tcpip/stack/rand.go | 40 + pkg/tcpip/stack/registration.go | 1487 + pkg/tcpip/stack/route.go | 598 + pkg/tcpip/stack/route_export.go | 14 + pkg/tcpip/stack/route_mutex.go | 96 + pkg/tcpip/stack/route_stack_mutex.go | 96 + pkg/tcpip/stack/save_restore.go | 29 + pkg/tcpip/stack/stack.go | 2508 ++ pkg/tcpip/stack/stack_mutex.go | 96 + pkg/tcpip/stack/stack_options.go | 125 + pkg/tcpip/stack/stack_state_autogen.go | 2496 ++ pkg/tcpip/stack/stack_unsafe_state_autogen.go | 3 + pkg/tcpip/stack/state_conn_mutex.go | 96 + pkg/tcpip/stack/transport_demuxer.go | 733 + pkg/tcpip/stack/transport_endpoints_mutex.go | 96 + pkg/tcpip/stack/tuple_list.go | 239 + pkg/tcpip/stdclock.go | 114 + pkg/tcpip/stdclock_state.go | 30 + pkg/tcpip/tcpip.go | 2899 ++ pkg/tcpip/tcpip_linux_state_autogen.go | 6 + pkg/tcpip/tcpip_state.go | 28 + pkg/tcpip/tcpip_state_autogen.go | 3386 +++ pkg/tcpip/timer.go | 212 + pkg/tcpip/transport/datagram.go | 49 + pkg/tcpip/transport/icmp/endpoint.go | 828 + pkg/tcpip/transport/icmp/endpoint_state.go | 92 + pkg/tcpip/transport/icmp/icmp_packet_list.go | 239 + .../transport/icmp/icmp_state_autogen.go | 204 + pkg/tcpip/transport/icmp/protocol.go | 150 + .../transport/internal/network/endpoint.go | 1065 + .../internal/network/endpoint_state.go | 62 + .../internal/network/network_state_autogen.go | 121 + pkg/tcpip/transport/internal/noop/endpoint.go | 177 + .../internal/noop/noop_state_autogen.go | 41 + pkg/tcpip/transport/packet/endpoint.go | 623 + pkg/tcpip/transport/packet/endpoint_mutex.go | 96 + .../transport/packet/endpoint_rcv_mutex.go | 64 + pkg/tcpip/transport/packet/endpoint_state.go | 74 + .../transport/packet/last_error_mutex.go | 64 + pkg/tcpip/transport/packet/packet_list.go | 239 + .../transport/packet/packet_mmap_mutex.go | 96 + .../transport/packet/packet_state_autogen.go | 181 + pkg/tcpip/transport/raw/endpoint.go | 783 + pkg/tcpip/transport/raw/endpoint_state.go | 78 + pkg/tcpip/transport/raw/protocol.go | 55 + pkg/tcpip/transport/raw/raw_packet_list.go | 239 + pkg/tcpip/transport/raw/raw_state_autogen.go | 222 + pkg/tcpip/transport/tcp/accept.go | 724 + pkg/tcpip/transport/tcp/accept_mutex.go | 64 + pkg/tcpip/transport/tcp/connect.go | 1532 ++ pkg/tcpip/transport/tcp/connect_unsafe.go | 30 + pkg/tcpip/transport/tcp/cubic.go | 318 + pkg/tcpip/transport/tcp/dispatcher.go | 534 + pkg/tcpip/transport/tcp/dispatcher_mutex.go | 64 + pkg/tcpip/transport/tcp/endpoint.go | 3367 +++ pkg/tcpip/transport/tcp/endpoint_state.go | 347 + pkg/tcpip/transport/tcp/ep_queue_mutex.go | 64 + pkg/tcpip/transport/tcp/forwarder.go | 231 + pkg/tcpip/transport/tcp/forwarder_mutex.go | 64 + .../transport/tcp/forwarder_request_mutex.go | 64 + pkg/tcpip/transport/tcp/hasher_mutex.go | 64 + pkg/tcpip/transport/tcp/keepalive_mutex.go | 64 + pkg/tcpip/transport/tcp/last_error_mutex.go | 64 + .../transport/tcp/pending_processing_mutex.go | 64 + pkg/tcpip/transport/tcp/protocol.go | 606 + pkg/tcpip/transport/tcp/protocol_mutex.go | 96 + pkg/tcpip/transport/tcp/rack.go | 459 + pkg/tcpip/transport/tcp/rcv.go | 616 + pkg/tcpip/transport/tcp/rcv_queue_mutex.go | 64 + pkg/tcpip/transport/tcp/reno.go | 118 + pkg/tcpip/transport/tcp/reno_recovery.go | 68 + pkg/tcpip/transport/tcp/rtt_mutex.go | 64 + pkg/tcpip/transport/tcp/sack.go | 105 + pkg/tcpip/transport/tcp/sack_recovery.go | 122 + pkg/tcpip/transport/tcp/sack_scoreboard.go | 306 + pkg/tcpip/transport/tcp/segment.go | 251 + pkg/tcpip/transport/tcp/segment_heap.go | 51 + pkg/tcpip/transport/tcp/segment_queue.go | 99 + .../transport/tcp/segment_queue_mutex.go | 64 + pkg/tcpip/transport/tcp/segment_state.go | 35 + pkg/tcpip/transport/tcp/segment_unsafe.go | 23 + pkg/tcpip/transport/tcp/snd.go | 1905 ++ pkg/tcpip/transport/tcp/snd_queue_mutex.go | 64 + pkg/tcpip/transport/tcp/state.go | 480 + pkg/tcpip/transport/tcp/tcp_endpoint_list.go | 239 + pkg/tcpip/transport/tcp/tcp_segment_list.go | 239 + pkg/tcpip/transport/tcp/tcp_segment_refs.go | 141 + pkg/tcpip/transport/tcp/tcp_state_autogen.go | 1935 ++ .../transport/tcp/tcp_unsafe_state_autogen.go | 3 + pkg/tcpip/transport/tcp/timer.go | 160 + .../transport/tcpconntrack/tcp_conntrack.go | 417 + .../tcpconntrack_state_autogen.go | 91 + pkg/tcpip/transport/transport.go | 16 + .../transport/transport_state_autogen.go | 3 + pkg/tcpip/transport/udp/endpoint.go | 1098 + pkg/tcpip/transport/udp/endpoint_state.go | 96 + pkg/tcpip/transport/udp/forwarder.go | 112 + pkg/tcpip/transport/udp/protocol.go | 138 + pkg/tcpip/transport/udp/udp_packet_list.go | 239 + pkg/tcpip/transport/udp/udp_state_autogen.go | 225 + pkg/timing/timing.go | 901 + pkg/timing/timing_state_autogen.go | 3 + pkg/unet/unet.go | 547 + pkg/unet/unet_state_autogen.go | 3 + pkg/unet/unet_unsafe.go | 286 + pkg/unet/unet_unsafe_state_autogen.go | 3 + pkg/waiter/waiter.go | 280 + pkg/waiter/waiter_list.go | 239 + pkg/waiter/waiter_state_autogen.go | 128 + pkg/xdp/completionqueue.go | 119 + pkg/xdp/fillqueue.go | 121 + pkg/xdp/rxqueue.go | 105 + pkg/xdp/txqueue.go | 116 + pkg/xdp/umem.go | 107 + pkg/xdp/xdp.go | 323 + pkg/xdp/xdp_state_autogen.go | 11 + pkg/xdp/xdp_unsafe.go | 123 + pkg/xdp/xdp_unsafe_state_autogen.go | 3 + reformat.sh | 9 + remove-unused.sh | 36 + rename-module.sh | 37 + 712 files changed, 185689 insertions(+) create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 LICENSE create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/abi/abi.go create mode 100644 pkg/abi/abi_linux.go create mode 100644 pkg/abi/abi_linux_state_autogen.go create mode 100644 pkg/abi/abi_state_autogen.go create mode 100644 pkg/abi/flag.go create mode 100644 pkg/abi/gasket/gasket.go create mode 100644 pkg/abi/gasket/gasket_abi_autogen_unsafe.go create mode 100644 pkg/abi/gasket/gasket_state_autogen.go create mode 100644 pkg/abi/linux/aio.go create mode 100644 pkg/abi/linux/arch_amd64.go create mode 100644 pkg/abi/linux/audit.go create mode 100644 pkg/abi/linux/bpf.go create mode 100644 pkg/abi/linux/capability.go create mode 100644 pkg/abi/linux/clone.go create mode 100644 pkg/abi/linux/context.go create mode 100644 pkg/abi/linux/dev.go create mode 100644 pkg/abi/linux/elf.go create mode 100644 pkg/abi/linux/epoll.go create mode 100644 pkg/abi/linux/epoll_amd64.go create mode 100644 pkg/abi/linux/epoll_arm64.go create mode 100644 pkg/abi/linux/errno/errno.go create mode 100644 pkg/abi/linux/errno/errno_state_autogen.go create mode 100644 pkg/abi/linux/errqueue.go create mode 100644 pkg/abi/linux/eventfd.go create mode 100644 pkg/abi/linux/exec.go create mode 100644 pkg/abi/linux/fadvise.go create mode 100644 pkg/abi/linux/fcntl.go create mode 100644 pkg/abi/linux/file.go create mode 100644 pkg/abi/linux/file_amd64.go create mode 100644 pkg/abi/linux/file_arm64.go create mode 100644 pkg/abi/linux/fs.go create mode 100644 pkg/abi/linux/fuse.go create mode 100644 pkg/abi/linux/futex.go create mode 100644 pkg/abi/linux/inotify.go create mode 100644 pkg/abi/linux/ioctl.go create mode 100644 pkg/abi/linux/ioctl_tun.go create mode 100644 pkg/abi/linux/iouring.go create mode 100644 pkg/abi/linux/ip.go create mode 100644 pkg/abi/linux/ipc.go create mode 100644 pkg/abi/linux/keyctl.go create mode 100644 pkg/abi/linux/limits.go create mode 100644 pkg/abi/linux/linux.go create mode 100644 pkg/abi/linux/linux_abi_autogen_unsafe.go create mode 100644 pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go create mode 100644 pkg/abi/linux/linux_amd64_state_autogen.go create mode 100644 pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go create mode 100644 pkg/abi/linux/linux_arm64_state_autogen.go create mode 100644 pkg/abi/linux/linux_state_autogen.go create mode 100644 pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go create mode 100644 pkg/abi/linux/linux_unsafe_state_autogen.go create mode 100644 pkg/abi/linux/membarrier.go create mode 100644 pkg/abi/linux/mm.go create mode 100644 pkg/abi/linux/mm_amd64.go create mode 100644 pkg/abi/linux/mm_arm64.go create mode 100644 pkg/abi/linux/mqueue.go create mode 100644 pkg/abi/linux/msgqueue.go create mode 100644 pkg/abi/linux/netdevice.go create mode 100644 pkg/abi/linux/netfilter.go create mode 100644 pkg/abi/linux/netfilter_arp.go create mode 100644 pkg/abi/linux/netfilter_bridge.go create mode 100644 pkg/abi/linux/netfilter_ipv4.go create mode 100644 pkg/abi/linux/netfilter_ipv6.go create mode 100644 pkg/abi/linux/netlink.go create mode 100644 pkg/abi/linux/netlink_netfilter.go create mode 100644 pkg/abi/linux/netlink_route.go create mode 100644 pkg/abi/linux/nf_tables.go create mode 100644 pkg/abi/linux/poll.go create mode 100644 pkg/abi/linux/prctl.go create mode 100644 pkg/abi/linux/ptrace.go create mode 100644 pkg/abi/linux/ptrace_amd64.go create mode 100644 pkg/abi/linux/ptrace_arm64.go create mode 100644 pkg/abi/linux/rseq.go create mode 100644 pkg/abi/linux/rusage.go create mode 100644 pkg/abi/linux/sched.go create mode 100644 pkg/abi/linux/seccomp.go create mode 100644 pkg/abi/linux/sem.go create mode 100644 pkg/abi/linux/sem_amd64.go create mode 100644 pkg/abi/linux/sem_arm64.go create mode 100644 pkg/abi/linux/shm.go create mode 100644 pkg/abi/linux/signal.go create mode 100644 pkg/abi/linux/signalfd.go create mode 100644 pkg/abi/linux/socket.go create mode 100644 pkg/abi/linux/splice.go create mode 100644 pkg/abi/linux/tcp.go create mode 100644 pkg/abi/linux/time.go create mode 100644 pkg/abi/linux/timer.go create mode 100644 pkg/abi/linux/tty.go create mode 100644 pkg/abi/linux/uio.go create mode 100644 pkg/abi/linux/utsname.go create mode 100644 pkg/abi/linux/vfio.go create mode 100644 pkg/abi/linux/vfio_unsafe.go create mode 100644 pkg/abi/linux/wait.go create mode 100644 pkg/abi/linux/xattr.go create mode 100644 pkg/abi/nvgpu/classes.go create mode 100644 pkg/abi/nvgpu/ctrl.go create mode 100644 pkg/abi/nvgpu/frontend.go create mode 100644 pkg/abi/nvgpu/frontend_unsafe.go create mode 100644 pkg/abi/nvgpu/nvgpu.go create mode 100644 pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go create mode 100644 pkg/abi/nvgpu/nvgpu_state_autogen.go create mode 100644 pkg/abi/nvgpu/nvgpu_unsafe_abi_autogen_unsafe.go create mode 100644 pkg/abi/nvgpu/nvgpu_unsafe_state_autogen.go create mode 100644 pkg/abi/nvgpu/status.go create mode 100644 pkg/abi/nvgpu/uvm.go create mode 100644 pkg/abi/sentry/sentry.go create mode 100644 pkg/abi/sentry/sentry_state_autogen.go create mode 100644 pkg/abi/sentry/syscall.go create mode 100644 pkg/abi/tpu/tpu.go create mode 100644 pkg/abi/tpu/tpu_state_autogen.go create mode 100644 pkg/atomicbitops/32b_32bit.go create mode 100644 pkg/atomicbitops/32b_64bit.go create mode 100644 pkg/atomicbitops/aligned_32bit_unsafe.go create mode 100644 pkg/atomicbitops/aligned_64bit.go create mode 100644 pkg/atomicbitops/atomicbitops.go create mode 100644 pkg/atomicbitops/atomicbitops_32bit_state_autogen.go create mode 100644 pkg/atomicbitops/atomicbitops_32bit_unsafe_state_autogen.go create mode 100644 pkg/atomicbitops/atomicbitops_64bit_state_autogen.go create mode 100644 pkg/atomicbitops/atomicbitops_amd64.s create mode 100644 pkg/atomicbitops/atomicbitops_arm64.go create mode 100644 pkg/atomicbitops/atomicbitops_arm64.s create mode 100644 pkg/atomicbitops/atomicbitops_arm64_state_autogen.go create mode 100644 pkg/atomicbitops/atomicbitops_float64.go create mode 100644 pkg/atomicbitops/atomicbitops_noasm.go create mode 100644 pkg/atomicbitops/atomicbitops_state_autogen.go create mode 100644 pkg/binary/binary.go create mode 100644 pkg/binary/binary_state_autogen.go create mode 100644 pkg/bits/bits.go create mode 100644 pkg/bits/bits32.go create mode 100644 pkg/bits/bits64.go create mode 100644 pkg/bits/bits_state_autogen.go create mode 100644 pkg/bits/uint64_arch.go create mode 100644 pkg/bits/uint64_arch_amd64_asm.s create mode 100644 pkg/bits/uint64_arch_arm64_asm.s create mode 100644 pkg/bits/uint64_arch_generic.go create mode 100644 pkg/buffer/buffer.go create mode 100644 pkg/buffer/buffer_state.go create mode 100644 pkg/buffer/buffer_state_autogen.go create mode 100644 pkg/buffer/buffer_unsafe_state_autogen.go create mode 100644 pkg/buffer/chunk.go create mode 100644 pkg/buffer/chunk_refs.go create mode 100644 pkg/buffer/view.go create mode 100644 pkg/buffer/view_list.go create mode 100644 pkg/buffer/view_unsafe.go create mode 100644 pkg/cleanup/cleanup.go create mode 100644 pkg/cleanup/cleanup_state_autogen.go create mode 100644 pkg/compressio/compressio.go create mode 100644 pkg/compressio/compressio_state_autogen.go create mode 100644 pkg/compressio/nocompressio.go create mode 100644 pkg/context/context.go create mode 100644 pkg/context/context_state_autogen.go create mode 100644 pkg/coretag/coretag.go create mode 100644 pkg/coretag/coretag_state_autogen.go create mode 100644 pkg/coretag/coretag_unsafe.go create mode 100644 pkg/coretag/coretag_unsafe_state_autogen.go create mode 100644 pkg/cpuid/cpuid.go create mode 100644 pkg/cpuid/cpuid_amd64.go create mode 100644 pkg/cpuid/cpuid_amd64_state_autogen.go create mode 100644 pkg/cpuid/cpuid_arm64.go create mode 100644 pkg/cpuid/cpuid_arm64_state_autogen.go create mode 100644 pkg/cpuid/cpuid_state_autogen.go create mode 100644 pkg/cpuid/features_amd64.go create mode 100644 pkg/cpuid/features_arm64.go create mode 100644 pkg/cpuid/hwcap_amd64.go create mode 100644 pkg/cpuid/hwcap_arm64.go create mode 100644 pkg/cpuid/native_amd64.go create mode 100644 pkg/cpuid/native_amd64.s create mode 100644 pkg/cpuid/native_arm64.go create mode 100644 pkg/cpuid/static_amd64.go create mode 100644 pkg/errors/errors.go create mode 100644 pkg/errors/errors_state_autogen.go create mode 100644 pkg/errors/linuxerr/internal.go create mode 100644 pkg/errors/linuxerr/linuxerr.go create mode 100644 pkg/errors/linuxerr/linuxerr_state_autogen.go create mode 100644 pkg/eventfd/eventfd.go create mode 100644 pkg/eventfd/eventfd_state_autogen.go create mode 100644 pkg/eventfd/eventfd_unsafe.go create mode 100644 pkg/eventfd/eventfd_unsafe_state_autogen.go create mode 100644 pkg/fd/fd.go create mode 100644 pkg/fd/fd_state_autogen.go create mode 100644 pkg/fdchannel/fdchannel_unsafe.go create mode 100644 pkg/fdchannel/fdchannel_unsafe_state_autogen.go create mode 100644 pkg/fdnotifier/fdnotifier.go create mode 100644 pkg/fdnotifier/fdnotifier_state_autogen.go create mode 100644 pkg/fdnotifier/fdnotifier_unsafe_state_autogen.go create mode 100644 pkg/fdnotifier/poll_unsafe.go create mode 100644 pkg/flipcall/ctrl_futex.go create mode 100644 pkg/flipcall/flipcall.go create mode 100644 pkg/flipcall/flipcall_linux_state_autogen.go create mode 100644 pkg/flipcall/flipcall_state_autogen.go create mode 100644 pkg/flipcall/flipcall_unsafe.go create mode 100644 pkg/flipcall/flipcall_unsafe_state_autogen.go create mode 100644 pkg/flipcall/futex_linux.go create mode 100644 pkg/flipcall/io.go create mode 100644 pkg/flipcall/packet_window.go create mode 100644 pkg/fspath/builder.go create mode 100644 pkg/fspath/fspath.go create mode 100644 pkg/fspath/fspath_state_autogen.go create mode 100644 pkg/fsutil/fsutil.go create mode 100644 pkg/fsutil/fsutil_amd64_unsafe.go create mode 100644 pkg/fsutil/fsutil_amd64_unsafe_state_autogen.go create mode 100644 pkg/fsutil/fsutil_arm64_unsafe.go create mode 100644 pkg/fsutil/fsutil_arm64_unsafe_state_autogen.go create mode 100644 pkg/fsutil/fsutil_state_autogen.go create mode 100644 pkg/fsutil/fsutil_unsafe.go create mode 100644 pkg/fsutil/fsutil_unsafe_state_autogen.go create mode 100644 pkg/gohacks/linkname_go113_unsafe.go create mode 100644 pkg/gohacks/noescape_unsafe.go create mode 100644 pkg/gohacks/slice_go113_unsafe.go create mode 100644 pkg/gohacks/slice_go120_unsafe.go create mode 100644 pkg/gohacks/string_go113_unsafe.go create mode 100644 pkg/gohacks/string_go120_unsafe.go create mode 100644 pkg/goid/goid.go create mode 100644 pkg/goid/goid_122_amd64.s create mode 100644 pkg/goid/goid_122_arm64.s create mode 100644 pkg/goid/goid_123_amd64.s create mode 100644 pkg/goid/goid_123_arm64.s create mode 100644 pkg/goid/goid_125_amd64.s create mode 100644 pkg/goid/goid_125_arm64.s create mode 100644 pkg/gomaxprocs/gomaxprocs.go create mode 100644 pkg/gomaxprocs/gomaxprocs_mutex.go create mode 100644 pkg/gomaxprocs/gomaxprocs_state_autogen.go create mode 100644 pkg/hostarch/access_type.go create mode 100644 pkg/hostarch/addr.go create mode 100644 pkg/hostarch/addr_range.go create mode 100644 pkg/hostarch/addr_range_seq_unsafe.go create mode 100644 pkg/hostarch/addr_unsafe.go create mode 100644 pkg/hostarch/hostarch.go create mode 100644 pkg/hostarch/hostarch_arm64.go create mode 100644 pkg/hostarch/hostarch_arm64_state_autogen.go create mode 100644 pkg/hostarch/hostarch_state_autogen.go create mode 100644 pkg/hostarch/hostarch_unsafe_state_autogen.go create mode 100644 pkg/hostarch/hostarch_x86.go create mode 100644 pkg/hostarch/hostarch_x86_state_autogen.go create mode 100644 pkg/hostarch/memory_type.go create mode 100644 pkg/hostarch/sizes_util.go create mode 100644 pkg/hostsyscall/hostsyscall.go create mode 100644 pkg/hostsyscall/hostsyscall_amd64.s create mode 100644 pkg/hostsyscall/hostsyscall_arm64.s create mode 100644 pkg/hostsyscall/hostsyscall_state_autogen.go create mode 100644 pkg/hosttid/hosttid.go create mode 100644 pkg/hosttid/hosttid_124_amd64.s create mode 100644 pkg/hosttid/hosttid_124_arm64.s create mode 100644 pkg/hosttid/hosttid_125_amd64.s create mode 100644 pkg/hosttid/hosttid_125_arm64.s create mode 100644 pkg/hosttid/hosttid_state_autogen.go create mode 100644 pkg/ilist/ilist_state_autogen.go create mode 100644 pkg/ilist/interface_list.go create mode 100644 pkg/linewriter/linewriter.go create mode 100644 pkg/log/bug.go create mode 100644 pkg/log/glog.go create mode 100644 pkg/log/json.go create mode 100644 pkg/log/json_k8s.go create mode 100644 pkg/log/log.go create mode 100644 pkg/log/rate_limited.go create mode 100644 pkg/marshal/marshal.go create mode 100644 pkg/marshal/marshal_state_autogen.go create mode 100644 pkg/marshal/primitive/primitive.go create mode 100644 pkg/marshal/primitive/primitive_abi_autogen_unsafe.go create mode 100644 pkg/marshal/primitive/primitive_state_autogen.go create mode 100644 pkg/marshal/util.go create mode 100644 pkg/memutil/memfd_linux_unsafe.go create mode 100644 pkg/memutil/memutil_linux_unsafe_state_autogen.go create mode 100644 pkg/memutil/memutil_state_autogen.go create mode 100644 pkg/memutil/memutil_unsafe.go create mode 100644 pkg/memutil/memutil_unsafe_state_autogen.go create mode 100644 pkg/memutil/mmap.go create mode 100644 pkg/pool/pool.go create mode 100644 pkg/pool/pool_state_autogen.go create mode 100644 pkg/rand/rand.go create mode 100644 pkg/rand/rand_linux.go create mode 100644 pkg/rand/rand_linux_state_autogen.go create mode 100644 pkg/rand/rand_state_autogen.go create mode 100644 pkg/rand/rng.go create mode 100644 pkg/rawfile/blockingpoll_amd64.s create mode 100644 pkg/rawfile/blockingpoll_arm64.s create mode 100644 pkg/rawfile/blockingpoll_noyield_unsafe.go create mode 100644 pkg/rawfile/blockingpoll_yield_unsafe.go create mode 100644 pkg/rawfile/rawfile_unsafe.go create mode 100644 pkg/rawfile/rawfile_unsafe_state_autogen.go create mode 100644 pkg/refs/refcounter.go create mode 100644 pkg/refs/refs_map.go create mode 100644 pkg/refs/refs_state_autogen.go create mode 100644 pkg/ringdeque/ringdeque.go create mode 100644 pkg/safecopy/atomic_amd64.s create mode 100644 pkg/safecopy/atomic_arm64.s create mode 100644 pkg/safecopy/memclr_amd64.s create mode 100644 pkg/safecopy/memclr_arm64.s create mode 100644 pkg/safecopy/memcpy_amd64.s create mode 100644 pkg/safecopy/memcpy_arm64.s create mode 100644 pkg/safecopy/safecopy.go create mode 100644 pkg/safecopy/safecopy_amd64_unsafe.go create mode 100644 pkg/safecopy/safecopy_amd64_unsafe_state_autogen.go create mode 100644 pkg/safecopy/safecopy_arm64.go create mode 100644 pkg/safecopy/safecopy_arm64_state_autogen.go create mode 100644 pkg/safecopy/safecopy_state_autogen.go create mode 100644 pkg/safecopy/safecopy_unsafe.go create mode 100644 pkg/safecopy/safecopy_unsafe_state_autogen.go create mode 100644 pkg/safecopy/sighandler_amd64.s create mode 100644 pkg/safecopy/sighandler_arm64.s create mode 100644 pkg/safecopy/xrstor_amd64.s create mode 100644 pkg/secio/full_reader.go create mode 100644 pkg/secio/secio.go create mode 100644 pkg/secio/secio_state_autogen.go create mode 100644 pkg/sighandling/sighandling.go create mode 100644 pkg/sighandling/sighandling_darwin.go create mode 100644 pkg/sighandling/sighandling_linux_unsafe.go create mode 100644 pkg/sighandling/sighandling_linux_unsafe_state_autogen.go create mode 100644 pkg/sighandling/sighandling_state_autogen.go create mode 100644 pkg/sleep/sleep_unsafe.go create mode 100644 pkg/sleep/sleep_unsafe_state_autogen.go create mode 100644 pkg/state/addr_range.go create mode 100644 pkg/state/addr_set.go create mode 100644 pkg/state/decode.go create mode 100644 pkg/state/decode_unsafe.go create mode 100644 pkg/state/deferred_list.go create mode 100644 pkg/state/encode.go create mode 100644 pkg/state/encode_unsafe.go create mode 100644 pkg/state/ods_list.go create mode 100644 pkg/state/pretty/pretty.go create mode 100644 pkg/state/pretty/pretty_state_autogen.go create mode 100644 pkg/state/state.go create mode 100644 pkg/state/state_norace.go create mode 100644 pkg/state/state_race.go create mode 100644 pkg/state/statefile/statefile.go create mode 100644 pkg/state/statefile/statefile_state_autogen.go create mode 100644 pkg/state/stats.go create mode 100644 pkg/state/types.go create mode 100644 pkg/state/wire/wire.go create mode 100644 pkg/sync/aliases.go create mode 100644 pkg/sync/checklocks_off_unsafe.go create mode 100644 pkg/sync/checklocks_on_unsafe.go create mode 100644 pkg/sync/fence.go create mode 100644 pkg/sync/fence_amd64.s create mode 100644 pkg/sync/fence_arm64.s create mode 100644 pkg/sync/gate_unsafe.go create mode 100644 pkg/sync/goyield_go113_unsafe.go create mode 100644 pkg/sync/goyield_unsafe.go create mode 100644 pkg/sync/locking/atomicptrmap_ancestors_unsafe.go create mode 100644 pkg/sync/locking/atomicptrmap_goroutine_unsafe.go create mode 100644 pkg/sync/locking/lockdep.go create mode 100644 pkg/sync/locking/lockdep_norace.go create mode 100644 pkg/sync/locking/locking.go create mode 100644 pkg/sync/mutex_unsafe.go create mode 100644 pkg/sync/nocopy.go create mode 100644 pkg/sync/norace_unsafe.go create mode 100644 pkg/sync/race_amd64.s create mode 100644 pkg/sync/race_arm64.s create mode 100644 pkg/sync/race_unsafe.go create mode 100644 pkg/sync/runtime.go create mode 100644 pkg/sync/runtime_amd64.go create mode 100644 pkg/sync/runtime_constants_go125.go create mode 100644 pkg/sync/runtime_constants_not_go125.go create mode 100644 pkg/sync/runtime_exectracer2.go create mode 100644 pkg/sync/runtime_go121_unsafe.go create mode 100644 pkg/sync/runtime_go124_unsafe.go create mode 100644 pkg/sync/runtime_go126_unsafe.go create mode 100644 pkg/sync/runtime_not_go121_unsafe.go create mode 100644 pkg/sync/runtime_other.go create mode 100644 pkg/sync/runtime_spinning_amd64.s create mode 100644 pkg/sync/runtime_spinning_go125_amd64.s create mode 100644 pkg/sync/runtime_spinning_other.s create mode 100644 pkg/sync/runtime_unsafe.go create mode 100644 pkg/sync/rwmutex_unsafe.go create mode 100644 pkg/sync/seqcount.go create mode 100644 pkg/sync/sync.go create mode 100644 pkg/syncevent/broadcaster.go create mode 100644 pkg/syncevent/receiver.go create mode 100644 pkg/syncevent/source.go create mode 100644 pkg/syncevent/syncevent.go create mode 100644 pkg/syncevent/syncevent_state_autogen.go create mode 100644 pkg/syncevent/syncevent_unsafe_state_autogen.go create mode 100644 pkg/syncevent/waiter_unsafe.go create mode 100644 pkg/syserr/host_darwin.go create mode 100644 pkg/syserr/host_linux.go create mode 100644 pkg/syserr/netstack.go create mode 100644 pkg/syserr/syserr.go create mode 100644 pkg/syserr/syserr_linux_state_autogen.go create mode 100644 pkg/syserr/syserr_state_autogen.go create mode 100644 pkg/tcpip/adapters/gonet/err.go create mode 100644 pkg/tcpip/adapters/gonet/err_linuxwindows.go create mode 100644 pkg/tcpip/adapters/gonet/err_stub.go create mode 100644 pkg/tcpip/adapters/gonet/gonet.go create mode 100644 pkg/tcpip/adapters/gonet/gonet_state_autogen.go create mode 100644 pkg/tcpip/checksum/checksum.go create mode 100644 pkg/tcpip/checksum/checksum_state_autogen.go create mode 100644 pkg/tcpip/checksum/checksum_unsafe.go create mode 100644 pkg/tcpip/checksum/checksum_unsafe_state_autogen.go create mode 100644 pkg/tcpip/errors.go create mode 100644 pkg/tcpip/errors_linux.go create mode 100644 pkg/tcpip/faketime/faketime.go create mode 100644 pkg/tcpip/faketime/faketime_state_autogen.go create mode 100644 pkg/tcpip/hash/jenkins/jenkins.go create mode 100644 pkg/tcpip/hash/jenkins/jenkins_state_autogen.go create mode 100644 pkg/tcpip/header/arp.go create mode 100644 pkg/tcpip/header/checksum.go create mode 100644 pkg/tcpip/header/datagram.go create mode 100644 pkg/tcpip/header/eth.go create mode 100644 pkg/tcpip/header/gue.go create mode 100644 pkg/tcpip/header/header_state_autogen.go create mode 100644 pkg/tcpip/header/icmpv4.go create mode 100644 pkg/tcpip/header/icmpv6.go create mode 100644 pkg/tcpip/header/igmp.go create mode 100644 pkg/tcpip/header/igmpv3.go create mode 100644 pkg/tcpip/header/interfaces.go create mode 100644 pkg/tcpip/header/ipv4.go create mode 100644 pkg/tcpip/header/ipv6.go create mode 100644 pkg/tcpip/header/ipv6_extension_headers.go create mode 100644 pkg/tcpip/header/ipv6_fragment.go create mode 100644 pkg/tcpip/header/mld.go create mode 100644 pkg/tcpip/header/mldv2.go create mode 100644 pkg/tcpip/header/mldv2_igmpv3_common.go create mode 100644 pkg/tcpip/header/ndp_neighbor_advert.go create mode 100644 pkg/tcpip/header/ndp_neighbor_solicit.go create mode 100644 pkg/tcpip/header/ndp_options.go create mode 100644 pkg/tcpip/header/ndp_router_advert.go create mode 100644 pkg/tcpip/header/ndp_router_solicit.go create mode 100644 pkg/tcpip/header/ndpoptionidentifier_string.go create mode 100644 pkg/tcpip/header/parse/parse.go create mode 100644 pkg/tcpip/header/parse/parse_state_autogen.go create mode 100644 pkg/tcpip/header/tcp.go create mode 100644 pkg/tcpip/header/udp.go create mode 100644 pkg/tcpip/header/virtionet.go create mode 100644 pkg/tcpip/internal/tcp/tcp.go create mode 100644 pkg/tcpip/internal/tcp/tcp_state_autogen.go create mode 100644 pkg/tcpip/link/channel/channel.go create mode 100644 pkg/tcpip/link/channel/channel_state_autogen.go create mode 100644 pkg/tcpip/link/channel/endpoint_mutex.go create mode 100644 pkg/tcpip/link/channel/queue_mutex.go create mode 100644 pkg/tcpip/link/ethernet/ethernet.go create mode 100644 pkg/tcpip/link/ethernet/ethernet_state_autogen.go create mode 100644 pkg/tcpip/link/fdbased/endpoint.go create mode 100644 pkg/tcpip/link/fdbased/endpoint_mutex.go create mode 100644 pkg/tcpip/link/fdbased/endpoint_unsafe.go create mode 100644 pkg/tcpip/link/fdbased/fdbased_nonlinux_state_autogen.go create mode 100644 pkg/tcpip/link/fdbased/fdbased_state_autogen.go create mode 100644 pkg/tcpip/link/fdbased/fdbased_unsafe_state_autogen.go create mode 100644 pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go create mode 100644 pkg/tcpip/link/fdbased/mmap.go create mode 100644 pkg/tcpip/link/fdbased/mmap_nonlinux.go create mode 100644 pkg/tcpip/link/fdbased/mmap_unsafe.go create mode 100644 pkg/tcpip/link/fdbased/packet_dispatchers.go create mode 100644 pkg/tcpip/link/fdbased/processor_mutex.go create mode 100644 pkg/tcpip/link/fdbased/processors.go create mode 100644 pkg/tcpip/link/fdbased/save_restore.go create mode 100644 pkg/tcpip/link/loopback/endpoint_mutex.go create mode 100644 pkg/tcpip/link/loopback/loopback.go create mode 100644 pkg/tcpip/link/loopback/loopback_state_autogen.go create mode 100644 pkg/tcpip/link/muxed/endpoint_mutex.go create mode 100644 pkg/tcpip/link/muxed/injectable.go create mode 100644 pkg/tcpip/link/muxed/muxed_state_autogen.go create mode 100644 pkg/tcpip/link/nested/nested.go create mode 100644 pkg/tcpip/link/nested/nested_state_autogen.go create mode 100644 pkg/tcpip/link/packetsocket/packetsocket.go create mode 100644 pkg/tcpip/link/packetsocket/packetsocket_state_autogen.go create mode 100644 pkg/tcpip/link/pipe/endpoint_mutex.go create mode 100644 pkg/tcpip/link/pipe/pipe.go create mode 100644 pkg/tcpip/link/pipe/pipe_state_autogen.go create mode 100644 pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go create mode 100644 pkg/tcpip/link/qdisc/fifo/fifo.go create mode 100644 pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go create mode 100644 pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go create mode 100644 pkg/tcpip/link/sharedmem/endpoint_mutex.go create mode 100644 pkg/tcpip/link/sharedmem/pipe/pipe.go create mode 100644 pkg/tcpip/link/sharedmem/pipe/pipe_state_autogen.go create mode 100644 pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go create mode 100644 pkg/tcpip/link/sharedmem/pipe/pipe_unsafe_state_autogen.go create mode 100644 pkg/tcpip/link/sharedmem/pipe/rx.go create mode 100644 pkg/tcpip/link/sharedmem/pipe/tx.go create mode 100644 pkg/tcpip/link/sharedmem/queue/queue_state_autogen.go create mode 100644 pkg/tcpip/link/sharedmem/queue/rx.go create mode 100644 pkg/tcpip/link/sharedmem/queue/tx.go create mode 100644 pkg/tcpip/link/sharedmem/queuepair.go create mode 100644 pkg/tcpip/link/sharedmem/rx.go create mode 100644 pkg/tcpip/link/sharedmem/server_endpoint_mutex.go create mode 100644 pkg/tcpip/link/sharedmem/server_rx.go create mode 100644 pkg/tcpip/link/sharedmem/server_tx.go create mode 100644 pkg/tcpip/link/sharedmem/sharedmem.go create mode 100644 pkg/tcpip/link/sharedmem/sharedmem_server.go create mode 100644 pkg/tcpip/link/sharedmem/sharedmem_state_autogen.go create mode 100644 pkg/tcpip/link/sharedmem/sharedmem_unsafe.go create mode 100644 pkg/tcpip/link/sharedmem/sharedmem_unsafe_state_autogen.go create mode 100644 pkg/tcpip/link/sharedmem/tx.go create mode 100644 pkg/tcpip/link/sniffer/pcap.go create mode 100644 pkg/tcpip/link/sniffer/sniffer.go create mode 100644 pkg/tcpip/link/sniffer/sniffer_state_autogen.go create mode 100644 pkg/tcpip/link/stopfd/stopfd.go create mode 100644 pkg/tcpip/link/stopfd/stopfd_state_autogen.go create mode 100644 pkg/tcpip/link/tun/device.go create mode 100644 pkg/tcpip/link/tun/device_mutex.go create mode 100644 pkg/tcpip/link/tun/endpoint_mutex.go create mode 100644 pkg/tcpip/link/tun/protocol.go create mode 100644 pkg/tcpip/link/tun/tun_endpoint_refs.go create mode 100644 pkg/tcpip/link/tun/tun_state_autogen.go create mode 100644 pkg/tcpip/link/tun/tun_unsafe.go create mode 100644 pkg/tcpip/link/tun/tun_unsafe_state_autogen.go create mode 100644 pkg/tcpip/link/veth/endpoint_mutex.go create mode 100644 pkg/tcpip/link/veth/veth.go create mode 100644 pkg/tcpip/link/veth/veth_mutex.go create mode 100644 pkg/tcpip/link/veth/veth_state_autogen.go create mode 100644 pkg/tcpip/link/waitable/endpoint_mutex.go create mode 100644 pkg/tcpip/link/waitable/waitable.go create mode 100644 pkg/tcpip/link/waitable/waitable_state_autogen.go create mode 100644 pkg/tcpip/link/xdp/endpoint.go create mode 100644 pkg/tcpip/link/xdp/endpoint_mutex.go create mode 100644 pkg/tcpip/link/xdp/xdp_state_autogen.go create mode 100644 pkg/tcpip/network/arp/arp.go create mode 100644 pkg/tcpip/network/arp/arp_state_autogen.go create mode 100644 pkg/tcpip/network/arp/stats.go create mode 100644 pkg/tcpip/network/hash/hash.go create mode 100644 pkg/tcpip/network/hash/hash_state_autogen.go create mode 100644 pkg/tcpip/network/internal/fragmentation/fragmentation.go create mode 100644 pkg/tcpip/network/internal/fragmentation/fragmentation_state_autogen.go create mode 100644 pkg/tcpip/network/internal/fragmentation/reassembler.go create mode 100644 pkg/tcpip/network/internal/fragmentation/reassembler_list.go create mode 100644 pkg/tcpip/network/internal/ip/duplicate_address_detection.go create mode 100644 pkg/tcpip/network/internal/ip/errors.go create mode 100644 pkg/tcpip/network/internal/ip/generic_multicast_protocol.go create mode 100644 pkg/tcpip/network/internal/ip/ip_state_autogen.go create mode 100644 pkg/tcpip/network/internal/ip/stats.go create mode 100644 pkg/tcpip/network/internal/multicast/multicast_state_autogen.go create mode 100644 pkg/tcpip/network/internal/multicast/route_table.go create mode 100644 pkg/tcpip/network/ipv4/icmp.go create mode 100644 pkg/tcpip/network/ipv4/igmp.go create mode 100644 pkg/tcpip/network/ipv4/ipv4.go create mode 100644 pkg/tcpip/network/ipv4/ipv4_export.go create mode 100644 pkg/tcpip/network/ipv4/ipv4_state_autogen.go create mode 100644 pkg/tcpip/network/ipv4/stats.go create mode 100644 pkg/tcpip/network/ipv6/dhcpv6configurationfromndpra_string.go create mode 100644 pkg/tcpip/network/ipv6/icmp.go create mode 100644 pkg/tcpip/network/ipv6/ipv6.go create mode 100644 pkg/tcpip/network/ipv6/ipv6_export.go create mode 100644 pkg/tcpip/network/ipv6/ipv6_state_autogen.go create mode 100644 pkg/tcpip/network/ipv6/mld.go create mode 100644 pkg/tcpip/network/ipv6/ndp.go create mode 100644 pkg/tcpip/network/ipv6/stats.go create mode 100644 pkg/tcpip/ports/flags.go create mode 100644 pkg/tcpip/ports/ports.go create mode 100644 pkg/tcpip/ports/ports_state_autogen.go create mode 100644 pkg/tcpip/route_list.go create mode 100644 pkg/tcpip/seqnum/seqnum.go create mode 100644 pkg/tcpip/seqnum/seqnum_state_autogen.go create mode 100644 pkg/tcpip/sock_err_list.go create mode 100644 pkg/tcpip/socketops.go create mode 100644 pkg/tcpip/stack/address_state_mutex.go create mode 100644 pkg/tcpip/stack/address_state_refs.go create mode 100644 pkg/tcpip/stack/addressable_endpoint_state.go create mode 100644 pkg/tcpip/stack/addressable_endpoint_state_mutex.go create mode 100644 pkg/tcpip/stack/bridge.go create mode 100644 pkg/tcpip/stack/bridge_mutex.go create mode 100644 pkg/tcpip/stack/bucket_mutex.go create mode 100644 pkg/tcpip/stack/cleanup_endpoints_mutex.go create mode 100644 pkg/tcpip/stack/conn_mutex.go create mode 100644 pkg/tcpip/stack/conn_track_mutex.go create mode 100644 pkg/tcpip/stack/conntrack.go create mode 100644 pkg/tcpip/stack/endpoints_by_nic_mutex.go create mode 100644 pkg/tcpip/stack/gro/gro.go create mode 100644 pkg/tcpip/stack/gro/gro_packet_list.go create mode 100644 pkg/tcpip/stack/gro/gro_state_autogen.go create mode 100644 pkg/tcpip/stack/headertype_string.go create mode 100644 pkg/tcpip/stack/hook_string.go create mode 100644 pkg/tcpip/stack/icmp_rate_limit.go create mode 100644 pkg/tcpip/stack/iptables.go create mode 100644 pkg/tcpip/stack/iptables_mutex.go create mode 100644 pkg/tcpip/stack/iptables_targets.go create mode 100644 pkg/tcpip/stack/iptables_types.go create mode 100644 pkg/tcpip/stack/multi_port_endpoint_mutex.go create mode 100644 pkg/tcpip/stack/neighbor_cache.go create mode 100644 pkg/tcpip/stack/neighbor_cache_mutex.go create mode 100644 pkg/tcpip/stack/neighbor_entry.go create mode 100644 pkg/tcpip/stack/neighbor_entry_list.go create mode 100644 pkg/tcpip/stack/neighbor_entry_mutex.go create mode 100644 pkg/tcpip/stack/neighborstate_string.go create mode 100644 pkg/tcpip/stack/nftables_types.go create mode 100644 pkg/tcpip/stack/nic.go create mode 100644 pkg/tcpip/stack/nic_mutex.go create mode 100644 pkg/tcpip/stack/nic_stats.go create mode 100644 pkg/tcpip/stack/nud.go create mode 100644 pkg/tcpip/stack/packet_buffer.go create mode 100644 pkg/tcpip/stack/packet_buffer_list.go create mode 100644 pkg/tcpip/stack/packet_buffer_refs.go create mode 100644 pkg/tcpip/stack/packet_buffer_unsafe.go create mode 100644 pkg/tcpip/stack/packet_endpoint_list_mutex.go create mode 100644 pkg/tcpip/stack/packet_eps_mutex.go create mode 100644 pkg/tcpip/stack/packets_pending_link_resolution_mutex.go create mode 100644 pkg/tcpip/stack/pending_packets.go create mode 100644 pkg/tcpip/stack/rand.go create mode 100644 pkg/tcpip/stack/registration.go create mode 100644 pkg/tcpip/stack/route.go create mode 100644 pkg/tcpip/stack/route_export.go create mode 100644 pkg/tcpip/stack/route_mutex.go create mode 100644 pkg/tcpip/stack/route_stack_mutex.go create mode 100644 pkg/tcpip/stack/save_restore.go create mode 100644 pkg/tcpip/stack/stack.go create mode 100644 pkg/tcpip/stack/stack_mutex.go create mode 100644 pkg/tcpip/stack/stack_options.go create mode 100644 pkg/tcpip/stack/stack_state_autogen.go create mode 100644 pkg/tcpip/stack/stack_unsafe_state_autogen.go create mode 100644 pkg/tcpip/stack/state_conn_mutex.go create mode 100644 pkg/tcpip/stack/transport_demuxer.go create mode 100644 pkg/tcpip/stack/transport_endpoints_mutex.go create mode 100644 pkg/tcpip/stack/tuple_list.go create mode 100644 pkg/tcpip/stdclock.go create mode 100644 pkg/tcpip/stdclock_state.go create mode 100644 pkg/tcpip/tcpip.go create mode 100644 pkg/tcpip/tcpip_linux_state_autogen.go create mode 100644 pkg/tcpip/tcpip_state.go create mode 100644 pkg/tcpip/tcpip_state_autogen.go create mode 100644 pkg/tcpip/timer.go create mode 100644 pkg/tcpip/transport/datagram.go create mode 100644 pkg/tcpip/transport/icmp/endpoint.go create mode 100644 pkg/tcpip/transport/icmp/endpoint_state.go create mode 100644 pkg/tcpip/transport/icmp/icmp_packet_list.go create mode 100644 pkg/tcpip/transport/icmp/icmp_state_autogen.go create mode 100644 pkg/tcpip/transport/icmp/protocol.go create mode 100644 pkg/tcpip/transport/internal/network/endpoint.go create mode 100644 pkg/tcpip/transport/internal/network/endpoint_state.go create mode 100644 pkg/tcpip/transport/internal/network/network_state_autogen.go create mode 100644 pkg/tcpip/transport/internal/noop/endpoint.go create mode 100644 pkg/tcpip/transport/internal/noop/noop_state_autogen.go create mode 100644 pkg/tcpip/transport/packet/endpoint.go create mode 100644 pkg/tcpip/transport/packet/endpoint_mutex.go create mode 100644 pkg/tcpip/transport/packet/endpoint_rcv_mutex.go create mode 100644 pkg/tcpip/transport/packet/endpoint_state.go create mode 100644 pkg/tcpip/transport/packet/last_error_mutex.go create mode 100644 pkg/tcpip/transport/packet/packet_list.go create mode 100644 pkg/tcpip/transport/packet/packet_mmap_mutex.go create mode 100644 pkg/tcpip/transport/packet/packet_state_autogen.go create mode 100644 pkg/tcpip/transport/raw/endpoint.go create mode 100644 pkg/tcpip/transport/raw/endpoint_state.go create mode 100644 pkg/tcpip/transport/raw/protocol.go create mode 100644 pkg/tcpip/transport/raw/raw_packet_list.go create mode 100644 pkg/tcpip/transport/raw/raw_state_autogen.go create mode 100644 pkg/tcpip/transport/tcp/accept.go create mode 100644 pkg/tcpip/transport/tcp/accept_mutex.go create mode 100644 pkg/tcpip/transport/tcp/connect.go create mode 100644 pkg/tcpip/transport/tcp/connect_unsafe.go create mode 100644 pkg/tcpip/transport/tcp/cubic.go create mode 100644 pkg/tcpip/transport/tcp/dispatcher.go create mode 100644 pkg/tcpip/transport/tcp/dispatcher_mutex.go create mode 100644 pkg/tcpip/transport/tcp/endpoint.go create mode 100644 pkg/tcpip/transport/tcp/endpoint_state.go create mode 100644 pkg/tcpip/transport/tcp/ep_queue_mutex.go create mode 100644 pkg/tcpip/transport/tcp/forwarder.go create mode 100644 pkg/tcpip/transport/tcp/forwarder_mutex.go create mode 100644 pkg/tcpip/transport/tcp/forwarder_request_mutex.go create mode 100644 pkg/tcpip/transport/tcp/hasher_mutex.go create mode 100644 pkg/tcpip/transport/tcp/keepalive_mutex.go create mode 100644 pkg/tcpip/transport/tcp/last_error_mutex.go create mode 100644 pkg/tcpip/transport/tcp/pending_processing_mutex.go create mode 100644 pkg/tcpip/transport/tcp/protocol.go create mode 100644 pkg/tcpip/transport/tcp/protocol_mutex.go create mode 100644 pkg/tcpip/transport/tcp/rack.go create mode 100644 pkg/tcpip/transport/tcp/rcv.go create mode 100644 pkg/tcpip/transport/tcp/rcv_queue_mutex.go create mode 100644 pkg/tcpip/transport/tcp/reno.go create mode 100644 pkg/tcpip/transport/tcp/reno_recovery.go create mode 100644 pkg/tcpip/transport/tcp/rtt_mutex.go create mode 100644 pkg/tcpip/transport/tcp/sack.go create mode 100644 pkg/tcpip/transport/tcp/sack_recovery.go create mode 100644 pkg/tcpip/transport/tcp/sack_scoreboard.go create mode 100644 pkg/tcpip/transport/tcp/segment.go create mode 100644 pkg/tcpip/transport/tcp/segment_heap.go create mode 100644 pkg/tcpip/transport/tcp/segment_queue.go create mode 100644 pkg/tcpip/transport/tcp/segment_queue_mutex.go create mode 100644 pkg/tcpip/transport/tcp/segment_state.go create mode 100644 pkg/tcpip/transport/tcp/segment_unsafe.go create mode 100644 pkg/tcpip/transport/tcp/snd.go create mode 100644 pkg/tcpip/transport/tcp/snd_queue_mutex.go create mode 100644 pkg/tcpip/transport/tcp/state.go create mode 100644 pkg/tcpip/transport/tcp/tcp_endpoint_list.go create mode 100644 pkg/tcpip/transport/tcp/tcp_segment_list.go create mode 100644 pkg/tcpip/transport/tcp/tcp_segment_refs.go create mode 100644 pkg/tcpip/transport/tcp/tcp_state_autogen.go create mode 100644 pkg/tcpip/transport/tcp/tcp_unsafe_state_autogen.go create mode 100644 pkg/tcpip/transport/tcp/timer.go create mode 100644 pkg/tcpip/transport/tcpconntrack/tcp_conntrack.go create mode 100644 pkg/tcpip/transport/tcpconntrack/tcpconntrack_state_autogen.go create mode 100644 pkg/tcpip/transport/transport.go create mode 100644 pkg/tcpip/transport/transport_state_autogen.go create mode 100644 pkg/tcpip/transport/udp/endpoint.go create mode 100644 pkg/tcpip/transport/udp/endpoint_state.go create mode 100644 pkg/tcpip/transport/udp/forwarder.go create mode 100644 pkg/tcpip/transport/udp/protocol.go create mode 100644 pkg/tcpip/transport/udp/udp_packet_list.go create mode 100644 pkg/tcpip/transport/udp/udp_state_autogen.go create mode 100644 pkg/timing/timing.go create mode 100644 pkg/timing/timing_state_autogen.go create mode 100644 pkg/unet/unet.go create mode 100644 pkg/unet/unet_state_autogen.go create mode 100644 pkg/unet/unet_unsafe.go create mode 100644 pkg/unet/unet_unsafe_state_autogen.go create mode 100644 pkg/waiter/waiter.go create mode 100644 pkg/waiter/waiter_list.go create mode 100644 pkg/waiter/waiter_state_autogen.go create mode 100644 pkg/xdp/completionqueue.go create mode 100644 pkg/xdp/fillqueue.go create mode 100644 pkg/xdp/rxqueue.go create mode 100644 pkg/xdp/txqueue.go create mode 100644 pkg/xdp/umem.go create mode 100644 pkg/xdp/xdp.go create mode 100644 pkg/xdp/xdp_state_autogen.go create mode 100644 pkg/xdp/xdp_unsafe.go create mode 100644 pkg/xdp/xdp_unsafe_state_autogen.go create mode 100644 reformat.sh create mode 100644 remove-unused.sh create mode 100644 rename-module.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ec29afb --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.idea/ +.DS_Store diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..01ba465 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,8 @@ +# This is the list of gVisor authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. +# +# Please send a patch if you would like to be included in this list. +Google LLC diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f7a006d --- /dev/null +++ b/LICENSE @@ -0,0 +1,254 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------ + +Some files carry the following license, noted at the top of each file: + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +------------------ + +Some files carry the "BSD" license, noted at the top of each file: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ab5073f --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# gVisor + +This branch is a synthetic branch, containing only Go sources, that is +compatible with standard Go tools. See the master branch for authoritative +sources and tests. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7ba1162 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/sagernet/gvisor + +go 1.24.1 + +require ( + github.com/google/btree v1.1.2 + golang.org/x/sys v0.26.0 + golang.org/x/time v0.7.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7523721 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= diff --git a/pkg/abi/abi.go b/pkg/abi/abi.go new file mode 100644 index 0000000..465a7d3 --- /dev/null +++ b/pkg/abi/abi.go @@ -0,0 +1,44 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package abi describes the interface between a kernel and userspace. +package abi + +import ( + "fmt" +) + +// OS describes the target operating system for an ABI. +// +// Note that OS is architecture-independent. The details of the OS ABI will +// vary between architectures. +type OS int + +const ( + // Linux is the Linux ABI. + Linux OS = iota +) + +// String implements fmt.Stringer. +func (o OS) String() string { + switch o { + case Linux: + return "linux" + default: + return fmt.Sprintf("OS(%d)", o) + } +} + +// ABI is an interface that defines OS-specific interactions. +type ABI interface{} diff --git a/pkg/abi/abi_linux.go b/pkg/abi/abi_linux.go new file mode 100644 index 0000000..008bbca --- /dev/null +++ b/pkg/abi/abi_linux.go @@ -0,0 +1,21 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package abi + +// Host specifies the host ABI. +const Host = Linux diff --git a/pkg/abi/abi_linux_state_autogen.go b/pkg/abi/abi_linux_state_autogen.go new file mode 100644 index 0000000..8a13904 --- /dev/null +++ b/pkg/abi/abi_linux_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package abi diff --git a/pkg/abi/abi_state_autogen.go b/pkg/abi/abi_state_autogen.go new file mode 100644 index 0000000..d54002c --- /dev/null +++ b/pkg/abi/abi_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package abi diff --git a/pkg/abi/flag.go b/pkg/abi/flag.go new file mode 100644 index 0000000..dcdd66d --- /dev/null +++ b/pkg/abi/flag.go @@ -0,0 +1,85 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abi + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// A FlagSet is a slice of bit-flags and their name. +type FlagSet []struct { + Flag uint64 + Name string +} + +// Parse returns a pretty version of val, using the flag names for known flags. +// Unknown flags remain numeric. +func (s FlagSet) Parse(val uint64) string { + var flags []string + + for _, f := range s { + if val&f.Flag == f.Flag { + flags = append(flags, f.Name) + val &^= f.Flag + } + } + + if val != 0 { + flags = append(flags, "0x"+strconv.FormatUint(val, 16)) + } + + if len(flags) == 0 { + // Prefer 0 to an empty string. + return "0x0" + } + + return strings.Join(flags, "|") +} + +// ValueSet is a map of syscall values to their name. Parse will use the name +// or the value if unknown. +type ValueSet map[uint64]string + +// Parse returns the name of the value associated with `val`. Unknown values +// are converted to hex. +func (s ValueSet) Parse(val uint64) string { + if v, ok := s[val]; ok { + return v + } + return fmt.Sprintf("%#x", val) +} + +// ParseDecimal returns the name of the value associated with `val`. Unknown +// values are converted to decimal. +func (s ValueSet) ParseDecimal(val uint64) string { + if v, ok := s[val]; ok { + return v + } + return fmt.Sprintf("%d", val) +} + +// ParseName returns the flag value associated with 'name'. Returns false +// if no value is found. +func (s ValueSet) ParseName(name string) (uint64, bool) { + for k, v := range s { + if v == name { + return k, true + } + } + return math.MaxUint64, false +} diff --git a/pkg/abi/gasket/gasket.go b/pkg/abi/gasket/gasket.go new file mode 100644 index 0000000..c3829b3 --- /dev/null +++ b/pkg/abi/gasket/gasket.go @@ -0,0 +1,127 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gasket describes the userspace interface for Gasket devices. +package gasket + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/abi/linux" +) + +// Ioctl represents a gasket ioctl command. +type Ioctl uint32 + +// From https://github.com/tensorflow/tpu/blob/master/tools/driver/include/linux/google/gasket.h +var ( + GASKET_IOCTL_BASE = uint32(0xDC) + GASKET_IOCTL_RESET = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 0, SizeOfUnsignedLong)) + GASKET_IOCTL_SET_EVENTFD = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 1, SizeofGasketInterruptEventFd)) + GASKET_IOCTL_CLEAR_EVENTFD = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 2, SizeOfUnsignedLong)) + GASKET_IOCTL_NUMBER_PAGE_TABLES = Ioctl(linux.IOR(GASKET_IOCTL_BASE, 4, SizeOfUnsignedLong)) + GASKET_IOCTL_PAGE_TABLE_SIZE = Ioctl(linux.IOWR(GASKET_IOCTL_BASE, 5, SizeofGasketPageTableIoctl)) + GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE = Ioctl(linux.IOWR(GASKET_IOCTL_BASE, 6, SizeofGasketPageTableIoctl)) + GASKET_IOCTL_PARTITION_PAGE_TABLE = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 7, SizeofGasketPageTableIoctl)) + GASKET_IOCTL_MAP_BUFFER = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 8, SizeofGasketPageTableIoctl)) + GASKET_IOCTL_UNMAP_BUFFER = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 9, SizeofGasketPageTableIoctl)) + GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS = Ioctl(linux.IO(GASKET_IOCTL_BASE, 10)) + GASKET_IOCTL_REGISTER_INTERRUPT = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 11, SizeofGasketInterruptMapping)) + GASKET_IOCTL_UNREGISTER_INTERRUPT = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 12, SizeOfUnsignedLong)) + GASKET_IOCTL_MAP_DMA_BUF = Ioctl(linux.IOW(GASKET_IOCTL_BASE, 13, SizeofGasketPageTableDmaBufIoctl)) +) + +func (i Ioctl) String() string { + switch i { + case GASKET_IOCTL_RESET: + return "GASKET_IOCTL_RESET" + case GASKET_IOCTL_SET_EVENTFD: + return "GASKET_IOCTL_SET_EVENTFD" + case GASKET_IOCTL_CLEAR_EVENTFD: + return "GASKET_IOCTL_CLEAR_EVENTFD" + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + return "GASKET_IOCTL_NUMBER_PAGE_TABLES" + case GASKET_IOCTL_PAGE_TABLE_SIZE: + return "GASKET_IOCTL_PAGE_TABLE_SIZE" + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + return "GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE" + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + return "GASKET_IOCTL_PARTITION_PAGE_TABLE" + case GASKET_IOCTL_MAP_BUFFER: + return "GASKET_IOCTL_MAP_BUFFER" + case GASKET_IOCTL_UNMAP_BUFFER: + return "GASKET_IOCTL_UNMAP_BUFFER" + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + return "GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS" + case GASKET_IOCTL_REGISTER_INTERRUPT: + return "GASKET_IOCTL_REGISTER_INTERRUPT" + case GASKET_IOCTL_UNREGISTER_INTERRUPT: + return "GASKET_IOCTL_UNREGISTER_INTERRUPT" + case GASKET_IOCTL_MAP_DMA_BUF: + return "GASKET_IOCTL_MAP_DMA_BUF" + default: + return fmt.Sprintf("UNKNOWN GASKET COMMAND %d", uint32(i)) + } +} + +// GasketInterruptEventFd is the common structure for ioctls associating an +// eventfd with a device interrupt, when using the Gasket interrupt module. +// +// +marshal +type GasketInterruptEventFd struct { + Interrupt uint64 + EventFD uint64 +} + +// GasketPageTableIoctl is a common structure for ioctls mapping and unmapping +// buffers when using the Gasket page_table module. +// +// +marshal +type GasketPageTableIoctl struct { + PageTableIndex uint64 + Size uint64 + HostAddress uint64 + DeviceAddress uint64 +} + +// GasketInterruptMapping is a structure for ioctls associating an eventfd and +// interrupt controlling bar register with a device interrupt, when using the +// Gasket interrupt module. +// +// +marshal +type GasketInterruptMapping struct { + Interrupt uint64 + EventFD uint64 + BarIndex uint64 + RegOffset uint64 +} + +// GasketPageTableDmaBufIoctl is a structure for dma_buf mapping ioctl +// parameters. +// +// +marshal +type GasketPageTableDmaBufIoctl struct { + PageTableIndex uint64 + DeviceAddress uint64 + DMABufID int32 `marshal:"unaligned"` // Struct ends mid 64bit word. +} + +// Ioctl parameter struct sizes. +var ( + SizeofGasketInterruptEventFd = uint32((*GasketInterruptEventFd)(nil).SizeBytes()) + SizeofGasketPageTableIoctl = uint32((*GasketPageTableIoctl)(nil).SizeBytes()) + SizeofGasketInterruptMapping = uint32((*GasketInterruptMapping)(nil).SizeBytes()) + SizeofGasketPageTableDmaBufIoctl = uint32((*GasketPageTableDmaBufIoctl)(nil).SizeBytes()) + SizeOfUnsignedLong = uint32(8) +) diff --git a/pkg/abi/gasket/gasket_abi_autogen_unsafe.go b/pkg/abi/gasket/gasket_abi_autogen_unsafe.go new file mode 100644 index 0000000..3b6fafb --- /dev/null +++ b/pkg/abi/gasket/gasket_abi_autogen_unsafe.go @@ -0,0 +1,428 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +package gasket + +import ( + "io" + "reflect" + "runtime" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Marshallable types used by this file. +var ( + _ marshal.Marshallable = (*GasketInterruptEventFd)(nil) + _ marshal.Marshallable = (*GasketInterruptMapping)(nil) + _ marshal.Marshallable = (*GasketPageTableDmaBufIoctl)(nil) + _ marshal.Marshallable = (*GasketPageTableIoctl)(nil) +) + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (g *GasketInterruptEventFd) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (g *GasketInterruptEventFd) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.Interrupt)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.EventFD)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (g *GasketInterruptEventFd) UnmarshalBytes(src []byte) []byte { + g.Interrupt = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.EventFD = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (g *GasketInterruptEventFd) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (g *GasketInterruptEventFd) MarshalUnsafe(dst []byte) []byte { + size := g.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(g), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (g *GasketInterruptEventFd) UnmarshalUnsafe(src []byte) []byte { + size := g.SizeBytes() + gohacks.Memmove(unsafe.Pointer(g), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (g *GasketInterruptEventFd) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (g *GasketInterruptEventFd) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyOutN(cc, addr, g.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (g *GasketInterruptEventFd) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (g *GasketInterruptEventFd) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyInN(cc, addr, g.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (g *GasketInterruptEventFd) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (g *GasketInterruptMapping) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (g *GasketInterruptMapping) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.Interrupt)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.EventFD)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.BarIndex)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.RegOffset)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (g *GasketInterruptMapping) UnmarshalBytes(src []byte) []byte { + g.Interrupt = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.EventFD = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.BarIndex = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.RegOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (g *GasketInterruptMapping) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (g *GasketInterruptMapping) MarshalUnsafe(dst []byte) []byte { + size := g.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(g), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (g *GasketInterruptMapping) UnmarshalUnsafe(src []byte) []byte { + size := g.SizeBytes() + gohacks.Memmove(unsafe.Pointer(g), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (g *GasketInterruptMapping) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (g *GasketInterruptMapping) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyOutN(cc, addr, g.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (g *GasketInterruptMapping) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (g *GasketInterruptMapping) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyInN(cc, addr, g.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (g *GasketInterruptMapping) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (g *GasketPageTableDmaBufIoctl) SizeBytes() int { + return 20 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (g *GasketPageTableDmaBufIoctl) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.PageTableIndex)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.DeviceAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(g.DMABufID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (g *GasketPageTableDmaBufIoctl) UnmarshalBytes(src []byte) []byte { + g.PageTableIndex = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.DeviceAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.DMABufID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (g *GasketPageTableDmaBufIoctl) Packed() bool { + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (g *GasketPageTableDmaBufIoctl) MarshalUnsafe(dst []byte) []byte { + // Type GasketPageTableDmaBufIoctl doesn't have a packed layout in memory, fallback to MarshalBytes. + return g.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (g *GasketPageTableDmaBufIoctl) UnmarshalUnsafe(src []byte) []byte { + // Type GasketPageTableDmaBufIoctl doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return g.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (g *GasketPageTableDmaBufIoctl) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type GasketPageTableDmaBufIoctl doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(g.SizeBytes()) // escapes: okay. + g.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (g *GasketPageTableDmaBufIoctl) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyOutN(cc, addr, g.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (g *GasketPageTableDmaBufIoctl) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type GasketPageTableDmaBufIoctl doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(g.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + g.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (g *GasketPageTableDmaBufIoctl) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyInN(cc, addr, g.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (g *GasketPageTableDmaBufIoctl) WriteTo(writer io.Writer) (int64, error) { + // Type GasketPageTableDmaBufIoctl doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, g.SizeBytes()) + g.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (g *GasketPageTableIoctl) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (g *GasketPageTableIoctl) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.PageTableIndex)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.HostAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(g.DeviceAddress)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (g *GasketPageTableIoctl) UnmarshalBytes(src []byte) []byte { + g.PageTableIndex = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.HostAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + g.DeviceAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (g *GasketPageTableIoctl) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (g *GasketPageTableIoctl) MarshalUnsafe(dst []byte) []byte { + size := g.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(g), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (g *GasketPageTableIoctl) UnmarshalUnsafe(src []byte) []byte { + size := g.SizeBytes() + gohacks.Memmove(unsafe.Pointer(g), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (g *GasketPageTableIoctl) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (g *GasketPageTableIoctl) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyOutN(cc, addr, g.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (g *GasketPageTableIoctl) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (g *GasketPageTableIoctl) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return g.CopyInN(cc, addr, g.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (g *GasketPageTableIoctl) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(g))) + hdr.Len = g.SizeBytes() + hdr.Cap = g.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that g + // must live until the use above. + runtime.KeepAlive(g) // escapes: replaced by intrinsic. + return int64(length), err +} diff --git a/pkg/abi/gasket/gasket_state_autogen.go b/pkg/abi/gasket/gasket_state_autogen.go new file mode 100644 index 0000000..4f4681b --- /dev/null +++ b/pkg/abi/gasket/gasket_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package gasket diff --git a/pkg/abi/linux/aio.go b/pkg/abi/linux/aio.go new file mode 100644 index 0000000..044f783 --- /dev/null +++ b/pkg/abi/linux/aio.go @@ -0,0 +1,96 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import "encoding/binary" + +// AIORing is struct aio_ring, from fs/aio.c, without the trailing +// variable-length array. +type AIORing struct { + ID uint32 + Nr uint32 + Head uint32 + Tail uint32 + Magic uint32 + CompatFeatures uint32 + IncompatFeatures uint32 + HeaderLength uint32 +} + +// AIORingSize is sizeof(struct aio_ring). +const AIORingSize = 32 + +// AIO_RING_MAGIC is fs/aio.c:AIO_RING_MAGIC, the expected value of +// AIORing.Magic. +const AIO_RING_MAGIC = 0xa10a10a1 + +// I/O commands. +const ( + IOCB_CMD_PREAD = 0 + IOCB_CMD_PWRITE = 1 + IOCB_CMD_FSYNC = 2 + IOCB_CMD_FDSYNC = 3 + // 4 was the experimental IOCB_CMD_PREADX. + IOCB_CMD_POLL = 5 + IOCB_CMD_NOOP = 6 + IOCB_CMD_PREADV = 7 + IOCB_CMD_PWRITEV = 8 +) + +// I/O flags. +const ( + IOCB_FLAG_RESFD = 1 + IOCB_FLAG_IOPRIO = 2 +) + +// IOCallback describes an I/O request. +// +// The priority field is currently ignored in the implementation below. Also +// note that the IOCB_FLAG_RESFD feature is not supported. +// +// +marshal +type IOCallback struct { + Data uint64 + Key uint32 + _ uint32 + + OpCode uint16 + ReqPrio int16 + FD int32 + + Buf uint64 + Bytes uint64 + Offset int64 + + Reserved2 uint64 + Flags uint32 + + // eventfd to signal if IOCB_FLAG_RESFD is set in flags. + ResFD int32 +} + +// IOEvent describes an I/O result. +// +// +marshal +// +stateify savable +type IOEvent struct { + Data uint64 + Obj uint64 + Result int64 + Result2 int64 +} + +// IOEventSize is the size of an ioEvent encoded. +var IOEventSize = binary.Size(IOEvent{}) diff --git a/pkg/abi/linux/arch_amd64.go b/pkg/abi/linux/arch_amd64.go new file mode 100644 index 0000000..064c0a6 --- /dev/null +++ b/pkg/abi/linux/arch_amd64.go @@ -0,0 +1,24 @@ +// Copyright 2020 The gVisor Authors. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package linux + +// Start and end addresses of the vsyscall page. +const ( + VSyscallStartAddr uint64 = 0xffffffffff600000 + VSyscallEndAddr uint64 = 0xffffffffff601000 +) diff --git a/pkg/abi/linux/audit.go b/pkg/abi/linux/audit.go new file mode 100644 index 0000000..6cca69a --- /dev/null +++ b/pkg/abi/linux/audit.go @@ -0,0 +1,23 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Audit numbers identify different system call APIs, from +const ( + // AUDIT_ARCH_X86_64 identifies AMD64. + AUDIT_ARCH_X86_64 = 0xc000003e + // AUDIT_ARCH_AARCH64 identifies ARM64. + AUDIT_ARCH_AARCH64 = 0xc00000b7 +) diff --git a/pkg/abi/linux/bpf.go b/pkg/abi/linux/bpf.go new file mode 100644 index 0000000..9422fcf --- /dev/null +++ b/pkg/abi/linux/bpf.go @@ -0,0 +1,35 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// BPFInstruction is a raw BPF virtual machine instruction. +// +// +marshal slice:BPFInstructionSlice +// +stateify savable +type BPFInstruction struct { + // OpCode is the operation to execute. + OpCode uint16 + + // JumpIfTrue is the number of instructions to skip if OpCode is a + // conditional instruction and the condition is true. + JumpIfTrue uint8 + + // JumpIfFalse is the number of instructions to skip if OpCode is a + // conditional instruction and the condition is false. + JumpIfFalse uint8 + + // K is a constant parameter. The meaning depends on the value of OpCode. + K uint32 +} diff --git a/pkg/abi/linux/capability.go b/pkg/abi/linux/capability.go new file mode 100644 index 0000000..a98aca8 --- /dev/null +++ b/pkg/abi/linux/capability.go @@ -0,0 +1,337 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "strings" +) + +// A Capability represents the ability to perform a privileged operation. +type Capability int + +// Capabilities defined by Linux. Taken from the kernel's +// include/uapi/linux/capability.h. See capabilities(7) or that file for more +// detailed capability descriptions. +const ( + CAP_CHOWN = Capability(0) + CAP_DAC_OVERRIDE = Capability(1) + CAP_DAC_READ_SEARCH = Capability(2) + CAP_FOWNER = Capability(3) + CAP_FSETID = Capability(4) + CAP_KILL = Capability(5) + CAP_SETGID = Capability(6) + CAP_SETUID = Capability(7) + CAP_SETPCAP = Capability(8) + CAP_LINUX_IMMUTABLE = Capability(9) + CAP_NET_BIND_SERVICE = Capability(10) + CAP_NET_BROADCAST = Capability(11) + CAP_NET_ADMIN = Capability(12) + CAP_NET_RAW = Capability(13) + CAP_IPC_LOCK = Capability(14) + CAP_IPC_OWNER = Capability(15) + CAP_SYS_MODULE = Capability(16) + CAP_SYS_RAWIO = Capability(17) + CAP_SYS_CHROOT = Capability(18) + CAP_SYS_PTRACE = Capability(19) + CAP_SYS_PACCT = Capability(20) + CAP_SYS_ADMIN = Capability(21) + CAP_SYS_BOOT = Capability(22) + CAP_SYS_NICE = Capability(23) + CAP_SYS_RESOURCE = Capability(24) + CAP_SYS_TIME = Capability(25) + CAP_SYS_TTY_CONFIG = Capability(26) + CAP_MKNOD = Capability(27) + CAP_LEASE = Capability(28) + CAP_AUDIT_WRITE = Capability(29) + CAP_AUDIT_CONTROL = Capability(30) + CAP_SETFCAP = Capability(31) + CAP_MAC_OVERRIDE = Capability(32) + CAP_MAC_ADMIN = Capability(33) + CAP_SYSLOG = Capability(34) + CAP_WAKE_ALARM = Capability(35) + CAP_BLOCK_SUSPEND = Capability(36) + CAP_AUDIT_READ = Capability(37) + CAP_PERFMON = Capability(38) + CAP_BPF = Capability(39) + CAP_CHECKPOINT_RESTORE = Capability(40) + + // CAP_LAST_CAP is the highest-numbered capability. + // Search for "CAP_LAST_CAP" to find other places that need to change. + CAP_LAST_CAP = CAP_CHECKPOINT_RESTORE +) + +// Ok returns true if cp is a supported capability. +func (cp Capability) Ok() bool { + return cp >= 0 && cp <= CAP_LAST_CAP +} + +// String returns the capability name. +func (cp Capability) String() string { + switch cp { + case CAP_CHOWN: + return "CAP_CHOWN" + case CAP_DAC_OVERRIDE: + return "CAP_DAC_OVERRIDE" + case CAP_DAC_READ_SEARCH: + return "CAP_DAC_READ_SEARCH" + case CAP_FOWNER: + return "CAP_FOWNER" + case CAP_FSETID: + return "CAP_FSETID" + case CAP_KILL: + return "CAP_KILL" + case CAP_SETGID: + return "CAP_SETGID" + case CAP_SETUID: + return "CAP_SETUID" + case CAP_SETPCAP: + return "CAP_SETPCAP" + case CAP_LINUX_IMMUTABLE: + return "CAP_LINUX_IMMUTABLE" + case CAP_NET_BIND_SERVICE: + return "CAP_NET_BIND_SERVICE" + case CAP_NET_BROADCAST: + return "CAP_NET_BROADCAST" + case CAP_NET_ADMIN: + return "CAP_NET_ADMIN" + case CAP_NET_RAW: + return "CAP_NET_RAW" + case CAP_IPC_LOCK: + return "CAP_IPC_LOCK" + case CAP_IPC_OWNER: + return "CAP_IPC_OWNER" + case CAP_SYS_MODULE: + return "CAP_SYS_MODULE" + case CAP_SYS_RAWIO: + return "CAP_SYS_RAWIO" + case CAP_SYS_CHROOT: + return "CAP_SYS_CHROOT" + case CAP_SYS_PTRACE: + return "CAP_SYS_PTRACE" + case CAP_SYS_PACCT: + return "CAP_SYS_PACCT" + case CAP_SYS_ADMIN: + return "CAP_SYS_ADMIN" + case CAP_SYS_BOOT: + return "CAP_SYS_BOOT" + case CAP_SYS_NICE: + return "CAP_SYS_NICE" + case CAP_SYS_RESOURCE: + return "CAP_SYS_RESOURCE" + case CAP_SYS_TIME: + return "CAP_SYS_TIME" + case CAP_SYS_TTY_CONFIG: + return "CAP_SYS_TTY_CONFIG" + case CAP_MKNOD: + return "CAP_MKNOD" + case CAP_LEASE: + return "CAP_LEASE" + case CAP_AUDIT_WRITE: + return "CAP_AUDIT_WRITE" + case CAP_AUDIT_CONTROL: + return "CAP_AUDIT_CONTROL" + case CAP_SETFCAP: + return "CAP_SETFCAP" + case CAP_MAC_OVERRIDE: + return "CAP_MAC_OVERRIDE" + case CAP_MAC_ADMIN: + return "CAP_MAC_ADMIN" + case CAP_SYSLOG: + return "CAP_SYSLOG" + case CAP_WAKE_ALARM: + return "CAP_WAKE_ALARM" + case CAP_BLOCK_SUSPEND: + return "CAP_BLOCK_SUSPEND" + case CAP_AUDIT_READ: + return "CAP_AUDIT_READ" + default: + return "UNKNOWN" + } +} + +// TrimmedString returns the capability name without the "CAP_" prefix. +func (cp Capability) TrimmedString() string { + const capPrefix = "CAP_" + s := cp.String() + if !strings.HasPrefix(s, capPrefix) { + return s + } + // This could use strings.TrimPrefix, but that function doesn't guarantee + // that it won't allocate a new string, whereas string slicing does. + // In the case of this function, since Capability.String returns a constant + // string, the underlying set of bytes backing that string will never be + // garbage-collected. Therefore, we always want to use a string slice that + // points to this same constant set of bytes, rather than risking + // allocating a new string. + return s[len(capPrefix):] +} + +// CapabilityFromString converts a string to a capability. +// If the capability doesn't exist, its second return value is `false`. +// The capability name is expected to include the "CAP_" prefix. +func CapabilityFromString(capability string) (Capability, bool) { + for cp := Capability(0); cp <= CAP_LAST_CAP; cp++ { + if !cp.Ok() { + continue + } + if cp.String() == capability { + return cp, true + } + } + return -1, false +} + +// AllCapabilities returns a list of all defined capabilities. +func AllCapabilities() []Capability { + allCapapabilities := make([]Capability, 0, CAP_LAST_CAP+1) + for cp := Capability(0); cp <= CAP_LAST_CAP; cp++ { + if !cp.Ok() { + continue + } + allCapapabilities = append(allCapapabilities, cp) + } + return allCapapabilities +} + +// Version numbers used by the capget/capset syscalls, defined in Linux's +// include/uapi/linux/capability.h. +const ( + // LINUX_CAPABILITY_VERSION_1 causes the data pointer to be + // interpreted as a pointer to a single cap_user_data_t. Since capability + // sets are 64 bits and the "capability sets" in cap_user_data_t are 32 + // bits only, this causes the upper 32 bits to be implicitly 0. + LINUX_CAPABILITY_VERSION_1 = 0x19980330 + + // LINUX_CAPABILITY_VERSION_2 and LINUX_CAPABILITY_VERSION_3 cause the + // data pointer to be interpreted as a pointer to an array of 2 + // cap_user_data_t, using the second to store the 32 MSB of each capability + // set. Versions 2 and 3 are identical, but Linux printk's a warning on use + // of version 2 due to a userspace API defect. + LINUX_CAPABILITY_VERSION_2 = 0x20071026 + LINUX_CAPABILITY_VERSION_3 = 0x20080522 + + // HighestCapabilityVersion is the highest supported + // LINUX_CAPABILITY_VERSION_* version. + HighestCapabilityVersion = LINUX_CAPABILITY_VERSION_3 +) + +// Constants that are used by file capability extended attributes, defined +// in Linux's include/uapi/linux/capability.h. +const ( + // VFS_CAP_FLAGS_EFFECTIVE allows the effective capability set to be + // initialized with the permitted file capabilities. + VFS_CAP_FLAGS_EFFECTIVE = 0x000001 + // VFS_CAP_REVISION_2 allows for file capability masks that are 64 + // bits in size, and was necessary as the number of supported + // capabilities grew beyond 32. + VFS_CAP_REVISION_2 = 0x02000000 + // VFS_CAP_REVISION_3 are provided to support namespaced file capabilities. + // As with version 2 file capabilities, version 3 capability + // masks are 64 bits in size. But in addition, the root user + // ID of namespace is encoded in the security.capability + // extended attribute. + VFS_CAP_REVISION_3 = 0x03000000 + VFS_CAP_REVISION_MASK = 0xFF000000 + // XATTR_CAPS_SZ_2 is sizeof(struct vfs_cap_data). + XATTR_CAPS_SZ_2 = 20 + // XATTR_CAPS_SZ_3 is sizeof(struct vfs_ns_cap_data). + XATTR_CAPS_SZ_3 = 24 +) + +// VfsCapData is equivalent to Linux's struct vfs_cap_data. +// +// +marshal +type VfsCapData struct { + MagicEtc uint32 + PermittedLo uint32 + InheritableLo uint32 + PermittedHi uint32 + InheritableHi uint32 +} + +// Permitted returns the permitted capability set. +func (c *VfsCapData) Permitted() uint64 { + return uint64(c.PermittedHi)<<32 | uint64(c.PermittedLo) +} + +// Inheritable returns the inheritable capability set. +func (c *VfsCapData) Inheritable() uint64 { + return uint64(c.InheritableHi)<<32 | uint64(c.InheritableLo) +} + +// IsRevision2 returns true if c is v2. +func (c *VfsCapData) IsRevision2() bool { + return (c.MagicEtc & VFS_CAP_REVISION_MASK) == VFS_CAP_REVISION_2 +} + +// ToString marshals c into bytes and returns it as a string. +func (c *VfsCapData) ToString() string { + buf := make([]byte, c.SizeBytes()) + c.MarshalUnsafe(buf) + return string(buf) +} + +// VfsNsCapData is equivalent to Linux's struct vfs_ns_cap_data. +// +// +marshal +type VfsNsCapData struct { + VfsCapData + RootID uint32 +} + +// ConvertToV3 converts c to v3 file capabilities. +func (c *VfsNsCapData) ConvertToV3(rootid uint32) { + c.RootID = rootid + if c.IsRevision2() { + // Change to v3 while retaining the effective bit. + c.MagicEtc = VFS_CAP_REVISION_3 | c.MagicEtc&VFS_CAP_FLAGS_EFFECTIVE + } +} + +// ConvertToV2 converts c to v2 file capabilities. +func (c *VfsNsCapData) ConvertToV2() { + c.RootID = 0 + if !c.IsRevision2() { + // Change to v2 while retaining the effective bit. + c.MagicEtc = VFS_CAP_REVISION_2 | c.MagicEtc&VFS_CAP_FLAGS_EFFECTIVE + } +} + +// ToString marshals c into bytes and returns it as a string. +func (c *VfsNsCapData) ToString() string { + if c.IsRevision2() { + return c.VfsCapData.ToString() + } + buf := make([]byte, c.SizeBytes()) + c.MarshalUnsafe(buf) + return string(buf) +} + +// CapUserHeader is equivalent to Linux's cap_user_header_t. +// +// +marshal +type CapUserHeader struct { + Version uint32 + Pid int32 +} + +// CapUserData is equivalent to Linux's cap_user_data_t. +// +// +marshal slice:CapUserDataSlice +type CapUserData struct { + Effective uint32 + Permitted uint32 + Inheritable uint32 +} diff --git a/pkg/abi/linux/clone.go b/pkg/abi/linux/clone.go new file mode 100644 index 0000000..ae1276d --- /dev/null +++ b/pkg/abi/linux/clone.go @@ -0,0 +1,71 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Clone constants per clone(2). +const ( + CSIGNAL = 0xff + + CLONE_VM = 0x100 + CLONE_FS = 0x200 + CLONE_FILES = 0x400 + CLONE_SIGHAND = 0x800 + CLONE_PIDFD = 0x1000 + CLONE_PTRACE = 0x2000 + CLONE_VFORK = 0x4000 + CLONE_PARENT = 0x8000 + CLONE_THREAD = 0x10000 + CLONE_NEWNS = 0x20000 + CLONE_SYSVSEM = 0x40000 + CLONE_SETTLS = 0x80000 + CLONE_PARENT_SETTID = 0x100000 + CLONE_CHILD_CLEARTID = 0x200000 + CLONE_DETACHED = 0x400000 + CLONE_UNTRACED = 0x800000 + CLONE_CHILD_SETTID = 0x1000000 + CLONE_NEWCGROUP = 0x2000000 + CLONE_NEWUTS = 0x4000000 + CLONE_NEWIPC = 0x8000000 + CLONE_NEWUSER = 0x10000000 + CLONE_NEWPID = 0x20000000 + CLONE_NEWNET = 0x40000000 + CLONE_IO = 0x80000000 + + // Only passable via clone3(2). + CLONE_CLEAR_SIGHAND = 0x100000000 + CLONE_INTO_CGROUP = 0x200000000 + + // Sizeof first published struct. + CLONE_ARGS_SIZE_VER0 = 64 + // Sizeof third published struct. + CLONE_ARGS_SIZE_VER2 = 88 +) + +// CloneArgs is struct clone_args, from include/uapi/linux/sched.h. +// +// +marshal +type CloneArgs struct { + Flags uint64 + Pidfd uint64 + ChildTID uint64 + ParentTID uint64 + ExitSignal uint64 + Stack uint64 + StackSize uint64 + TLS uint64 + SetTID uint64 + SetTIDSize uint64 + Cgroup uint64 +} diff --git a/pkg/abi/linux/context.go b/pkg/abi/linux/context.go new file mode 100644 index 0000000..3c2b008 --- /dev/null +++ b/pkg/abi/linux/context.go @@ -0,0 +1,36 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "github.com/sagernet/gvisor/pkg/context" +) + +// contextID is the linux package's type for context.Context.Value keys. +type contextID int + +const ( + // CtxSignalNoInfoFunc is a Context.Value key for a function to send signals. + CtxSignalNoInfoFunc contextID = iota +) + +// SignalNoInfoFuncFromContext returns a callback function that can be used to send a +// signal to the given context. +func SignalNoInfoFuncFromContext(ctx context.Context) func(Signal) error { + if f := ctx.Value(CtxSignalNoInfoFunc); f != nil { + return f.(func(Signal) error) + } + return nil +} diff --git a/pkg/abi/linux/dev.go b/pkg/abi/linux/dev.go new file mode 100644 index 0000000..48f25e9 --- /dev/null +++ b/pkg/abi/linux/dev.go @@ -0,0 +1,72 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// MakeDeviceID encodes a major and minor device number into a single device ID. +// +// Format (see linux/kdev_t.h:new_encode_dev): +// +// Bits 7:0 - minor bits 7:0 +// Bits 19:8 - major bits 11:0 +// Bits 31:20 - minor bits 19:8 +func MakeDeviceID(major uint16, minor uint32) uint32 { + return (minor & 0xff) | ((uint32(major) & 0xfff) << 8) | ((minor >> 8) << 20) +} + +// DecodeDeviceID decodes a device ID into major and minor device numbers. +func DecodeDeviceID(rdev uint32) (uint16, uint32) { + major := uint16((rdev >> 8) & 0xfff) + minor := (rdev & 0xff) | ((rdev >> 20) << 8) + return major, minor +} + +// Character device IDs. +// +// See Documentations/devices.txt and uapi/linux/major.h. +const ( + // UNNAMED_MAJOR is the major device number for "unnamed" devices, whose + // minor numbers are dynamically allocated by the kernel. + UNNAMED_MAJOR = 0 + + // MEM_MAJOR is the major device number for "memory" character devices. + MEM_MAJOR = 1 + + // TTYAUX_MAJOR is the major device number for alternate TTY devices. + TTYAUX_MAJOR = 5 + + // MISC_MAJOR is the major device number for non-serial mice, misc feature + // devices. + MISC_MAJOR = 10 + + // UNIX98_PTY_MASTER_MAJOR is the initial major device number for + // Unix98 PTY masters. + UNIX98_PTY_MASTER_MAJOR = 128 + + // UNIX98_PTY_REPLICA_MAJOR is the initial major device number for + // Unix98 PTY replicas. + UNIX98_PTY_REPLICA_MAJOR = 136 +) + +// Minor device numbers for TTYAUX_MAJOR. +const ( + // PTMX_MINOR is the minor device number for /dev/ptmx. + PTMX_MINOR = 2 +) + +// from Linux include/drm/drm_accel.h +const ( + // ACCEL_MAJOR is the major device number for compute accelerator devices. + ACCEL_MAJOR = 121 +) diff --git a/pkg/abi/linux/elf.go b/pkg/abi/linux/elf.go new file mode 100644 index 0000000..c571354 --- /dev/null +++ b/pkg/abi/linux/elf.go @@ -0,0 +1,158 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Linux auxiliary vector entry types. +const ( + // AT_NULL is the end of the auxiliary vector. + AT_NULL = 0 + + // AT_IGNORE should be ignored. + AT_IGNORE = 1 + + // AT_EXECFD is the file descriptor of the program. + AT_EXECFD = 2 + + // AT_PHDR points to the program headers. + AT_PHDR = 3 + + // AT_PHENT is the size of a program header entry. + AT_PHENT = 4 + + // AT_PHNUM is the number of program headers. + AT_PHNUM = 5 + + // AT_PAGESZ is the system page size. + AT_PAGESZ = 6 + + // AT_BASE is the base address of the interpreter. + AT_BASE = 7 + + // AT_FLAGS are flags. + AT_FLAGS = 8 + + // AT_ENTRY is the program entry point. + AT_ENTRY = 9 + + // AT_NOTELF indicates that the program is not an ELF binary. + AT_NOTELF = 10 + + // AT_UID is the real UID. + AT_UID = 11 + + // AT_EUID is the effective UID. + AT_EUID = 12 + + // AT_GID is the real GID. + AT_GID = 13 + + // AT_EGID is the effective GID. + AT_EGID = 14 + + // AT_PLATFORM is a string identifying the CPU. + AT_PLATFORM = 15 + + // AT_HWCAP are arch-dependent CPU capabilities. + AT_HWCAP = 16 + + // AT_CLKTCK is the frequency used by times(2). + AT_CLKTCK = 17 + + // AT_SECURE indicate secure mode. + AT_SECURE = 23 + + // AT_BASE_PLATFORM is a string identifying the "real" platform. It may + // differ from AT_PLATFORM. + AT_BASE_PLATFORM = 24 + + // AT_RANDOM points to 16-bytes of random data. + AT_RANDOM = 25 + + // AT_HWCAP2 is an extension of AT_HWCAP. + AT_HWCAP2 = 26 + + // AT_EXECFN is the path used to execute the program. + AT_EXECFN = 31 + + // AT_SYSINFO_EHDR is the address of the VDSO. + AT_SYSINFO_EHDR = 33 +) + +// ELF ET_CORE and ptrace GETREGSET/SETREGSET register set types. +// +// See include/uapi/linux/elf.h. +const ( + // NT_PRSTATUS is for general purpose register. + NT_PRSTATUS = 0x1 + + // NT_PRFPREG is for float point register. + NT_PRFPREG = 0x2 + + // NT_X86_XSTATE is for x86 extended state using xsave. + NT_X86_XSTATE = 0x202 + + // NT_ARM_TLS is for ARM TLS register. + NT_ARM_TLS = 0x401 +) + +// ElfHeader64 is the ELF64 file header. +// +// +marshal +type ElfHeader64 struct { + Ident [16]byte // File identification. + Type uint16 // File type. + Machine uint16 // Machine architecture. + Version uint32 // ELF format version. + Entry uint64 // Entry point. + Phoff uint64 // Program header file offset. + Shoff uint64 // Section header file offset. + Flags uint32 // Architecture-specific flags. + Ehsize uint16 // Size of ELF header in bytes. + Phentsize uint16 // Size of program header entry. + Phnum uint16 // Number of program header entries. + Shentsize uint16 // Size of section header entry. + Shnum uint16 // Number of section header entries. + Shstrndx uint16 // Section name strings section. +} + +// ElfSection64 is the ELF64 Section header. +// +// +marshal +type ElfSection64 struct { + Name uint32 // Section name (index into the section header string table). + Type uint32 // Section type. + Flags uint64 // Section flags. + Addr uint64 // Address in memory image. + Off uint64 // Offset in file. + Size uint64 // Size in bytes. + Link uint32 // Index of a related section. + Info uint32 // Depends on section type. + Addralign uint64 // Alignment in bytes. + Entsize uint64 // Size of each entry in section. +} + +// ElfProg64 is the ELF64 Program header. +// +// +marshal +type ElfProg64 struct { + Type uint32 // Entry type. + Flags uint32 // Access permission flags. + Off uint64 // File offset of contents. + Vaddr uint64 // Virtual address in memory image. + Paddr uint64 // Physical address (not used). + Filesz uint64 // Size of contents in file. + Memsz uint64 // Size of contents in memory. + Align uint64 // Alignment in memory and file. +} diff --git a/pkg/abi/linux/epoll.go b/pkg/abi/linux/epoll.go new file mode 100644 index 0000000..67706f5 --- /dev/null +++ b/pkg/abi/linux/epoll.go @@ -0,0 +1,58 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Event masks. +const ( + EPOLLIN = 0x1 + EPOLLPRI = 0x2 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDNORM = 0x40 + EPOLLRDBAND = 0x80 + EPOLLWRNORM = 0x100 + EPOLLWRBAND = 0x200 + EPOLLMSG = 0x400 + EPOLLRDHUP = 0x2000 +) + +// Per-file descriptor flags. +const ( + EPOLLEXCLUSIVE = 1 << 28 + EPOLLWAKEUP = 1 << 29 + EPOLLONESHOT = 1 << 30 + EPOLLET = 1 << 31 + + // EP_PRIVATE_BITS is fs/eventpoll.c:EP_PRIVATE_BITS, the set of all bits + // in an epoll event mask that correspond to flags rather than I/O events. + EP_PRIVATE_BITS = EPOLLEXCLUSIVE | EPOLLWAKEUP | EPOLLONESHOT | EPOLLET +) + +// Operation flags. +const ( + EPOLL_CLOEXEC = 0x80000 + EPOLL_NONBLOCK = 0x800 +) + +// Control operations. +const ( + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +// SizeOfEpollEvent is the size of EpollEvent struct. +var SizeOfEpollEvent = (*EpollEvent)(nil).SizeBytes() diff --git a/pkg/abi/linux/epoll_amd64.go b/pkg/abi/linux/epoll_amd64.go new file mode 100644 index 0000000..7d5b9fd --- /dev/null +++ b/pkg/abi/linux/epoll_amd64.go @@ -0,0 +1,30 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package linux + +// EpollEvent is equivalent to struct epoll_event from epoll(2). +// +// +marshal slice:EpollEventSlice +type EpollEvent struct { + Events uint32 + // Linux makes struct epoll_event::data a __u64. We represent it as + // [2]int32 because, on amd64, Linux also makes struct epoll_event + // __attribute__((packed)), such that there is no padding between Events + // and Data. + Data [2]int32 +} diff --git a/pkg/abi/linux/epoll_arm64.go b/pkg/abi/linux/epoll_arm64.go new file mode 100644 index 0000000..5e5960d --- /dev/null +++ b/pkg/abi/linux/epoll_arm64.go @@ -0,0 +1,29 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package linux + +// EpollEvent is equivalent to struct epoll_event from epoll(2). +// +// +marshal slice:EpollEventSlice +type EpollEvent struct { + Events uint32 + // Linux makes struct epoll_event a __u64, necessitating 4 bytes of padding + // here. + _ int32 + Data [2]int32 +} diff --git a/pkg/abi/linux/errno/errno.go b/pkg/abi/linux/errno/errno.go new file mode 100644 index 0000000..b5e5df2 --- /dev/null +++ b/pkg/abi/linux/errno/errno.go @@ -0,0 +1,187 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errno holds errno codes for abi/linux. +package errno + +// Errno represents a Linux errno value. +type Errno uint32 + +// Errno values from include/uapi/asm-generic/errno-base.h. +const ( + NOERRNO = iota + EPERM + ENOENT + ESRCH + EINTR + EIO + ENXIO + E2BIG + ENOEXEC + EBADF + ECHILD // 10 + EAGAIN + ENOMEM + EACCES + EFAULT + ENOTBLK + EBUSY + EEXIST + EXDEV + ENODEV + ENOTDIR // 20 + EISDIR + EINVAL + ENFILE + EMFILE + ENOTTY + ETXTBSY + EFBIG + ENOSPC + ESPIPE + EROFS // 30 + EMLINK + EPIPE + EDOM + ERANGE + // Errno values from include/uapi/asm-generic/errno.h. + EDEADLK + ENAMETOOLONG + ENOLCK + ENOSYS + ENOTEMPTY + ELOOP // 40 + _ // Skip for EWOULDBLOCK = EAGAIN. + ENOMSG // 42 + EIDRM + ECHRNG + EL2NSYNC + EL3HLT + EL3RST + ELNRNG + EUNATCH + ENOCSI + EL2HLT // 50 + EBADE + EBADR + EXFULL + ENOANO + EBADRQC + EBADSLT + _ // Skip for EDEADLOCK = EDEADLK. + EBFONT + ENOSTR // 60 + ENODATA + ETIME + ENOSR + ENONET + ENOPKG + EREMOTE + ENOLINK + EADV + ESRMNT + ECOMM // 70 + EPROTO + EMULTIHOP + EDOTDOT + EBADMSG + EOVERFLOW + ENOTUNIQ + EBADFD + EREMCHG + ELIBACC + ELIBBAD // 80 + ELIBSCN + ELIBMAX + ELIBEXEC + EILSEQ + ERESTART + ESTRPIPE + EUSERS + ENOTSOCK + EDESTADDRREQ + EMSGSIZE // 90 + EPROTOTYPE + ENOPROTOOPT + EPROTONOSUPPORT + ESOCKTNOSUPPORT + EOPNOTSUPP + EPFNOSUPPORT + EAFNOSUPPORT + EADDRINUSE + EADDRNOTAVAIL + ENETDOWN // 100 + ENETUNREACH + ENETRESET + ECONNABORTED + ECONNRESET + ENOBUFS + EISCONN + ENOTCONN + ESHUTDOWN + ETOOMANYREFS + ETIMEDOUT // 110 + ECONNREFUSED + EHOSTDOWN + EHOSTUNREACH + EALREADY + EINPROGRESS + ESTALE + EUCLEAN + ENOTNAM + ENAVAIL + EISNAM // 120 + EREMOTEIO + EDQUOT + ENOMEDIUM + EMEDIUMTYPE + ECANCELED + ENOKEY + EKEYEXPIRED + EKEYREVOKED + EKEYREJECTED + EOWNERDEAD // 130 + ENOTRECOVERABLE + ERFKILL + EHWPOISON +) + +// errnos derived from other errnos. +const ( + EWOULDBLOCK = EAGAIN + EDEADLOCK = EDEADLK +) + +// errnos for internal errors. +const ( + // ERESTARTSYS is returned by an interrupted syscall to indicate that it + // should be converted to EINTR if interrupted by a signal delivered to a + // user handler without SA_RESTART set, and restarted otherwise. + ERESTARTSYS = 512 + + // ERESTARTNOINTR is returned by an interrupted syscall to indicate that it + // should always be restarted. + ERESTARTNOINTR = 513 + + // ERESTARTNOHAND is returned by an interrupted syscall to indicate that it + // should be converted to EINTR if interrupted by a signal delivered to a + // user handler, and restarted otherwise. + ERESTARTNOHAND = 514 + + // ERESTART_RESTARTBLOCK is returned by an interrupted syscall to indicate + // that it should be restarted using a custom function. The interrupted + // syscall must register a custom restart function by calling + // Task.SetRestartSyscallFn. + ERESTART_RESTARTBLOCK = 516 +) diff --git a/pkg/abi/linux/errno/errno_state_autogen.go b/pkg/abi/linux/errno/errno_state_autogen.go new file mode 100644 index 0000000..4c4ae64 --- /dev/null +++ b/pkg/abi/linux/errno/errno_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package errno diff --git a/pkg/abi/linux/errqueue.go b/pkg/abi/linux/errqueue.go new file mode 100644 index 0000000..4f06f8e --- /dev/null +++ b/pkg/abi/linux/errqueue.go @@ -0,0 +1,93 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Socket error origin codes as defined in include/uapi/linux/errqueue.h. +const ( + SO_EE_ORIGIN_NONE = 0 + SO_EE_ORIGIN_LOCAL = 1 + SO_EE_ORIGIN_ICMP = 2 + SO_EE_ORIGIN_ICMP6 = 3 +) + +// SockExtendedErr represents struct sock_extended_err in Linux defined in +// include/uapi/linux/errqueue.h. +// +// +marshal +type SockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +// SockErrCMsg represents the IP*_RECVERR control message. +type SockErrCMsg interface { + marshal.Marshallable + + CMsgLevel() uint32 + CMsgType() uint32 +} + +// SockErrCMsgIPv4 is the IP_RECVERR control message used in +// recvmsg(MSG_ERRQUEUE) by ipv4 sockets. This is equilavent to `struct errhdr` +// defined in net/ipv4/ip_sockglue.c:ip_recv_error(). +// +// +marshal +type SockErrCMsgIPv4 struct { + SockExtendedErr + Offender SockAddrInet +} + +var _ SockErrCMsg = (*SockErrCMsgIPv4)(nil) + +// CMsgLevel implements SockErrCMsg.CMsgLevel. +func (*SockErrCMsgIPv4) CMsgLevel() uint32 { + return SOL_IP +} + +// CMsgType implements SockErrCMsg.CMsgType. +func (*SockErrCMsgIPv4) CMsgType() uint32 { + return IP_RECVERR +} + +// SockErrCMsgIPv6 is the IPV6_RECVERR control message used in +// recvmsg(MSG_ERRQUEUE) by ipv6 sockets. This is equilavent to `struct errhdr` +// defined in net/ipv6/datagram.c:ipv6_recv_error(). +// +// +marshal +type SockErrCMsgIPv6 struct { + SockExtendedErr + Offender SockAddrInet6 +} + +var _ SockErrCMsg = (*SockErrCMsgIPv6)(nil) + +// CMsgLevel implements SockErrCMsg.CMsgLevel. +func (*SockErrCMsgIPv6) CMsgLevel() uint32 { + return SOL_IPV6 +} + +// CMsgType implements SockErrCMsg.CMsgType. +func (*SockErrCMsgIPv6) CMsgType() uint32 { + return IPV6_RECVERR +} diff --git a/pkg/abi/linux/eventfd.go b/pkg/abi/linux/eventfd.go new file mode 100644 index 0000000..9c479fc --- /dev/null +++ b/pkg/abi/linux/eventfd.go @@ -0,0 +1,22 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Constants for eventfd2(2). +const ( + EFD_SEMAPHORE = 0x1 + EFD_CLOEXEC = O_CLOEXEC + EFD_NONBLOCK = O_NONBLOCK +) diff --git a/pkg/abi/linux/exec.go b/pkg/abi/linux/exec.go new file mode 100644 index 0000000..579d46c --- /dev/null +++ b/pkg/abi/linux/exec.go @@ -0,0 +1,18 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// TASK_COMM_LEN is the task command name length. +const TASK_COMM_LEN = 16 diff --git a/pkg/abi/linux/fadvise.go b/pkg/abi/linux/fadvise.go new file mode 100644 index 0000000..97e2e45 --- /dev/null +++ b/pkg/abi/linux/fadvise.go @@ -0,0 +1,25 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Fadvise constants. +const ( + POSIX_FADV_NORMAL = 0 + POSIX_FADV_RANDOM = 1 + POSIX_FADV_SEQUENTIAL = 2 + POSIX_FADV_WILLNEED = 3 + POSIX_FADV_DONTNEED = 4 + POSIX_FADV_NOREUSE = 5 +) diff --git a/pkg/abi/linux/fcntl.go b/pkg/abi/linux/fcntl.go new file mode 100644 index 0000000..4853024 --- /dev/null +++ b/pkg/abi/linux/fcntl.go @@ -0,0 +1,79 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Commands from linux/fcntl.h. +const ( + F_DUPFD = 0 + F_GETFD = 1 + F_SETFD = 2 + F_GETFL = 3 + F_SETFL = 4 + F_GETLK = 5 + F_SETLK = 6 + F_SETLKW = 7 + F_SETOWN = 8 + F_GETOWN = 9 + F_SETSIG = 10 + F_GETSIG = 11 + F_SETOWN_EX = 15 + F_GETOWN_EX = 16 + F_OFD_GETLK = 36 + F_OFD_SETLK = 37 + F_OFD_SETLKW = 38 + F_DUPFD_CLOEXEC = 1024 + 6 + F_SETPIPE_SZ = 1024 + 7 + F_GETPIPE_SZ = 1024 + 8 +) + +// Commands for F_SETLK. +const ( + F_RDLCK = 0 + F_WRLCK = 1 + F_UNLCK = 2 +) + +// Flags for fcntl. +const ( + FD_CLOEXEC = 0o0000001 +) + +// Flock is the lock structure for F_SETLK. +// +// +marshal +type Flock struct { + Type int16 + Whence int16 + _ [4]byte + Start int64 + Len int64 + PID int32 + _ [4]byte +} + +// Owner types for F_SETOWN_EX and F_GETOWN_EX. +const ( + F_OWNER_TID = 0 + F_OWNER_PID = 1 + F_OWNER_PGRP = 2 +) + +// FOwnerEx is the owner structure for F_SETOWN_EX and F_GETOWN_EX. +// +// +marshal +type FOwnerEx struct { + Type int32 + PID int32 +} diff --git a/pkg/abi/linux/file.go b/pkg/abi/linux/file.go new file mode 100644 index 0000000..01a5b59 --- /dev/null +++ b/pkg/abi/linux/file.go @@ -0,0 +1,437 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "fmt" + "strings" + + "github.com/sagernet/gvisor/pkg/abi" +) + +// Constants for open(2). +const ( + O_ACCMODE = 0o00000003 + O_RDONLY = 0o00000000 + O_WRONLY = 0o00000001 + O_RDWR = 0o00000002 + O_CREAT = 0o00000100 + O_EXCL = 0o00000200 + O_NOCTTY = 0o00000400 + O_TRUNC = 0o00001000 + O_APPEND = 0o00002000 + O_NONBLOCK = 0o00004000 + O_DSYNC = 0o00010000 + O_ASYNC = 0o00020000 + O_NOATIME = 0o01000000 + O_CLOEXEC = 0o02000000 + O_SYNC = 0o04000000 // __O_SYNC in Linux + O_PATH = 0o10000000 + O_TMPFILE = 0o20000000 // __O_TMPFILE in Linux +) + +// Constants for fstatat(2). +const ( + AT_SYMLINK_NOFOLLOW = 0x100 +) + +// Constants for mount(2). +const ( + MS_RDONLY = 0x1 + MS_NOSUID = 0x2 + MS_NODEV = 0x4 + MS_NOEXEC = 0x8 + MS_SYNCHRONOUS = 0x10 + MS_REMOUNT = 0x20 + MS_MANDLOCK = 0x40 + MS_DIRSYNC = 0x80 + MS_NOATIME = 0x400 + MS_NODIRATIME = 0x800 + MS_BIND = 0x1000 + MS_MOVE = 0x2000 + MS_REC = 0x4000 + + MS_POSIXACL = 0x10000 + MS_UNBINDABLE = 0x20000 + MS_PRIVATE = 0x40000 + MS_SLAVE = 0x80000 + MS_SHARED = 0x100000 + MS_RELATIME = 0x200000 + MS_KERNMOUNT = 0x400000 + MS_I_VERSION = 0x800000 + MS_STRICTATIME = 0x1000000 + + MS_MGC_VAL = 0xC0ED0000 + MS_MGC_MSK = 0xffff0000 +) + +// Constants for umount2(2). +const ( + MNT_FORCE = 0x1 + MNT_DETACH = 0x2 + MNT_EXPIRE = 0x4 + UMOUNT_NOFOLLOW = 0x8 +) + +// Constants for unlinkat(2). +const ( + AT_REMOVEDIR = 0x200 +) + +// Constants for linkat(2) and fchownat(2). +const ( + AT_SYMLINK_FOLLOW = 0x400 + AT_EMPTY_PATH = 0x1000 +) + +// Constants for faccessat2(2). +const ( + AT_EACCESS = 0x200 +) + +// Constants for all file-related ...at(2) syscalls. +const ( + AT_FDCWD = -100 +) + +// Special values for the ns field in utimensat(2). +const ( + UTIME_NOW = ((1 << 30) - 1) + UTIME_OMIT = ((1 << 30) - 2) +) + +// MaxSymlinkTraversals is the maximum number of links that will be followed by +// the kernel to resolve a symlink. +const MaxSymlinkTraversals = 40 + +// Constants for flock(2). +const ( + LOCK_SH = 1 // shared lock + LOCK_EX = 2 // exclusive lock + LOCK_NB = 4 // or'd with one of the above to prevent blocking + LOCK_UN = 8 // remove lock +) + +// Values for mode_t. +const ( + S_IFMT = 0o170000 + S_IFSOCK = 0o140000 + S_IFLNK = 0o120000 + S_IFREG = 0o100000 + S_IFBLK = 0o60000 + S_IFDIR = 0o40000 + S_IFCHR = 0o20000 + S_IFIFO = 0o10000 + + FileTypeMask = S_IFMT + ModeSocket = S_IFSOCK + ModeSymlink = S_IFLNK + ModeRegular = S_IFREG + ModeBlockDevice = S_IFBLK + ModeDirectory = S_IFDIR + ModeCharacterDevice = S_IFCHR + ModeNamedPipe = S_IFIFO + + S_ISUID = 0o4000 + S_ISGID = 0o2000 + S_ISVTX = 0o1000 + + ModeSetUID = S_ISUID + ModeSetGID = S_ISGID + ModeSticky = S_ISVTX + + ModeUserAll = 0o700 + ModeUserRead = 0o400 + ModeUserWrite = 0o200 + ModeUserExec = 0o100 + ModeGroupAll = 0o070 + ModeGroupRead = 0o040 + ModeGroupWrite = 0o020 + ModeGroupExec = 0o010 + ModeOtherAll = 0o007 + ModeOtherRead = 0o004 + ModeOtherWrite = 0o002 + ModeOtherExec = 0o001 + PermissionsMask = 0o777 +) + +// Values for linux_dirent64.d_type. +const ( + DT_UNKNOWN = 0 + DT_FIFO = 1 + DT_CHR = 2 + DT_DIR = 4 + DT_BLK = 6 + DT_REG = 8 + DT_LNK = 10 + DT_SOCK = 12 + DT_WHT = 14 +) + +// DirentType are the friendly strings for linux_dirent64.d_type. +var DirentType = abi.ValueSet{ + DT_UNKNOWN: "DT_UNKNOWN", + DT_FIFO: "DT_FIFO", + DT_CHR: "DT_CHR", + DT_DIR: "DT_DIR", + DT_BLK: "DT_BLK", + DT_REG: "DT_REG", + DT_LNK: "DT_LNK", + DT_SOCK: "DT_SOCK", + DT_WHT: "DT_WHT", +} + +// Values for fs on-disk file types. +const ( + FT_UNKNOWN = 0 + FT_REG_FILE = 1 + FT_DIR = 2 + FT_CHRDEV = 3 + FT_BLKDEV = 4 + FT_FIFO = 5 + FT_SOCK = 6 + FT_SYMLINK = 7 + FT_MAX = 8 +) + +// Conversion from fs on-disk file type to dirent type. +var direntTypeByFileType = [FT_MAX]uint8{ + FT_UNKNOWN: DT_UNKNOWN, + FT_REG_FILE: DT_REG, + FT_DIR: DT_DIR, + FT_CHRDEV: DT_CHR, + FT_BLKDEV: DT_BLK, + FT_FIFO: DT_FIFO, + FT_SOCK: DT_SOCK, + FT_SYMLINK: DT_LNK, +} + +// FileTypeToDirentType converts the on-disk file type (FT_*) to the directory +// entry type (DT_*). +func FileTypeToDirentType(filetype uint8) uint8 { + if filetype >= FT_MAX { + return DT_UNKNOWN + } + return direntTypeByFileType[filetype] +} + +// Values for preadv2/pwritev2. +const ( + // NOTE(b/120162627): gVisor does not implement the RWF_HIPRI feature, but + // the flag is accepted as a valid flag argument for preadv2/pwritev2 and + // silently ignored. + RWF_HIPRI = 0x00000001 + RWF_DSYNC = 0x00000002 + RWF_SYNC = 0x00000004 + RWF_VALID = RWF_HIPRI | RWF_DSYNC | RWF_SYNC +) + +// SizeOfStat is the size of a Stat struct. +var SizeOfStat = (*Stat)(nil).SizeBytes() + +// Flags for statx. +const ( + AT_NO_AUTOMOUNT = 0x800 + AT_STATX_SYNC_TYPE = 0x6000 + AT_STATX_SYNC_AS_STAT = 0x0000 + AT_STATX_FORCE_SYNC = 0x2000 + AT_STATX_DONT_SYNC = 0x4000 +) + +// Mask values for statx. +const ( + STATX_TYPE = 0x00000001 + STATX_MODE = 0x00000002 + STATX_NLINK = 0x00000004 + STATX_UID = 0x00000008 + STATX_GID = 0x00000010 + STATX_ATIME = 0x00000020 + STATX_MTIME = 0x00000040 + STATX_CTIME = 0x00000080 + STATX_INO = 0x00000100 + STATX_SIZE = 0x00000200 + STATX_BLOCKS = 0x00000400 + STATX_BASIC_STATS = 0x000007ff + STATX_BTIME = 0x00000800 + STATX_ALL = 0x00000fff + STATX__RESERVED = 0x80000000 +) + +// Bitmasks for Statx.Attributes and Statx.AttributesMask, from +// include/uapi/linux/stat.h. +const ( + STATX_ATTR_COMPRESSED = 0x00000004 + STATX_ATTR_IMMUTABLE = 0x00000010 + STATX_ATTR_APPEND = 0x00000020 + STATX_ATTR_NODUMP = 0x00000040 + STATX_ATTR_ENCRYPTED = 0x00000800 + STATX_ATTR_AUTOMOUNT = 0x00001000 +) + +// Statx represents struct statx. +// +// +marshal boundCheck slice:StatxSlice +type Statx struct { + Mask uint32 + Blksize uint32 + Attributes uint64 + Nlink uint32 + UID uint32 + GID uint32 + Mode uint16 + _ uint16 + Ino uint64 + Size uint64 + Blocks uint64 + AttributesMask uint64 + Atime StatxTimestamp + Btime StatxTimestamp + Ctime StatxTimestamp + Mtime StatxTimestamp + RdevMajor uint32 + RdevMinor uint32 + DevMajor uint32 + DevMinor uint32 +} + +// String implements fmt.Stringer.String. +func (s *Statx) String() string { + return fmt.Sprintf("Statx{Mask: %#x, Mode: %s, UID: %d, GID: %d, Ino: %d, DevMajor: %d, DevMinor: %d, Size: %d, Blocks: %d, Blksize: %d, Nlink: %d, Atime: %s, Btime: %s, Ctime: %s, Mtime: %s, Attributes: %d, AttributesMask: %d, RdevMajor: %d, RdevMinor: %d}", + s.Mask, FileMode(s.Mode), s.UID, s.GID, s.Ino, s.DevMajor, s.DevMinor, s.Size, s.Blocks, s.Blksize, s.Nlink, s.Atime.ToTime(), s.Btime.ToTime(), s.Ctime.ToTime(), s.Mtime.ToTime(), s.Attributes, s.AttributesMask, s.RdevMajor, s.RdevMinor) +} + +// SizeOfStatx is the size of a Statx struct. +var SizeOfStatx = (*Statx)(nil).SizeBytes() + +// FileMode represents a mode_t. +// +// +marshal +type FileMode uint16 + +// Permissions returns just the permission bits. +func (m FileMode) Permissions() FileMode { + return m & PermissionsMask +} + +// FileType returns just the file type bits. +func (m FileMode) FileType() FileMode { + return m & FileTypeMask +} + +// ExtraBits returns everything but the file type and permission bits. +func (m FileMode) ExtraBits() FileMode { + return m &^ (PermissionsMask | FileTypeMask) +} + +// IsDir returns true if file type represents a directory. +func (m FileMode) IsDir() bool { + return m.FileType() == S_IFDIR +} + +// String returns a string representation of m. +func (m FileMode) String() string { + var s []string + if ft := m.FileType(); ft != 0 { + s = append(s, fileType.Parse(uint64(ft))) + } + if eb := m.ExtraBits(); eb != 0 { + s = append(s, modeExtraBits.Parse(uint64(eb))) + } + s = append(s, fmt.Sprintf("0o%o", m.Permissions())) + return strings.Join(s, "|") +} + +// DirentType maps file types to dirent types appropriate for (struct +// dirent)::d_type. +func (m FileMode) DirentType() uint8 { + switch m.FileType() { + case ModeSocket: + return DT_SOCK + case ModeSymlink: + return DT_LNK + case ModeRegular: + return DT_REG + case ModeBlockDevice: + return DT_BLK + case ModeDirectory: + return DT_DIR + case ModeCharacterDevice: + return DT_CHR + case ModeNamedPipe: + return DT_FIFO + default: + return DT_UNKNOWN + } +} + +var modeExtraBits = abi.FlagSet{ + { + Flag: ModeSetUID, + Name: "S_ISUID", + }, + { + Flag: ModeSetGID, + Name: "S_ISGID", + }, + { + Flag: ModeSticky, + Name: "S_ISVTX", + }, +} + +var fileType = abi.ValueSet{ + ModeSocket: "S_IFSOCK", + ModeSymlink: "S_IFLINK", + ModeRegular: "S_IFREG", + ModeBlockDevice: "S_IFBLK", + ModeDirectory: "S_IFDIR", + ModeCharacterDevice: "S_IFCHR", + ModeNamedPipe: "S_IFIFO", +} + +// Constants for memfd_create(2). Source: include/uapi/linux/memfd.h +const ( + MFD_CLOEXEC = 0x0001 + MFD_ALLOW_SEALING = 0x0002 +) + +// Constants related to file seals. Source: include/uapi/{asm-generic,linux}/fcntl.h +const ( + F_LINUX_SPECIFIC_BASE = 1024 + F_ADD_SEALS = F_LINUX_SPECIFIC_BASE + 9 + F_GET_SEALS = F_LINUX_SPECIFIC_BASE + 10 + + F_SEAL_SEAL = 0x0001 // Prevent further seals from being set. + F_SEAL_SHRINK = 0x0002 // Prevent file from shrinking. + F_SEAL_GROW = 0x0004 // Prevent file from growing. + F_SEAL_WRITE = 0x0008 // Prevent writes. +) + +// Constants related to fallocate(2). Source: include/uapi/linux/falloc.h +const ( + FALLOC_FL_KEEP_SIZE = 0x01 + FALLOC_FL_PUNCH_HOLE = 0x02 + FALLOC_FL_NO_HIDE_STALE = 0x04 + FALLOC_FL_COLLAPSE_RANGE = 0x08 + FALLOC_FL_ZERO_RANGE = 0x10 + FALLOC_FL_INSERT_RANGE = 0x20 + FALLOC_FL_UNSHARE_RANGE = 0x40 +) + +// Constants related to close_range(2). Source: /include/uapi/linux/close_range.h +const ( + CLOSE_RANGE_UNSHARE = uint32(1 << 1) + CLOSE_RANGE_CLOEXEC = uint32(1 << 2) +) diff --git a/pkg/abi/linux/file_amd64.go b/pkg/abi/linux/file_amd64.go new file mode 100644 index 0000000..da6bb96 --- /dev/null +++ b/pkg/abi/linux/file_amd64.go @@ -0,0 +1,47 @@ +// Copyright 2018 The gVisor Authors. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package linux + +// Constants for open(2). +const ( + O_DIRECT = 0o00040000 + O_LARGEFILE = 0o00100000 + O_DIRECTORY = 0o00200000 + O_NOFOLLOW = 0o00400000 +) + +// Stat represents struct stat. +// +// +marshal +type Stat struct { + Dev uint64 + Ino uint64 + Nlink uint64 + Mode uint32 + UID uint32 + GID uint32 + _ int32 + Rdev uint64 + Size int64 + Blksize int64 + Blocks int64 + ATime Timespec + MTime Timespec + CTime Timespec + _ [3]int64 +} diff --git a/pkg/abi/linux/file_arm64.go b/pkg/abi/linux/file_arm64.go new file mode 100644 index 0000000..215d96c --- /dev/null +++ b/pkg/abi/linux/file_arm64.go @@ -0,0 +1,48 @@ +// Copyright 2019 The gVisor Authors. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package linux + +// Constants for open(2). +const ( + O_DIRECTORY = 0o00040000 + O_NOFOLLOW = 0o00100000 + O_DIRECT = 0o00200000 + O_LARGEFILE = 0o00400000 +) + +// Stat represents struct stat. +// +// +marshal +type Stat struct { + Dev uint64 + Ino uint64 + Mode uint32 + Nlink uint32 + UID uint32 + GID uint32 + Rdev uint64 + _ uint64 + Size int64 + Blksize int32 + _ int32 + Blocks int64 + ATime Timespec + MTime Timespec + CTime Timespec + _ [2]int32 +} diff --git a/pkg/abi/linux/fs.go b/pkg/abi/linux/fs.go new file mode 100644 index 0000000..e1c4a13 --- /dev/null +++ b/pkg/abi/linux/fs.go @@ -0,0 +1,129 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Filesystem types used in statfs(2). +// +// See linux/magic.h. +const ( + ANON_INODE_FS_MAGIC = 0x09041934 + CGROUP_SUPER_MAGIC = 0x27e0eb + DEVPTS_SUPER_MAGIC = 0x00001cd1 + EXT_SUPER_MAGIC = 0xef53 + FUSE_SUPER_MAGIC = 0x65735546 + MQUEUE_MAGIC = 0x19800202 + NSFS_MAGIC = 0x6e736673 + OVERLAYFS_SUPER_MAGIC = 0x794c7630 + PIPEFS_MAGIC = 0x50495045 + PROC_SUPER_MAGIC = 0x9fa0 + RAMFS_MAGIC = 0x09041934 + SOCKFS_MAGIC = 0x534F434B + SYSFS_MAGIC = 0x62656572 + TMPFS_MAGIC = 0x01021994 + V9FS_MAGIC = 0x01021997 +) + +// Filesystem path limits, from uapi/linux/limits.h. +const ( + NAME_MAX = 255 + PATH_MAX = 4096 +) + +// The bit mask f_flags in struct statfs, from include/linux/statfs.h +const ( + ST_RDONLY = 0x0001 + ST_NOSUID = 0x0002 + ST_NODEV = 0x0004 + ST_NOEXEC = 0x0008 + ST_SYNCHRONOUS = 0x0010 + ST_VALID = 0x0020 + ST_MANDLOCK = 0x0040 + ST_NOATIME = 0x0400 + ST_NODIRATIME = 0x0800 + ST_RELATIME = 0x1000 + ST_NOSYMFOLLOW = 0x2000 +) + +// Statfs is struct statfs, from uapi/asm-generic/statfs.h. +// +// +marshal +type Statfs struct { + // Type is one of the filesystem magic values, defined above. + Type uint64 + + // BlockSize is the optimal transfer block size in bytes. + BlockSize int64 + + // Blocks is the maximum number of data blocks the filesystem may store, in + // units of BlockSize. + Blocks uint64 + + // BlocksFree is the number of free data blocks, in units of BlockSize. + BlocksFree uint64 + + // BlocksAvailable is the number of data blocks free for use by + // unprivileged users, in units of BlockSize. + BlocksAvailable uint64 + + // Files is the number of used file nodes on the filesystem. + Files uint64 + + // FileFress is the number of free file nodes on the filesystem. + FilesFree uint64 + + // FSID is the filesystem ID. + FSID [2]int32 + + // NameLength is the maximum file name length. + NameLength uint64 + + // FragmentSize is equivalent to BlockSize. + FragmentSize int64 + + // Flags is the set of filesystem mount flags. + Flags uint64 + + // Spare is unused. + Spare [4]uint64 +} + +// Whence argument to lseek(2), from include/uapi/linux/fs.h. +const ( + SEEK_SET = 0 + SEEK_CUR = 1 + SEEK_END = 2 + SEEK_DATA = 3 + SEEK_HOLE = 4 +) + +// Sync_file_range flags, from include/uapi/linux/fs.h +const ( + SYNC_FILE_RANGE_WAIT_BEFORE = 1 + SYNC_FILE_RANGE_WRITE = 2 + SYNC_FILE_RANGE_WAIT_AFTER = 4 +) + +// Flag argument to renameat2(2), from include/uapi/linux/fs.h. +const ( + RENAME_NOREPLACE = (1 << 0) // Don't overwrite target. + RENAME_EXCHANGE = (1 << 1) // Exchange src and dst. + RENAME_WHITEOUT = (1 << 2) // Whiteout src. +) + +// Overlayfs constants from include/linux/fs.h. +const ( + WHITEOUT_MODE = 0 + WHITEOUT_DEV = 0 +) diff --git a/pkg/abi/linux/fuse.go b/pkg/abi/linux/fuse.go new file mode 100644 index 0000000..1110579 --- /dev/null +++ b/pkg/abi/linux/fuse.go @@ -0,0 +1,1139 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/marshal/primitive" +) + +// FUSEOpcode is a FUSE operation code. +// +// +marshal +type FUSEOpcode uint32 + +// FUSEOpID is a FUSE operation ID. +// +// +marshal +type FUSEOpID uint64 + +// FUSE_ROOT_ID is the id of root inode. +const FUSE_ROOT_ID = 1 + +// Opcodes for FUSE operations. +// +// Analogous to the opcodes in include/linux/fuse.h. +const ( + FUSE_LOOKUP FUSEOpcode = 1 + FUSE_FORGET = 2 /* no reply */ + FUSE_GETATTR = 3 + FUSE_SETATTR = 4 + FUSE_READLINK = 5 + FUSE_SYMLINK = 6 + _ + FUSE_MKNOD = 8 + FUSE_MKDIR = 9 + FUSE_UNLINK = 10 + FUSE_RMDIR = 11 + FUSE_RENAME = 12 + FUSE_LINK = 13 + FUSE_OPEN = 14 + FUSE_READ = 15 + FUSE_WRITE = 16 + FUSE_STATFS = 17 + FUSE_RELEASE = 18 + _ + FUSE_FSYNC = 20 + FUSE_SETXATTR = 21 + FUSE_GETXATTR = 22 + FUSE_LISTXATTR = 23 + FUSE_REMOVEXATTR = 24 + FUSE_FLUSH = 25 + FUSE_INIT = 26 + FUSE_OPENDIR = 27 + FUSE_READDIR = 28 + FUSE_RELEASEDIR = 29 + FUSE_FSYNCDIR = 30 + FUSE_GETLK = 31 + FUSE_SETLK = 32 + FUSE_SETLKW = 33 + FUSE_ACCESS = 34 + FUSE_CREATE = 35 + FUSE_INTERRUPT = 36 + FUSE_BMAP = 37 + FUSE_DESTROY = 38 + FUSE_IOCTL = 39 + FUSE_POLL = 40 + FUSE_NOTIFY_REPLY = 41 + FUSE_BATCH_FORGET = 42 + FUSE_FALLOCATE = 43 +) + +const ( + // FUSE_MIN_READ_BUFFER is the minimum size the read can be for any FUSE filesystem. + // This is the minimum size Linux supports. See linux.fuse.h. + FUSE_MIN_READ_BUFFER uint32 = 8192 +) + +// FUSEHeaderIn is the header read by the daemon with each request. +// +// +marshal +// +stateify savable +type FUSEHeaderIn struct { + // Len specifies the total length of the data, including this header. + Len uint32 + + // Opcode specifies the kind of operation of the request. + Opcode FUSEOpcode + + // Unique specifies the unique identifier for this request. + Unique FUSEOpID + + // NodeID is the ID of the filesystem object being operated on. + NodeID uint64 + + // UID is the UID of the requesting process. + UID uint32 + + // GID is the GID of the requesting process. + GID uint32 + + // PID is the PID of the requesting process. + PID uint32 + + _ uint32 +} + +// SizeOfFUSEHeaderIn is the size of the FUSEHeaderIn struct. +var SizeOfFUSEHeaderIn = uint32((*FUSEHeaderIn)(nil).SizeBytes()) + +// FUSEHeaderOut is the header written by the daemon when it processes +// a request and wants to send a reply (almost all operations require a +// reply; if they do not, this will be explicitly documented). +// +// +marshal +// +stateify savable +type FUSEHeaderOut struct { + // Len specifies the total length of the data, including this header. + Len uint32 + + // Error specifies the error that occurred (0 if none). + Error int32 + + // Unique specifies the unique identifier of the corresponding request. + Unique FUSEOpID +} + +// SizeOfFUSEHeaderOut is the size of the FUSEHeaderOut struct. +var SizeOfFUSEHeaderOut = uint32((*FUSEHeaderOut)(nil).SizeBytes()) + +// FUSE_INIT flags, consistent with the ones in include/uapi/linux/fuse.h. +// Our target version is 7.23 but we have few implemented in advance. +const ( + FUSE_ASYNC_READ = 1 << 0 + FUSE_POSIX_LOCKS = 1 << 1 + FUSE_FILE_OPS = 1 << 2 + FUSE_ATOMIC_O_TRUNC = 1 << 3 + FUSE_EXPORT_SUPPORT = 1 << 4 + FUSE_BIG_WRITES = 1 << 5 + FUSE_DONT_MASK = 1 << 6 + FUSE_SPLICE_WRITE = 1 << 7 + FUSE_SPLICE_MOVE = 1 << 8 + FUSE_SPLICE_READ = 1 << 9 + FUSE_FLOCK_LOCKS = 1 << 10 + FUSE_HAS_IOCTL_DIR = 1 << 11 + FUSE_AUTO_INVAL_DATA = 1 << 12 + FUSE_DO_READDIRPLUS = 1 << 13 + FUSE_READDIRPLUS_AUTO = 1 << 14 + FUSE_ASYNC_DIO = 1 << 15 + FUSE_WRITEBACK_CACHE = 1 << 16 + FUSE_NO_OPEN_SUPPORT = 1 << 17 + FUSE_MAX_PAGES = 1 << 22 // From FUSE 7.28 +) + +// currently supported FUSE protocol version numbers. +const ( + FUSE_KERNEL_VERSION = 7 + FUSE_KERNEL_MINOR_VERSION = 31 +) + +// Constants relevant to FUSE operations. +const ( + FUSE_NAME_MAX = 1024 + FUSE_PAGE_SIZE = 4096 + FUSE_DIRENT_ALIGN = 8 +) + +// FUSEInitIn is the request sent by the kernel to the daemon, +// to negotiate the version and flags. +// +// +marshal +type FUSEInitIn struct { + // Major version supported by kernel. + Major uint32 + + // Minor version supported by the kernel. + Minor uint32 + + // MaxReadahead is the maximum number of bytes to read-ahead + // decided by the kernel. + MaxReadahead uint32 + + // Flags of this init request. + Flags uint32 +} + +// FUSEInitOut is the reply sent by the daemon to the kernel +// for FUSEInitIn. We target FUSE 7.23; this struct supports 7.28. +// +// +marshal +type FUSEInitOut struct { + // Major version supported by daemon. + Major uint32 + + // Minor version supported by daemon. + Minor uint32 + + // MaxReadahead is the maximum number of bytes to read-ahead. + // Decided by the daemon, after receiving the value from kernel. + MaxReadahead uint32 + + // Flags of this init reply. + Flags uint32 + + // MaxBackground is the maximum number of pending background requests + // that the daemon wants. + MaxBackground uint16 + + // CongestionThreshold is the daemon-decided threshold for + // the number of the pending background requests. + CongestionThreshold uint16 + + // MaxWrite is the daemon's maximum size of a write buffer. + // Kernel adjusts it to the minimum (fuse/init.go:fuseMinMaxWrite). + // if the value from daemon is too small. + MaxWrite uint32 + + // TimeGran is the daemon's time granularity for mtime and ctime metadata. + // The unit is nanosecond. + // Value should be power of 10. + // 1 indicates full nanosecond granularity support. + TimeGran uint32 + + // MaxPages is the daemon's maximum number of pages for one write operation. + // Kernel adjusts it to the maximum (fuse/init.go:FUSE_MAX_MAX_PAGES). + // if the value from daemon is too large. + MaxPages uint16 + + _ uint16 + + _ [8]uint32 +} + +// FUSEStatfsOut is the reply sent by the daemon to the kernel +// for FUSE_STATFS. +// from https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/fuse.h#L252 +// +// +marshal +type FUSEStatfsOut struct { + // Blocks is the maximum number of data blocks the filesystem may store, in + // units of BlockSize. + Blocks uint64 + + // BlocksFree is the number of free data blocks, in units of BlockSize. + BlocksFree uint64 + + // BlocksAvailable is the number of data blocks free for use by + // unprivileged users, in units of BlockSize. + BlocksAvailable uint64 + + // Files is the number of used file nodes on the filesystem. + Files uint64 + + // FileFress is the number of free file nodes on the filesystem. + FilesFree uint64 + + // BlockSize is the optimal transfer block size in bytes. + BlockSize uint32 + + // NameLength is the maximum file name length. + NameLength uint32 + + // FragmentSize is equivalent to BlockSize. + FragmentSize uint32 + + _ uint32 + + Spare [6]uint32 +} + +// FUSE_GETATTR_FH is currently the only flag of FUSEGetAttrIn.GetAttrFlags. +// If it is set, the file handle (FUSEGetAttrIn.Fh) is used to indicate the +// object instead of the node id attribute in the request header. +const FUSE_GETATTR_FH = (1 << 0) + +// FUSEGetAttrIn is the request sent by the kernel to the daemon, +// to get the attribute of a inode. +// +// +marshal +type FUSEGetAttrIn struct { + // GetAttrFlags specifies whether getattr request is sent with a nodeid or + // with a file handle. + GetAttrFlags uint32 + + _ uint32 + + // Fh is the file handler when GetAttrFlags has FUSE_GETATTR_FH bit. If + // used, the operation is analogous to fstat(2). + Fh uint64 +} + +// FUSEAttr is the struct used in the response FUSEGetAttrOut. +// +// +marshal +type FUSEAttr struct { + // Ino is the inode number of this file. + Ino uint64 + + // Size is the size of this file. + Size uint64 + + // Blocks is the number of the 512B blocks allocated by this file. + Blocks uint64 + + // Atime is the time of last access. + Atime uint64 + + // Mtime is the time of last modification. + Mtime uint64 + + // Ctime is the time of last status change. + Ctime uint64 + + // AtimeNsec is the nano second part of Atime. + AtimeNsec uint32 + + // MtimeNsec is the nano second part of Mtime. + MtimeNsec uint32 + + // CtimeNsec is the nano second part of Ctime. + CtimeNsec uint32 + + // Mode contains the file type and mode. + Mode uint32 + + // Nlink is the number of the hard links. + Nlink uint32 + + // UID is user ID of the owner. + UID uint32 + + // GID is group ID of the owner. + GID uint32 + + // Rdev is the device ID if this is a special file. + Rdev uint32 + + // BlkSize is the block size for filesystem I/O. + BlkSize uint32 + + _ uint32 +} + +// ATimeNsec returns the last access time as the total time since the unix epoch +// in nanoseconds. +func (a FUSEAttr) ATimeNsec() int64 { + return int64(a.Atime)*time.Second.Nanoseconds() + int64(a.AtimeNsec) +} + +// MTimeNsec returns the last modification time as the total time since the unix +// epoch in nanoseconds. +func (a FUSEAttr) MTimeNsec() int64 { + return int64(a.Mtime)*time.Second.Nanoseconds() + int64(a.MtimeNsec) +} + +// CTimeNsec returns the last change time as the total time since the unix epoch +// in nanoseconds. +func (a FUSEAttr) CTimeNsec() int64 { + return int64(a.Ctime)*time.Second.Nanoseconds() + int64(a.CtimeNsec) +} + +// FUSEAttrOut is the reply sent by the daemon to the kernel +// for FUSEGetAttrIn and FUSESetAttrIn. +// +// +marshal +type FUSEAttrOut struct { + // AttrValid and AttrValidNsec describe the attribute cache duration + AttrValid uint64 + + // AttrValidNsec is the nanosecond part of the attribute cache duration + AttrValidNsec uint32 + + _ uint32 + + // Attr contains the metadata returned from the FUSE server + Attr FUSEAttr +} + +// FUSEEntryOut is the reply sent by the daemon to the kernel +// for FUSE_MKNOD, FUSE_MKDIR, FUSE_SYMLINK, FUSE_LINK and +// FUSE_LOOKUP. +// +// +marshal +type FUSEEntryOut struct { + // NodeID is the ID for current inode. + NodeID uint64 + + // Generation is the generation number of inode. + // Used to identify an inode that have different ID at different time. + Generation uint64 + + // EntryValid indicates timeout for an entry. + EntryValid uint64 + + // AttrValid indicates timeout for an entry's attributes. + AttrValid uint64 + + // EntryValidNsec indicates timeout for an entry in nanosecond. + EntryValidNSec uint32 + + // AttrValidNsec indicates timeout for an entry's attributes in nanosecond. + AttrValidNSec uint32 + + // Attr contains the attributes of an entry. + Attr FUSEAttr +} + +// CString represents a null terminated string which can be marshalled. +// +// +marshal dynamic +type CString string + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *CString) MarshalBytes(buf []byte) []byte { + copy(buf, *s) + buf[len(*s)] = 0 // null char + return buf[s.SizeBytes():] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *CString) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, CString is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *CString) SizeBytes() int { + // 1 extra byte for null-terminated string. + return len(*s) + 1 +} + +// FUSELookupIn is the request sent by the kernel to the daemon +// to look up a file name. +// +// +marshal dynamic +type FUSELookupIn struct { + // Name is a file name to be looked up. + Name CString +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSELookupIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSELookupIn is never unmarshalled") +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSELookupIn) MarshalBytes(buf []byte) []byte { + return r.Name.MarshalBytes(buf) +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSELookupIn) SizeBytes() int { + return r.Name.SizeBytes() +} + +// MAX_NON_LFS indicates the maximum offset without large file support. +const MAX_NON_LFS = ((1 << 31) - 1) + +// flags returned by OPEN request. +const ( + // FOPEN_DIRECT_IO indicates bypassing page cache for this opened file. + FOPEN_DIRECT_IO = 1 << 0 + // FOPEN_KEEP_CACHE avoids invalidating the data cache on open. + FOPEN_KEEP_CACHE = 1 << 1 + // FOPEN_NONSEEKABLE indicates the file cannot be seeked. + FOPEN_NONSEEKABLE = 1 << 2 +) + +// FUSEOpenIn is the request sent by the kernel to the daemon, +// to negotiate flags and get file handle. +// +// +marshal +type FUSEOpenIn struct { + // Flags of this open request. + Flags uint32 + + _ uint32 +} + +// FUSEOpenOut is the reply sent by the daemon to the kernel +// for FUSEOpenIn. +// +// +marshal +type FUSEOpenOut struct { + // Fh is the file handler for opened files. + Fh uint64 + + // OpenFlag for the opened files. + OpenFlag uint32 + + _ uint32 +} + +// FUSECreateOut is the reply sent by the daemon to the kernel +// for FUSECreateMeta. +// +// +marshal +type FUSECreateOut struct { + FUSEEntryOut + FUSEOpenOut +} + +// FUSE_READ flags, consistent with the ones in include/uapi/linux/fuse.h. +const ( + FUSE_READ_LOCKOWNER = 1 << 1 +) + +// FUSEReadIn is the request sent by the kernel to the daemon +// for FUSE_READ. +// +// +marshal +type FUSEReadIn struct { + // Fh is the file handle in userspace. + Fh uint64 + + // Offset is the read offset. + Offset uint64 + + // Size is the number of bytes to read. + Size uint32 + + // ReadFlags for this FUSE_READ request. + // Currently only contains FUSE_READ_LOCKOWNER. + ReadFlags uint32 + + // LockOwner is the id of the lock owner if there is one. + LockOwner uint64 + + // Flags for the underlying file. + Flags uint32 + + _ uint32 +} + +// FUSEWriteIn is the first part of the payload of the +// request sent by the kernel to the daemon +// for FUSE_WRITE (struct for FUSE version >= 7.9). +// +// The second part of the payload is the +// binary bytes of the data to be written. +// See FUSEWritePayloadIn that combines header & payload. +// +// +marshal +type FUSEWriteIn struct { + // Fh is the file handle in userspace. + Fh uint64 + + // Offset is the write offset. + Offset uint64 + + // Size is the number of bytes to write. + Size uint32 + + // ReadFlags for this FUSE_WRITE request. + WriteFlags uint32 + + // LockOwner is the id of the lock owner if there is one. + LockOwner uint64 + + // Flags for the underlying file. + Flags uint32 + + _ uint32 +} + +// SizeOfFUSEWriteIn is the size of the FUSEWriteIn struct. +var SizeOfFUSEWriteIn = uint32((*FUSEWriteIn)(nil).SizeBytes()) + +// FUSEWritePayloadIn combines header - FUSEWriteIn and payload +// in a single marshallable struct when sending request by the +// kernel to the daemon +// +// +marshal dynamic +type FUSEWritePayloadIn struct { + Header FUSEWriteIn + Payload primitive.ByteSlice +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEWritePayloadIn) SizeBytes() int { + if r == nil { + return (*FUSEWriteIn)(nil).SizeBytes() + } + return r.Header.SizeBytes() + r.Payload.SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEWritePayloadIn) MarshalBytes(dst []byte) []byte { + dst = r.Header.MarshalUnsafe(dst) + dst = r.Payload.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEWritePayloadIn) UnmarshalBytes(src []byte) []byte { + panic("Unimplemented, FUSEWritePayloadIn is never unmarshalled") +} + +// FUSEWriteOut is the payload of the reply sent by the daemon to the kernel +// for a FUSE_WRITE request. +// +// +marshal +type FUSEWriteOut struct { + // Size is the number of bytes written. + Size uint32 + + _ uint32 +} + +// FUSEReleaseIn is the request sent by the kernel to the daemon +// when there is no more reference to a file. +// +// +marshal +type FUSEReleaseIn struct { + // Fh is the file handler for the file to be released. + Fh uint64 + + // Flags of the file. + Flags uint32 + + // ReleaseFlags of this release request. + ReleaseFlags uint32 + + // LockOwner is the id of the lock owner if there is one. + LockOwner uint64 +} + +// FUSECreateMeta contains all the static fields of FUSECreateIn, +// which is used for FUSE_CREATE. +// +// +marshal +type FUSECreateMeta struct { + // Flags of the creating file. + Flags uint32 + + // Mode is the mode of the creating file. + Mode uint32 + + // Umask is the current file mode creation mask. + Umask uint32 + _ uint32 +} + +// FUSERenameIn sent by the kernel for FUSE_RENAME +// +// +marshal dynamic +type FUSERenameIn struct { + Newdir primitive.Uint64 + Oldname CString + Newname CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSERenameIn) MarshalBytes(dst []byte) []byte { + dst = r.Newdir.MarshalBytes(dst) + dst = r.Oldname.MarshalBytes(dst) + return r.Newname.MarshalBytes(dst) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSERenameIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSERmDirIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSERenameIn) SizeBytes() int { + return r.Newdir.SizeBytes() + r.Oldname.SizeBytes() + r.Newname.SizeBytes() +} + +// FUSECreateIn contains all the arguments sent by the kernel to the daemon, to +// atomically create and open a new regular file. +// +// +marshal dynamic +type FUSECreateIn struct { + // CreateMeta contains mode, rdev and umash fields for FUSE_MKNODS. + CreateMeta FUSECreateMeta + + // Name is the name of the node to create. + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSECreateIn) MarshalBytes(buf []byte) []byte { + buf = r.CreateMeta.MarshalBytes(buf) + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSECreateIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSECreateIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSECreateIn) SizeBytes() int { + return r.CreateMeta.SizeBytes() + r.Name.SizeBytes() +} + +// FUSEMknodMeta contains all the static fields of FUSEMknodIn, +// which is used for FUSE_MKNOD. +// +// +marshal +type FUSEMknodMeta struct { + // Mode of the inode to create. + Mode uint32 + + // Rdev encodes device major and minor information. + Rdev uint32 + + // Umask is the current file mode creation mask. + Umask uint32 + + _ uint32 +} + +// FUSEMknodIn contains all the arguments sent by the kernel +// to the daemon, to create a new file node. +// +// +marshal dynamic +type FUSEMknodIn struct { + // MknodMeta contains mode, rdev and umash fields for FUSE_MKNODS. + MknodMeta FUSEMknodMeta + // Name is the name of the node to create. + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEMknodIn) MarshalBytes(buf []byte) []byte { + buf = r.MknodMeta.MarshalBytes(buf) + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEMknodIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEMknodIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEMknodIn) SizeBytes() int { + return r.MknodMeta.SizeBytes() + r.Name.SizeBytes() +} + +// FUSESymlinkIn is the request sent by the kernel to the daemon, +// to create a symbolic link. +// +// +marshal dynamic +type FUSESymlinkIn struct { + // Name of symlink to create. + Name CString + + // Target of the symlink. + Target CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSESymlinkIn) MarshalBytes(buf []byte) []byte { + buf = r.Name.MarshalBytes(buf) + return r.Target.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSESymlinkIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEMknodIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSESymlinkIn) SizeBytes() int { + return r.Name.SizeBytes() + r.Target.SizeBytes() +} + +// FUSELinkIn is the request sent by the kernel to create a hard link. +// +// +marshal dynamic +type FUSELinkIn struct { + // OldNodeID is the ID of the inode that is being linked to. + OldNodeID primitive.Uint64 + // Name of the new hard link to create. + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSELinkIn) MarshalBytes(buf []byte) []byte { + buf = r.OldNodeID.MarshalBytes(buf) + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSELinkIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSELinkIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSELinkIn) SizeBytes() int { + return r.OldNodeID.SizeBytes() + r.Name.SizeBytes() +} + +// FUSEEmptyIn is used by operations without request body. +// +// +marshal dynamic +type FUSEEmptyIn struct{} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEEmptyIn) MarshalBytes(buf []byte) []byte { + return buf +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEEmptyIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEEmptyIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEEmptyIn) SizeBytes() int { + return 0 +} + +// FUSEMkdirMeta contains all the static fields of FUSEMkdirIn, +// which is used for FUSE_MKDIR. +// +// +marshal +type FUSEMkdirMeta struct { + // Mode of the directory of create. + Mode uint32 + // Umask is the user file creation mask. + Umask uint32 +} + +// FUSEMkdirIn contains all the arguments sent by the kernel +// to the daemon, to create a new directory. +// +// +marshal dynamic +type FUSEMkdirIn struct { + // MkdirMeta contains Mode and Umask of the directory to create. + MkdirMeta FUSEMkdirMeta + // Name of the directory to create. + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEMkdirIn) MarshalBytes(buf []byte) []byte { + buf = r.MkdirMeta.MarshalBytes(buf) + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEMkdirIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEMkdirIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEMkdirIn) SizeBytes() int { + return r.MkdirMeta.SizeBytes() + r.Name.SizeBytes() +} + +// FUSERmDirIn is the request sent by the kernel to the daemon +// when trying to remove a directory. +// +// +marshal dynamic +type FUSERmDirIn struct { + // Name is a directory name to be removed. + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSERmDirIn) MarshalBytes(buf []byte) []byte { + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSERmDirIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSERmDirIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSERmDirIn) SizeBytes() int { + return r.Name.SizeBytes() +} + +// FUSEDirents is a list of Dirents received from the FUSE daemon server. +// It is used for FUSE_READDIR. +// +// +marshal dynamic +type FUSEDirents struct { + Dirents []*FUSEDirent +} + +// FUSEDirent is a Dirent received from the FUSE daemon server. +// It is used for FUSE_READDIR. +// +// +marshal dynamic +type FUSEDirent struct { + // Meta contains all the static fields of FUSEDirent. + Meta FUSEDirentMeta + // Name is the filename of the dirent. + Name string +} + +// FUSEDirentMeta contains all the static fields of FUSEDirent. +// It is used for FUSE_READDIR. +// +// +marshal +type FUSEDirentMeta struct { + // Inode of the dirent. + Ino uint64 + // Offset of the dirent. + Off uint64 + // NameLen is the length of the dirent name. + NameLen uint32 + // Type of the dirent. + Type uint32 +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEDirents) SizeBytes() int { + var sizeBytes int + for _, dirent := range r.Dirents { + sizeBytes += dirent.SizeBytes() + } + + return sizeBytes +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEDirents) MarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEDirents is never marshalled") +} + +// UnmarshalBytes deserializes FUSEDirents from the src buffer. +func (r *FUSEDirents) UnmarshalBytes(src []byte) []byte { + for { + if len(src) <= (*FUSEDirentMeta)(nil).SizeBytes() { + break + } + + // Its unclear how many dirents there are in src. Each dirent is dynamically + // sized and so we can't make assumptions about how many dirents we can allocate. + if r.Dirents == nil { + r.Dirents = make([]*FUSEDirent, 0) + } + + // We have to allocate a struct for each dirent - there must be a better way + // to do this. Linux allocates 1 page to store all the dirents and then + // simply reads them from the page. + var dirent FUSEDirent + src = dirent.UnmarshalBytes(src) + r.Dirents = append(r.Dirents, &dirent) + } + return src +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEDirent) SizeBytes() int { + dataSize := r.Meta.SizeBytes() + len(r.Name) + + // Each Dirent must be padded such that its size is a multiple + // of FUSE_DIRENT_ALIGN. Similar to the fuse dirent alignment + // in linux/fuse.h. + return (dataSize + (FUSE_DIRENT_ALIGN - 1)) & ^(FUSE_DIRENT_ALIGN - 1) +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEDirent) MarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEDirent is never marshalled") +} + +// shiftNextDirent advances buf to the start of the next dirent, per +// FUSE ABI. buf should begin at the start of a dirent. +func (r *FUSEDirent) shiftNextDirent(buf []byte) []byte { + nextOff := r.SizeBytes() + if nextOff > len(buf) { // Handle overflow. + return buf[len(buf):] + } + return buf[nextOff:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEDirent) UnmarshalBytes(src []byte) []byte { + srcP := r.Meta.UnmarshalBytes(src) + + if r.Meta.NameLen > FUSE_NAME_MAX || r.Meta.NameLen > uint32(len(srcP)) { + // The name is too long and therefore invalid. We don't + // need to unmarshal the name since it'll be thrown away. + return r.shiftNextDirent(src) + } + + buf := make([]byte, r.Meta.NameLen) + name := primitive.ByteSlice(buf) + name.UnmarshalBytes(srcP[:r.Meta.NameLen]) + r.Name = string(name) + return r.shiftNextDirent(src) +} + +// FATTR_* consts are the attribute flags defined in include/uapi/linux/fuse.h. +// These should be or-ed together for setattr to know what has been changed. +const ( + FATTR_MODE = (1 << 0) + FATTR_UID = (1 << 1) + FATTR_GID = (1 << 2) + FATTR_SIZE = (1 << 3) + FATTR_ATIME = (1 << 4) + FATTR_MTIME = (1 << 5) + FATTR_FH = (1 << 6) + FATTR_ATIME_NOW = (1 << 7) + FATTR_MTIME_NOW = (1 << 8) + FATTR_LOCKOWNER = (1 << 9) + FATTR_CTIME = (1 << 10) +) + +// FUSESetAttrIn is the request sent by the kernel to the daemon, +// to set the attribute(s) of a file. +// +// +marshal +type FUSESetAttrIn struct { + // Valid indicates which attributes are modified by this request. + Valid uint32 + + _ uint32 + + // Fh is used to identify the file if FATTR_FH is set in Valid. + Fh uint64 + + // Size is the size that the request wants to change to. + Size uint64 + + // LockOwner is the owner of the lock that the request wants to change to. + LockOwner uint64 + + // Atime is the access time that the request wants to change to. + Atime uint64 + + // Mtime is the modification time that the request wants to change to. + Mtime uint64 + + // Ctime is the status change time that the request wants to change to. + Ctime uint64 + + // AtimeNsec is the nano second part of Atime. + AtimeNsec uint32 + + // MtimeNsec is the nano second part of Mtime. + MtimeNsec uint32 + + // CtimeNsec is the nano second part of Ctime. + CtimeNsec uint32 + + // Mode is the file mode that the request wants to change to. + Mode uint32 + + _ uint32 + + // UID is the user ID of the owner that the request wants to change to. + UID uint32 + + // GID is the group ID of the owner that the request wants to change to. + GID uint32 + + _ uint32 +} + +// FUSEUnlinkIn is the request sent by the kernel to the daemon +// when trying to unlink a node. +// +// +marshal dynamic +type FUSEUnlinkIn struct { + // Name of the node to unlink. + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEUnlinkIn) MarshalBytes(buf []byte) []byte { + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEUnlinkIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEUnlinkIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEUnlinkIn) SizeBytes() int { + return r.Name.SizeBytes() +} + +// FUSEFsyncIn is the request sent by the kernel to the daemon +// when trying to fsync a file. +// +// +marshal +type FUSEFsyncIn struct { + Fh uint64 + + FsyncFlags uint32 + + // padding + _ uint32 +} + +// FUSEAccessIn is the request sent by the kernel to the daemon when checking +// permissions on a file. +// +// +marshal +type FUSEAccessIn struct { + Mask uint32 + // padding + _ uint32 +} + +// FUSEFallocateIn is the request sent by the kernel to the daemon to perform +// a fallocate operation. +// +// +marshal +type FUSEFallocateIn struct { + Fh uint64 + Offset uint64 + Length uint64 + Mode uint32 + // padding + _ uint32 +} + +// FUSEFlushIn is the request sent by the kernel to the daemon after a file is +// closed. +// +// +marshal +type FUSEFlushIn struct { + Fh uint64 + _ uint32 // unused + _ uint32 // padding + LockOwner uint64 +} diff --git a/pkg/abi/linux/futex.go b/pkg/abi/linux/futex.go new file mode 100644 index 0000000..8138088 --- /dev/null +++ b/pkg/abi/linux/futex.go @@ -0,0 +1,80 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// From and . +// Flags are used in syscall futex(2). +const ( + FUTEX_WAIT = 0 + FUTEX_WAKE = 1 + FUTEX_FD = 2 + FUTEX_REQUEUE = 3 + FUTEX_CMP_REQUEUE = 4 + FUTEX_WAKE_OP = 5 + FUTEX_LOCK_PI = 6 + FUTEX_UNLOCK_PI = 7 + FUTEX_TRYLOCK_PI = 8 + FUTEX_WAIT_BITSET = 9 + FUTEX_WAKE_BITSET = 10 + FUTEX_WAIT_REQUEUE_PI = 11 + FUTEX_CMP_REQUEUE_PI = 12 + + FUTEX_PRIVATE_FLAG = 128 + FUTEX_CLOCK_REALTIME = 256 +) + +// These are flags are from and are used in FUTEX_WAKE_OP +// to define the operations. +const ( + FUTEX_OP_SET = 0 + FUTEX_OP_ADD = 1 + FUTEX_OP_OR = 2 + FUTEX_OP_ANDN = 3 + FUTEX_OP_XOR = 4 + FUTEX_OP_OPARG_SHIFT = 8 + FUTEX_OP_CMP_EQ = 0 + FUTEX_OP_CMP_NE = 1 + FUTEX_OP_CMP_LT = 2 + FUTEX_OP_CMP_LE = 3 + FUTEX_OP_CMP_GT = 4 + FUTEX_OP_CMP_GE = 5 +) + +// FUTEX_TID_MASK is the TID portion of a PI futex word. +const FUTEX_TID_MASK = 0x3fffffff + +// Constants used for priority-inheritance futexes. +const ( + FUTEX_WAITERS = 0x80000000 + FUTEX_OWNER_DIED = 0x40000000 +) + +// FUTEX_BITSET_MATCH_ANY has all bits set. +const FUTEX_BITSET_MATCH_ANY = 0xffffffff + +// ROBUST_LIST_LIMIT protects against a deliberately circular list. +const ROBUST_LIST_LIMIT = 2048 + +// RobustListHead corresponds to Linux's struct robust_list_head. +// +// +marshal +type RobustListHead struct { + List uint64 + FutexOffset uint64 + ListOpPending uint64 +} + +// SizeOfRobustListHead is the size of a RobustListHead struct. +var SizeOfRobustListHead = (*RobustListHead)(nil).SizeBytes() diff --git a/pkg/abi/linux/inotify.go b/pkg/abi/linux/inotify.go new file mode 100644 index 0000000..2d08194 --- /dev/null +++ b/pkg/abi/linux/inotify.go @@ -0,0 +1,97 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Inotify events observable by userspace. These directly correspond to +// filesystem operations and there may only be a single of them per inotify +// event read from an inotify fd. +const ( + // IN_ACCESS indicates a file was accessed. + IN_ACCESS = 0x00000001 + // IN_MODIFY indicates a file was modified. + IN_MODIFY = 0x00000002 + // IN_ATTRIB indicates a watch target's metadata changed. + IN_ATTRIB = 0x00000004 + // IN_CLOSE_WRITE indicates a writable file was closed. + IN_CLOSE_WRITE = 0x00000008 + // IN_CLOSE_NOWRITE indicates a non-writable file was closed. + IN_CLOSE_NOWRITE = 0x00000010 + // IN_OPEN indicates a file was opened. + IN_OPEN = 0x00000020 + // IN_MOVED_FROM indicates a file was moved from X. + IN_MOVED_FROM = 0x00000040 + // IN_MOVED_TO indicates a file was moved to Y. + IN_MOVED_TO = 0x00000080 + // IN_CREATE indicates a file was created in a watched directory. + IN_CREATE = 0x00000100 + // IN_DELETE indicates a file was deleted in a watched directory. + IN_DELETE = 0x00000200 + // IN_DELETE_SELF indicates a watch target itself was deleted. + IN_DELETE_SELF = 0x00000400 + // IN_MOVE_SELF indicates a watch target itself was moved. + IN_MOVE_SELF = 0x00000800 + // IN_ALL_EVENTS is a mask for all observable userspace events. + IN_ALL_EVENTS = 0x00000fff +) + +// Inotify control events. These may be present in their own events, or ORed +// with other observable events. +const ( + // IN_UNMOUNT indicates the backing filesystem was unmounted. + IN_UNMOUNT = 0x00002000 + // IN_Q_OVERFLOW indicates the event queued overflowed. + IN_Q_OVERFLOW = 0x00004000 + // IN_IGNORED indicates a watch was removed, either implicitly or through + // inotify_rm_watch(2). + IN_IGNORED = 0x00008000 + // IN_ISDIR indicates the subject of an event was a directory. + IN_ISDIR = 0x40000000 +) + +// Feature flags for inotify_add_watch(2). +const ( + // IN_ONLYDIR indicates that a path should be watched only if it's a + // directory. + IN_ONLYDIR = 0x01000000 + // IN_DONT_FOLLOW indicates that the watch path shouldn't be resolved if + // it's a symlink. + IN_DONT_FOLLOW = 0x02000000 + // IN_EXCL_UNLINK indicates events to this watch from unlinked objects + // should be filtered out. + IN_EXCL_UNLINK = 0x04000000 + // IN_MASK_ADD indicates the provided mask should be ORed into any existing + // watch on the provided path. + IN_MASK_ADD = 0x20000000 + // IN_ONESHOT indicates the watch should be removed after one event. + IN_ONESHOT = 0x80000000 +) + +// Feature flags for inotify_init1(2). +const ( + // IN_CLOEXEC is an alias for O_CLOEXEC. It indicates that the inotify + // fd should be closed on exec(2) and friends. + IN_CLOEXEC = 0x00080000 + // IN_NONBLOCK is an alias for O_NONBLOCK. It indicates I/O syscall on the + // inotify fd should not block. + IN_NONBLOCK = 0x00000800 +) + +// ALL_INOTIFY_BITS contains all the bits for all possible inotify events. It's +// defined in the Linux source at "include/linux/inotify.h". +const ALL_INOTIFY_BITS = IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | + IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | + IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF | IN_UNMOUNT | IN_Q_OVERFLOW | + IN_IGNORED | IN_ONLYDIR | IN_DONT_FOLLOW | IN_EXCL_UNLINK | IN_MASK_ADD | + IN_ISDIR | IN_ONESHOT diff --git a/pkg/abi/linux/ioctl.go b/pkg/abi/linux/ioctl.go new file mode 100644 index 0000000..6ed8d2a --- /dev/null +++ b/pkg/abi/linux/ioctl.go @@ -0,0 +1,186 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// ioctl(2) requests provided by asm-generic/ioctls.h +// +// These are ordered by request number (low byte). +const ( + TCGETS = 0x00005401 + TCSETS = 0x00005402 + TCSETSW = 0x00005403 + TCSETSF = 0x00005404 + TCSBRK = 0x00005409 + TIOCEXCL = 0x0000540c + TIOCNXCL = 0x0000540d + TIOCSCTTY = 0x0000540e + TIOCGPGRP = 0x0000540f + TIOCSPGRP = 0x00005410 + TIOCOUTQ = 0x00005411 + TIOCSTI = 0x00005412 + TIOCGWINSZ = 0x00005413 + TIOCSWINSZ = 0x00005414 + TIOCMGET = 0x00005415 + TIOCMBIS = 0x00005416 + TIOCMBIC = 0x00005417 + TIOCMSET = 0x00005418 + TIOCINQ = 0x0000541b + FIONREAD = TIOCINQ + TIOCPKT = 0x00005420 + FIONBIO = 0x00005421 + TIOCSETD = 0x00005423 + TIOCNOTTY = 0x00005422 + TIOCGETD = 0x00005424 + TCSBRKP = 0x00005425 + TIOCSBRK = 0x00005427 + TIOCCBRK = 0x00005428 + TIOCGSID = 0x00005429 + TIOCGPTN = 0x80045430 + TIOCSPTLCK = 0x40045431 + TIOCGDEV = 0x80045432 + TIOCVHANGUP = 0x00005437 + TIOCGPKT = 0x80045438 + TCFLSH = 0x0000540b + TIOCCONS = 0x0000541d + TIOCSSERIAL = 0x0000541f + TIOCGEXCL = 0x80045440 + TIOCGPTPEER = 0x80045441 + TIOCGICOUNT = 0x0000545d + FIONCLEX = 0x00005450 + FIOCLEX = 0x00005451 + FIOASYNC = 0x00005452 + FIOSETOWN = 0x00008901 + SIOCSPGRP = 0x00008902 + FIOGETOWN = 0x00008903 + SIOCGPGRP = 0x00008904 +) + +// ioctl(2) requests provided by uapi/linux/sockios.h +const ( + SIOCGIFNAME = 0x8910 + SIOCGIFCONF = 0x8912 + SIOCGIFFLAGS = 0x8913 + SIOCGIFADDR = 0x8915 + SIOCGIFDSTADDR = 0x8917 + SIOCGIFBRDADDR = 0x8919 + SIOCGIFNETMASK = 0x891b + SIOCGIFMETRIC = 0x891d + SIOCGIFMTU = 0x8921 + SIOCGIFMEM = 0x891f + SIOCGIFHWADDR = 0x8927 + SIOCGIFINDEX = 0x8933 + SIOCGIFPFLAGS = 0x8935 + SIOCGIFTXQLEN = 0x8942 + SIOCETHTOOL = 0x8946 + SIOCGMIIPHY = 0x8947 + SIOCGMIIREG = 0x8948 + SIOCGIFMAP = 0x8970 +) + +// ioctl(2) requests provided by uapi/asm-generic/sockios.h +const ( + SIOCGSTAMP = 0x8906 +) + +// ioctl(2) directions. Used to calculate requests number. +// Constants from asm-generic/ioctl.h. +const ( + IOC_NONE = 0 + IOC_WRITE = 1 + IOC_READ = 2 +) + +// Constants from asm-generic/ioctl.h. +const ( + IOC_NRBITS = 8 + IOC_TYPEBITS = 8 + IOC_SIZEBITS = 14 + IOC_DIRBITS = 2 + + IOC_NRSHIFT = 0 + IOC_TYPESHIFT = IOC_NRSHIFT + IOC_NRBITS + IOC_SIZESHIFT = IOC_TYPESHIFT + IOC_TYPEBITS + IOC_DIRSHIFT = IOC_SIZESHIFT + IOC_SIZEBITS +) + +// IOC outputs the result of _IOC macro in include/uapi/asm-generic/ioctl.h. +func IOC(dir, typ, nr, size uint32) uint32 { + return uint32(dir)<> IOC_NRSHIFT) & ((1 << IOC_NRBITS) - 1) +} + +// IOC_SIZE outputs the result of IOC_SIZE macro in +// include/uapi/asm-generic/ioctl.h. +func IOC_SIZE(nr uint32) uint32 { + return (nr >> IOC_SIZESHIFT) & ((1 << IOC_SIZEBITS) - 1) +} + +/* Used for packet mode */ +const ( + TIOCPKT_DATA = 0 + TIOCPKT_FLUSHREAD = 1 + TIOCPKT_FLUSHWRITE = 2 + TIOCPKT_STOP = 4 + TIOCPKT_START = 8 + TIOCPKT_NOSTOP = 16 + TIOCPKT_DOSTOP = 32 + TIOCPKT_IOCTL = 64 +) + +// Kcov ioctls from include/uapi/linux/kcov.h. +var ( + KCOV_INIT_TRACE = IOR('c', 1, 8) + KCOV_ENABLE = IO('c', 100) + KCOV_DISABLE = IO('c', 101) +) + +// Kcov trace types from include/uapi/linux/kcov.h. +const ( + KCOV_TRACE_PC = 0 + KCOV_TRACE_CMP = 1 +) + +// Kcov state constants from include/uapi/linux/kcov.h. +const ( + KCOV_MODE_DISABLED = 0 + KCOV_MODE_INIT = 1 + KCOV_MODE_TRACE_PC = 2 + KCOV_MODE_TRACE_CMP = 3 +) diff --git a/pkg/abi/linux/ioctl_tun.go b/pkg/abi/linux/ioctl_tun.go new file mode 100644 index 0000000..a2f85a5 --- /dev/null +++ b/pkg/abi/linux/ioctl_tun.go @@ -0,0 +1,33 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// ioctl(2) request numbers from linux/if_tun.h +var ( + TUNSETIFF = IOW('T', 202, 4) + TUNSETPERSIST = IOW('T', 203, 4) + TUNGETIFF = IOR('T', 210, 4) +) + +// Flags from net/if_tun.h +const ( + IFF_TUN = 0x0001 + IFF_TAP = 0x0002 + IFF_NO_PI = 0x1000 + IFF_NOFILTER = 0x1000 + IFF_TUN_EXCL = 0x8000 + // According to linux/if_tun.h "This flag has no real effect" + IFF_ONE_QUEUE = 0x2000 +) diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go new file mode 100644 index 0000000..0eeb62a --- /dev/null +++ b/pkg/abi/linux/iouring.go @@ -0,0 +1,238 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Constants for io_uring_setup(2). See include/uapi/linux/io_uring.h. +const ( + IORING_SETUP_IOPOLL = (1 << 0) + IORING_SETUP_SQPOLL = (1 << 1) + IORING_SETUP_SQ_AFF = (1 << 2) + IORING_SETUP_CQSIZE = (1 << 3) + IORING_SETUP_CLAMP = (1 << 4) + IORING_SETUP_ATTACH_WQ = (1 << 5) + IORING_SETUP_R_DISABLED = (1 << 6) + IORING_SETUP_SUBMIT_ALL = (1 << 7) +) + +// Constants for io_uring_enter(2). See include/uapi/linux/io_uring.h. +const ( + IORING_ENTER_GETEVENTS = (1 << 0) +) + +// Constants for IoUringParams.Features. See include/uapi/linux/io_uring.h. +const ( + IORING_FEAT_SINGLE_MMAP = (1 << 0) +) + +// Constants for IO_URING. See include/uapi/linux/io_uring.h. +const ( + IORING_SETUP_COOP_TASKRUN = (1 << 8) + IORING_SETUP_TASKRUN_FLAG = (1 << 9) + IORING_SETUP_SQE128 = (1 << 10) + IORING_SETUP_CQE32 = (1 << 11) +) + +// Constants for IO_URING. See io_uring/io_uring.c. +const ( + IORING_MAX_ENTRIES = (1 << 15) // 32768 + IORING_MAX_CQ_ENTRIES = (2 * IORING_MAX_ENTRIES) +) + +// Constants for the offsets for the application to mmap the data it needs. +// See include/uapi/linux/io_uring.h. +const ( + IORING_OFF_SQ_RING = 0 + IORING_OFF_CQ_RING = 0x8000000 + IORING_OFF_SQES = 0x10000000 +) + +// Constants for the IO_URING opcodes. See include/uapi/linux/io_uring.h. +const ( + IORING_OP_NOP = 0 + IORING_OP_READV = 1 +) + +// IORingIndex represents SQE array indexes. +// +// +marshal +type IORingIndex uint32 + +// IOSqRingOffsets implements io_sqring_offsets struct. +// IOSqRingOffsets represents offsets into IORings. +// See struct io_sqring_offsets in include/uapi/linux/io_uring.h. +// +// +marshal +type IOSqRingOffsets struct { + Head uint32 // Offset to io_rings.sq.head + Tail uint32 // Offset to io_rings.sq.tail + RingMask uint32 // Offset to io_rings.sq_ring_mask + RingEntries uint32 // Offset to io_rings.sq_ring_entries + Flags uint32 // Offset to io_rings.sq_flags + Dropped uint32 // Offset to io_rings.sq_dropped + Array uint32 // Offset to an array of SQE indices + Resv1 uint32 // Currently reserved and expected to be zero + Resv2 uint64 // Currently reserved and expected to be zero +} + +// IOCqRingOffsets implements io_cqring_offsets struct. +// IOCqRingOffsets represents offsets into IORings. +// See struct io_cqring_offsets in include/uapi/linux/io_uring.h. +// +// +marshal +type IOCqRingOffsets struct { + Head uint32 // Offset to io_rings.cq.head + Tail uint32 // Offset to io_rings.cq.tail + RingMask uint32 // Offset to io_rings.cq_ring_mask + RingEntries uint32 // Offset to io_rings.cq_ring_entries + Overflow uint32 // Offset to io_rings.cq_overflow + Cqes uint32 // Offset to io_rings.cqes + Flags uint32 // Offset to io_rings.cq_flags + Resv1 uint32 // Currently reserved and expected to be zero + Resv2 uint64 // Currently reserved and expected to be zero +} + +// IOUringParams implements io_uring_params struct. +// See struct io_uring_params in include/uapi/linux/io_uring.h. +// +// +marshal +type IOUringParams struct { + SqEntries uint32 + CqEntries uint32 + Flags uint32 + SqThreadCPU uint32 + SqThreadIdle uint32 + Features uint32 + WqFd uint32 + Resv [3]uint32 + SqOff IOSqRingOffsets + CqOff IOCqRingOffsets +} + +// IOUringCqe implements IO completion data structure (Completion Queue Entry) +// io_uring_cqe struct. As we don't currently support IORING_SETUP_CQE32 flag +// its size is 16 bytes. +// See struct io_uring_cqe in include/uapi/linux/io_uring.h. +// +// +marshal +// +stateify savable +type IOUringCqe struct { + UserData uint64 + Res int32 + Flags uint32 +} + +// IOUring implements io_uring struct. +// See struct io_uring in io_uring/io_uring.c. +// +// +marshal +// +stateify savable +type IOUring struct { + // Both head and tail should be cacheline aligned. And we assume that + // cacheline size is 64 bytes. + Head uint32 + _ [60]byte + Tail uint32 + _ [60]byte +} + +// IORings implements io_rings struct. +// This struct describes layout of the mapped region backed by the ringBuffersFile. +// See struct io_rings in io_uring/io_uring.c. +// +// +marshal +// +stateify savable +type IORings struct { + Sq IOUring + Cq IOUring + SqRingMask uint32 + CqRingMask uint32 + SqRingEntries uint32 + CqRingEntries uint32 + sqDropped uint32 + sqFlags int32 + cqFlags uint32 + CqOverflow uint32 + _ [32]byte // Padding so cqes is cacheline aligned + // Linux has an additional field struct io_uring_cqe cqes[], which represents + // a dynamic array. We don't include it here in order to enable marshalling. +} + +// IOUringSqe implements io_uring_sqe struct. +// This struct represents IO submission data structure (Submission Queue Entry). As we don't yet +// support IORING_SETUP_SQE128 flag, its size is 64 bytes with no extra padding at the end. +// See include/uapi/linux/io_uring.h. +// +// +marshal +// +stateify savable +type IOUringSqe struct { + Opcode uint8 + Flags uint8 + IoPrio uint16 + Fd int32 + OffOrAddrOrCmdOp uint64 + AddrOrSpliceOff uint64 + Len uint32 + specialFlags uint32 + UserData uint64 + BufIndexOrGroup uint16 + personality uint16 + spliceFDOrFileIndex int32 + addr3 uint64 + _ uint64 +} + +const ( + _IOSqRingOffset = 0 // +checkoffset . IORings.Sq + _IOSqRingOffsetHead = 0 // +checkoffset . IOUring.Head + _IOSqRingOffsetTail = 64 // +checkoffset . IOUring.Tail + _IOSqRingOffsetMask = 256 // +checkoffset . IORings.SqRingMask + _IOSqRingOffsetEntries = 264 // +checkoffset . IORings.SqRingEntries + _IOSqRingOffsetFlags = 276 // +checkoffset . IORings.sqFlags + _IOSqRingOffsetDropped = 272 // +checkoffset . IORings.sqDropped +) + +// PreComputedIOSqRingOffsets returns precomputed values for IOSqRingOffsets. +func PreComputedIOSqRingOffsets() IOSqRingOffsets { + return IOSqRingOffsets{ + Head: _IOSqRingOffset + _IOSqRingOffsetHead, + Tail: _IOSqRingOffset + _IOSqRingOffsetTail, + RingMask: _IOSqRingOffsetMask, + RingEntries: _IOSqRingOffsetEntries, + Flags: _IOSqRingOffsetFlags, + Dropped: _IOSqRingOffsetDropped, + } +} + +const ( + _IOCqRingOffset = 128 // +checkoffset . IORings.Cq + _IOCqRingOffsetHead = 0 // +checkoffset . IOUring.Head + _IOCqRingOffsetTail = 64 // +checkoffset . IOUring.Tail + _IOCqRingOffsetMask = 260 // +checkoffset . IORings.CqRingMask + _IOCqRingOffsetEntries = 268 // +checkoffset . IORings.CqRingEntries + _IOCqRingOffsetFlags = 280 // +checkoffset . IORings.cqFlags + _IOCqRingOffsetOverflow = 284 // +checkoffset . IORings.CqOverflow +) + +// PreComputedIOCqRingOffsets returns precomputed values for IOCqRingOffsets. +func PreComputedIOCqRingOffsets() IOCqRingOffsets { + return IOCqRingOffsets{ + Head: _IOCqRingOffset + _IOCqRingOffsetHead, + Tail: _IOCqRingOffset + _IOCqRingOffsetTail, + RingMask: _IOCqRingOffsetMask, + RingEntries: _IOCqRingOffsetEntries, + Overflow: _IOCqRingOffsetOverflow, + Flags: _IOCqRingOffsetFlags, + } +} diff --git a/pkg/abi/linux/ip.go b/pkg/abi/linux/ip.go new file mode 100644 index 0000000..4e4c4bc --- /dev/null +++ b/pkg/abi/linux/ip.go @@ -0,0 +1,171 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// IP protocols +const ( + IPPROTO_IP = 0 + IPPROTO_ICMP = 1 + IPPROTO_IGMP = 2 + IPPROTO_IPIP = 4 + IPPROTO_TCP = 6 + IPPROTO_EGP = 8 + IPPROTO_PUP = 12 + IPPROTO_UDP = 17 + IPPROTO_IDP = 22 + IPPROTO_TP = 29 + IPPROTO_DCCP = 33 + IPPROTO_IPV6 = 41 + IPPROTO_RSVP = 46 + IPPROTO_GRE = 47 + IPPROTO_ESP = 50 + IPPROTO_AH = 51 + IPPROTO_ICMPV6 = 58 + IPPROTO_MTP = 92 + IPPROTO_BEETPH = 94 + IPPROTO_ENCAP = 98 + IPPROTO_PIM = 103 + IPPROTO_COMP = 108 + IPPROTO_SCTP = 132 + IPPROTO_UDPLITE = 136 + IPPROTO_MPLS = 137 + IPPROTO_RAW = 255 +) + +// Socket options from uapi/linux/in.h +const ( + IP_TOS = 1 + IP_TTL = 2 + IP_HDRINCL = 3 + IP_OPTIONS = 4 + IP_ROUTER_ALERT = 5 + IP_RECVOPTS = 6 + IP_RETOPTS = 7 + IP_PKTINFO = 8 + IP_PKTOPTIONS = 9 + IP_MTU_DISCOVER = 10 + IP_RECVERR = 11 + IP_RECVTTL = 12 + IP_RECVTOS = 13 + IP_MTU = 14 + IP_FREEBIND = 15 + IP_IPSEC_POLICY = 16 + IP_XFRM_POLICY = 17 + IP_PASSSEC = 18 + IP_TRANSPARENT = 19 + IP_ORIGDSTADDR = 20 + IP_RECVORIGDSTADDR = IP_ORIGDSTADDR + IP_MINTTL = 21 + IP_NODEFRAG = 22 + IP_CHECKSUM = 23 + IP_BIND_ADDRESS_NO_PORT = 24 + IP_RECVFRAGSIZE = 25 + IP_RECVERR_RFC4884 = 26 + IP_MULTICAST_IF = 32 + IP_MULTICAST_TTL = 33 + IP_MULTICAST_LOOP = 34 + IP_ADD_MEMBERSHIP = 35 + IP_DROP_MEMBERSHIP = 36 + IP_UNBLOCK_SOURCE = 37 + IP_BLOCK_SOURCE = 38 + IP_ADD_SOURCE_MEMBERSHIP = 39 + IP_DROP_SOURCE_MEMBERSHIP = 40 + IP_MSFILTER = 41 + MCAST_JOIN_GROUP = 42 + MCAST_BLOCK_SOURCE = 43 + MCAST_UNBLOCK_SOURCE = 44 + MCAST_LEAVE_GROUP = 45 + MCAST_JOIN_SOURCE_GROUP = 46 + MCAST_LEAVE_SOURCE_GROUP = 47 + MCAST_MSFILTER = 48 + IP_MULTICAST_ALL = 49 + IP_UNICAST_IF = 50 + IP_LOCAL_PORT_RANGE = 51 +) + +// IP_MTU_DISCOVER values from uapi/linux/in.h +const ( + IP_PMTUDISC_DONT = 0 + IP_PMTUDISC_WANT = 1 + IP_PMTUDISC_DO = 2 + IP_PMTUDISC_PROBE = 3 + IP_PMTUDISC_INTERFACE = 4 + IP_PMTUDISC_OMIT = 5 +) + +// Socket options from uapi/linux/in6.h +const ( + IPV6_ADDRFORM = 1 + IPV6_2292PKTINFO = 2 + IPV6_2292HOPOPTS = 3 + IPV6_2292DSTOPTS = 4 + IPV6_2292RTHDR = 5 + IPV6_2292PKTOPTIONS = 6 + IPV6_CHECKSUM = 7 + IPV6_2292HOPLIMIT = 8 + IPV6_NEXTHOP = 9 + IPV6_FLOWINFO = 11 + IPV6_UNICAST_HOPS = 16 + IPV6_MULTICAST_IF = 17 + IPV6_MULTICAST_HOPS = 18 + IPV6_MULTICAST_LOOP = 19 + IPV6_ADD_MEMBERSHIP = 20 + IPV6_DROP_MEMBERSHIP = 21 + IPV6_ROUTER_ALERT = 22 + IPV6_MTU_DISCOVER = 23 + IPV6_MTU = 24 + IPV6_RECVERR = 25 + IPV6_V6ONLY = 26 + IPV6_JOIN_ANYCAST = 27 + IPV6_LEAVE_ANYCAST = 28 + IPV6_MULTICAST_ALL = 29 + IPV6_ROUTER_ALERT_ISOLATE = 30 + IPV6_RECVERR_RFC4884 = 31 + IPV6_FLOWLABEL_MGR = 32 + IPV6_FLOWINFO_SEND = 33 + IPV6_IPSEC_POLICY = 34 + IPV6_XFRM_POLICY = 35 + IPV6_HDRINCL = 36 + IPV6_RECVPKTINFO = 49 + IPV6_PKTINFO = 50 + IPV6_RECVHOPLIMIT = 51 + IPV6_HOPLIMIT = 52 + IPV6_RECVHOPOPTS = 53 + IPV6_HOPOPTS = 54 + IPV6_RTHDRDSTOPTS = 55 + IPV6_RECVRTHDR = 56 + IPV6_RTHDR = 57 + IPV6_RECVDSTOPTS = 58 + IPV6_DSTOPTS = 59 + IPV6_RECVPATHMTU = 60 + IPV6_PATHMTU = 61 + IPV6_DONTFRAG = 62 + IPV6_RECVTCLASS = 66 + IPV6_TCLASS = 67 + IPV6_AUTOFLOWLABEL = 70 + IPV6_ADDR_PREFERENCES = 72 + IPV6_MINHOPCOUNT = 73 + IPV6_ORIGDSTADDR = 74 + IPV6_RECVORIGDSTADDR = IPV6_ORIGDSTADDR + IPV6_TRANSPARENT = 75 + IPV6_UNICAST_IF = 76 + IPV6_RECVFRAGSIZE = 77 + IPV6_FREEBIND = 78 +) + +// Socket options from uapi/linux/icmpv6.h +const ( + ICMPV6_FILTER = 1 +) diff --git a/pkg/abi/linux/ipc.go b/pkg/abi/linux/ipc.go new file mode 100644 index 0000000..57af6e8 --- /dev/null +++ b/pkg/abi/linux/ipc.go @@ -0,0 +1,61 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Control commands used with semctl, shmctl, and msgctl. +// +// Source: include/uapi/linux/ipc.h. +const ( + IPC_RMID = 0 + IPC_SET = 1 + IPC_STAT = 2 + IPC_INFO = 3 +) + +// Resource get request flags. +// +// Source: include/uapi/linux/ipc.h +const ( + IPC_CREAT = 0o0001000 + IPC_EXCL = 0o0002000 + IPC_NOWAIT = 0o0004000 +) + +// IPC flags. +const ( + IPC_PRIVATE = 0 +) + +// In Linux, amd64 does not enable CONFIG_ARCH_WANT_IPC_PARSE_VERSION, so SysV +// IPC unconditionally uses the "new" 64-bit structures that are needed for +// features like 32-bit UIDs. + +// IPCPerm is equivalent to struct ipc64_perm. +// +// +marshal +type IPCPerm struct { + Key uint32 + UID uint32 + GID uint32 + CUID uint32 + CGID uint32 + Mode uint16 + _ uint16 + Seq uint16 + _ uint16 + _ uint32 + unused1 uint64 + unused2 uint64 +} diff --git a/pkg/abi/linux/keyctl.go b/pkg/abi/linux/keyctl.go new file mode 100644 index 0000000..4fedb33 --- /dev/null +++ b/pkg/abi/linux/keyctl.go @@ -0,0 +1,29 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Constants used by keyctl(2) and other keyrings-related syscalls. +// Source: include/uapi/linux/keyctl.h + +const ( + KEY_SPEC_SESSION_KEYRING = -3 +) + +const ( + KEYCTL_GET_KEYRING_ID = 0 + KEYCTL_JOIN_SESSION_KEYRING = 1 + KEYCTL_SETPERM = 5 + KEYCTL_DESCRIBE = 6 +) diff --git a/pkg/abi/linux/limits.go b/pkg/abi/linux/limits.go new file mode 100644 index 0000000..c74dfcd --- /dev/null +++ b/pkg/abi/linux/limits.go @@ -0,0 +1,88 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Resources for getrlimit(2)/setrlimit(2)/prlimit(2). +const ( + RLIMIT_CPU = 0 + RLIMIT_FSIZE = 1 + RLIMIT_DATA = 2 + RLIMIT_STACK = 3 + RLIMIT_CORE = 4 + RLIMIT_RSS = 5 + RLIMIT_NPROC = 6 + RLIMIT_NOFILE = 7 + RLIMIT_MEMLOCK = 8 + RLIMIT_AS = 9 + RLIMIT_LOCKS = 10 + RLIMIT_SIGPENDING = 11 + RLIMIT_MSGQUEUE = 12 + RLIMIT_NICE = 13 + RLIMIT_RTPRIO = 14 + RLIMIT_RTTIME = 15 +) + +// RLimit corresponds to Linux's struct rlimit. +type RLimit struct { + // Cur specifies the soft limit. + Cur uint64 + // Max specifies the hard limit. + Max uint64 +} + +const ( + // RLimInfinity is RLIM_INFINITY on Linux. + RLimInfinity = ^uint64(0) + + // DefaultStackSoftLimit is called _STK_LIM in Linux. + DefaultStackSoftLimit = 8 * 1024 * 1024 + + // DefaultNprocLimit is defined in kernel/fork.c:set_max_threads, and + // called MAX_THREADS / 2 in Linux. + DefaultNprocLimit = FUTEX_TID_MASK / 2 + + // DefaultNofileSoftLimit is called INR_OPEN_CUR in Linux. + DefaultNofileSoftLimit = 1024 + + // DefaultNofileHardLimit is called INR_OPEN_MAX in Linux. + DefaultNofileHardLimit = 4096 + + // DefaultMemlockLimit is called MLOCK_LIMIT in Linux. + DefaultMemlockLimit = 64 * 1024 + + // DefaultMsgqueueLimit is called MQ_BYTES_MAX in Linux. + DefaultMsgqueueLimit = 819200 +) + +// InitRLimits is a map of initial rlimits set by Linux in +// include/asm-generic/resource.h. +var InitRLimits = map[int]RLimit{ + RLIMIT_CPU: {RLimInfinity, RLimInfinity}, + RLIMIT_FSIZE: {RLimInfinity, RLimInfinity}, + RLIMIT_DATA: {RLimInfinity, RLimInfinity}, + RLIMIT_STACK: {DefaultStackSoftLimit, RLimInfinity}, + RLIMIT_CORE: {0, RLimInfinity}, + RLIMIT_RSS: {RLimInfinity, RLimInfinity}, + RLIMIT_NPROC: {DefaultNprocLimit, DefaultNprocLimit}, + RLIMIT_NOFILE: {DefaultNofileSoftLimit, DefaultNofileHardLimit}, + RLIMIT_MEMLOCK: {DefaultMemlockLimit, DefaultMemlockLimit}, + RLIMIT_AS: {RLimInfinity, RLimInfinity}, + RLIMIT_LOCKS: {RLimInfinity, RLimInfinity}, + RLIMIT_SIGPENDING: {0, 0}, + RLIMIT_MSGQUEUE: {DefaultMsgqueueLimit, DefaultMsgqueueLimit}, + RLIMIT_NICE: {0, 0}, + RLIMIT_RTPRIO: {0, 0}, + RLIMIT_RTTIME: {RLimInfinity, RLimInfinity}, +} diff --git a/pkg/abi/linux/linux.go b/pkg/abi/linux/linux.go new file mode 100644 index 0000000..3b4abec --- /dev/null +++ b/pkg/abi/linux/linux.go @@ -0,0 +1,42 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package linux contains the constants and types needed to interface with a +// Linux kernel. +package linux + +// NumSoftIRQ is the number of software IRQs, exposed via /proc/stat. +// +// Defined in linux/interrupt.h. +const NumSoftIRQ = 10 + +// Sysinfo is the structure provided by sysinfo on linux versions > 2.3.48. +// +// +marshal +type Sysinfo struct { + Uptime int64 + Loads [3]uint64 + TotalRAM uint64 + FreeRAM uint64 + SharedRAM uint64 + BufferRAM uint64 + TotalSwap uint64 + FreeSwap uint64 + Procs uint16 + _ [6]byte // Pad Procs to 64bits. + TotalHigh uint64 + FreeHigh uint64 + Unit uint32 `marshal:"unaligned"` // Struct ends mid-64-bit-word. + // The _f field in the glibc version of Sysinfo has size 0 on AMD64. +} diff --git a/pkg/abi/linux/linux_abi_autogen_unsafe.go b/pkg/abi/linux/linux_abi_autogen_unsafe.go new file mode 100644 index 0000000..20a0129 --- /dev/null +++ b/pkg/abi/linux/linux_abi_autogen_unsafe.go @@ -0,0 +1,22507 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +package linux + +import ( + "io" + "reflect" + "runtime" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Marshallable types used by this file. +var ( + _ marshal.Marshallable = (*BPFInstruction)(nil) + _ marshal.Marshallable = (*CString)(nil) + _ marshal.Marshallable = (*CapUserData)(nil) + _ marshal.Marshallable = (*CapUserHeader)(nil) + _ marshal.Marshallable = (*ClockT)(nil) + _ marshal.Marshallable = (*CloneArgs)(nil) + _ marshal.Marshallable = (*ControlMessageCredentials)(nil) + _ marshal.Marshallable = (*ControlMessageHeader)(nil) + _ marshal.Marshallable = (*ControlMessageIPPacketInfo)(nil) + _ marshal.Marshallable = (*ControlMessageIPv6PacketInfo)(nil) + _ marshal.Marshallable = (*ElfHeader64)(nil) + _ marshal.Marshallable = (*ElfProg64)(nil) + _ marshal.Marshallable = (*ElfSection64)(nil) + _ marshal.Marshallable = (*ErrorName)(nil) + _ marshal.Marshallable = (*EthtoolCmd)(nil) + _ marshal.Marshallable = (*EthtoolGFeatures)(nil) + _ marshal.Marshallable = (*EthtoolGetFeaturesBlock)(nil) + _ marshal.Marshallable = (*ExtensionName)(nil) + _ marshal.Marshallable = (*FOwnerEx)(nil) + _ marshal.Marshallable = (*FUSEAccessIn)(nil) + _ marshal.Marshallable = (*FUSEAttr)(nil) + _ marshal.Marshallable = (*FUSEAttrOut)(nil) + _ marshal.Marshallable = (*FUSECreateIn)(nil) + _ marshal.Marshallable = (*FUSECreateMeta)(nil) + _ marshal.Marshallable = (*FUSECreateOut)(nil) + _ marshal.Marshallable = (*FUSEDirent)(nil) + _ marshal.Marshallable = (*FUSEDirentMeta)(nil) + _ marshal.Marshallable = (*FUSEDirents)(nil) + _ marshal.Marshallable = (*FUSEEmptyIn)(nil) + _ marshal.Marshallable = (*FUSEEntryOut)(nil) + _ marshal.Marshallable = (*FUSEFallocateIn)(nil) + _ marshal.Marshallable = (*FUSEFlushIn)(nil) + _ marshal.Marshallable = (*FUSEFsyncIn)(nil) + _ marshal.Marshallable = (*FUSEGetAttrIn)(nil) + _ marshal.Marshallable = (*FUSEHeaderIn)(nil) + _ marshal.Marshallable = (*FUSEHeaderOut)(nil) + _ marshal.Marshallable = (*FUSEInitIn)(nil) + _ marshal.Marshallable = (*FUSEInitOut)(nil) + _ marshal.Marshallable = (*FUSELinkIn)(nil) + _ marshal.Marshallable = (*FUSELookupIn)(nil) + _ marshal.Marshallable = (*FUSEMkdirIn)(nil) + _ marshal.Marshallable = (*FUSEMkdirMeta)(nil) + _ marshal.Marshallable = (*FUSEMknodIn)(nil) + _ marshal.Marshallable = (*FUSEMknodMeta)(nil) + _ marshal.Marshallable = (*FUSEOpID)(nil) + _ marshal.Marshallable = (*FUSEOpcode)(nil) + _ marshal.Marshallable = (*FUSEOpenIn)(nil) + _ marshal.Marshallable = (*FUSEOpenOut)(nil) + _ marshal.Marshallable = (*FUSEReadIn)(nil) + _ marshal.Marshallable = (*FUSEReleaseIn)(nil) + _ marshal.Marshallable = (*FUSERenameIn)(nil) + _ marshal.Marshallable = (*FUSERmDirIn)(nil) + _ marshal.Marshallable = (*FUSESetAttrIn)(nil) + _ marshal.Marshallable = (*FUSEStatfsOut)(nil) + _ marshal.Marshallable = (*FUSESymlinkIn)(nil) + _ marshal.Marshallable = (*FUSEUnlinkIn)(nil) + _ marshal.Marshallable = (*FUSEWriteIn)(nil) + _ marshal.Marshallable = (*FUSEWriteOut)(nil) + _ marshal.Marshallable = (*FUSEWritePayloadIn)(nil) + _ marshal.Marshallable = (*FileMode)(nil) + _ marshal.Marshallable = (*Flock)(nil) + _ marshal.Marshallable = (*ICMP6Filter)(nil) + _ marshal.Marshallable = (*IFConf)(nil) + _ marshal.Marshallable = (*IFReq)(nil) + _ marshal.Marshallable = (*IOCallback)(nil) + _ marshal.Marshallable = (*IOCqRingOffsets)(nil) + _ marshal.Marshallable = (*IOEvent)(nil) + _ marshal.Marshallable = (*IORingIndex)(nil) + _ marshal.Marshallable = (*IORings)(nil) + _ marshal.Marshallable = (*IOSqRingOffsets)(nil) + _ marshal.Marshallable = (*IOUring)(nil) + _ marshal.Marshallable = (*IOUringCqe)(nil) + _ marshal.Marshallable = (*IOUringParams)(nil) + _ marshal.Marshallable = (*IOUringSqe)(nil) + _ marshal.Marshallable = (*IP6TEntry)(nil) + _ marshal.Marshallable = (*IP6TIP)(nil) + _ marshal.Marshallable = (*IP6TReplace)(nil) + _ marshal.Marshallable = (*IPCPerm)(nil) + _ marshal.Marshallable = (*IPTEntry)(nil) + _ marshal.Marshallable = (*IPTGetEntries)(nil) + _ marshal.Marshallable = (*IPTGetinfo)(nil) + _ marshal.Marshallable = (*IPTIP)(nil) + _ marshal.Marshallable = (*IPTOwnerInfo)(nil) + _ marshal.Marshallable = (*IPTReplace)(nil) + _ marshal.Marshallable = (*Inet6Addr)(nil) + _ marshal.Marshallable = (*Inet6MulticastRequest)(nil) + _ marshal.Marshallable = (*InetAddr)(nil) + _ marshal.Marshallable = (*InetMulticastRequest)(nil) + _ marshal.Marshallable = (*InetMulticastRequestWithNIC)(nil) + _ marshal.Marshallable = (*InterfaceAddrMessage)(nil) + _ marshal.Marshallable = (*InterfaceInfoMessage)(nil) + _ marshal.Marshallable = (*ItimerVal)(nil) + _ marshal.Marshallable = (*Itimerspec)(nil) + _ marshal.Marshallable = (*KernelIP6TEntry)(nil) + _ marshal.Marshallable = (*KernelIP6TGetEntries)(nil) + _ marshal.Marshallable = (*KernelIPTEntry)(nil) + _ marshal.Marshallable = (*KernelIPTGetEntries)(nil) + _ marshal.Marshallable = (*Linger)(nil) + _ marshal.Marshallable = (*MqAttr)(nil) + _ marshal.Marshallable = (*MsgBuf)(nil) + _ marshal.Marshallable = (*MsgInfo)(nil) + _ marshal.Marshallable = (*MsqidDS)(nil) + _ marshal.Marshallable = (*NFNATRange)(nil) + _ marshal.Marshallable = (*NFNATRange2)(nil) + _ marshal.Marshallable = (*NetFilterGenMsg)(nil) + _ marshal.Marshallable = (*NetlinkAttrHeader)(nil) + _ marshal.Marshallable = (*NetlinkErrorMessage)(nil) + _ marshal.Marshallable = (*NetlinkMessageHeader)(nil) + _ marshal.Marshallable = (*NfNATIPV4MultiRangeCompat)(nil) + _ marshal.Marshallable = (*NfNATIPV4Range)(nil) + _ marshal.Marshallable = (*NumaPolicy)(nil) + _ marshal.Marshallable = (*PollFD)(nil) + _ marshal.Marshallable = (*RSeqCriticalSection)(nil) + _ marshal.Marshallable = (*RobustListHead)(nil) + _ marshal.Marshallable = (*RouteMessage)(nil) + _ marshal.Marshallable = (*RtAttr)(nil) + _ marshal.Marshallable = (*Rusage)(nil) + _ marshal.Marshallable = (*SeccompData)(nil) + _ marshal.Marshallable = (*SeccompNotif)(nil) + _ marshal.Marshallable = (*SeccompNotifResp)(nil) + _ marshal.Marshallable = (*SeccompNotifSizes)(nil) + _ marshal.Marshallable = (*SemInfo)(nil) + _ marshal.Marshallable = (*Sembuf)(nil) + _ marshal.Marshallable = (*ShmInfo)(nil) + _ marshal.Marshallable = (*ShmParams)(nil) + _ marshal.Marshallable = (*ShmidDS)(nil) + _ marshal.Marshallable = (*SigAction)(nil) + _ marshal.Marshallable = (*Sigevent)(nil) + _ marshal.Marshallable = (*SignalInfo)(nil) + _ marshal.Marshallable = (*SignalSet)(nil) + _ marshal.Marshallable = (*SignalStack)(nil) + _ marshal.Marshallable = (*SignalfdSiginfo)(nil) + _ marshal.Marshallable = (*SockAddrInet)(nil) + _ marshal.Marshallable = (*SockAddrInet6)(nil) + _ marshal.Marshallable = (*SockAddrLink)(nil) + _ marshal.Marshallable = (*SockAddrNetlink)(nil) + _ marshal.Marshallable = (*SockAddrUnix)(nil) + _ marshal.Marshallable = (*SockErrCMsgIPv4)(nil) + _ marshal.Marshallable = (*SockErrCMsgIPv6)(nil) + _ marshal.Marshallable = (*SockExtendedErr)(nil) + _ marshal.Marshallable = (*Statfs)(nil) + _ marshal.Marshallable = (*Statx)(nil) + _ marshal.Marshallable = (*StatxTimestamp)(nil) + _ marshal.Marshallable = (*Sysinfo)(nil) + _ marshal.Marshallable = (*TCPInfo)(nil) + _ marshal.Marshallable = (*TableName)(nil) + _ marshal.Marshallable = (*Termios)(nil) + _ marshal.Marshallable = (*TimeT)(nil) + _ marshal.Marshallable = (*TimerID)(nil) + _ marshal.Marshallable = (*Timespec)(nil) + _ marshal.Marshallable = (*Timeval)(nil) + _ marshal.Marshallable = (*Tms)(nil) + _ marshal.Marshallable = (*Tpacket2Hdr)(nil) + _ marshal.Marshallable = (*TpacketHdr)(nil) + _ marshal.Marshallable = (*TpacketReq)(nil) + _ marshal.Marshallable = (*TpacketStats)(nil) + _ marshal.Marshallable = (*Utime)(nil) + _ marshal.Marshallable = (*UtsName)(nil) + _ marshal.Marshallable = (*VFIODeviceInfo)(nil) + _ marshal.Marshallable = (*VFIOIommuType1DmaMap)(nil) + _ marshal.Marshallable = (*VFIOIommuType1DmaUnmap)(nil) + _ marshal.Marshallable = (*VFIOIrqInfo)(nil) + _ marshal.Marshallable = (*VFIOIrqSet)(nil) + _ marshal.Marshallable = (*VFIORegionInfo)(nil) + _ marshal.Marshallable = (*VfsCapData)(nil) + _ marshal.Marshallable = (*VfsNsCapData)(nil) + _ marshal.Marshallable = (*WindowSize)(nil) + _ marshal.Marshallable = (*Winsize)(nil) + _ marshal.Marshallable = (*XTCounters)(nil) + _ marshal.Marshallable = (*XTEntryMatch)(nil) + _ marshal.Marshallable = (*XTEntryTarget)(nil) + _ marshal.Marshallable = (*XTErrorTarget)(nil) + _ marshal.Marshallable = (*XTGetRevision)(nil) + _ marshal.Marshallable = (*XTMultiport)(nil) + _ marshal.Marshallable = (*XTMultiportV1)(nil) + _ marshal.Marshallable = (*XTNATTargetV0)(nil) + _ marshal.Marshallable = (*XTNATTargetV1)(nil) + _ marshal.Marshallable = (*XTNATTargetV2)(nil) + _ marshal.Marshallable = (*XTOwnerMatchInfo)(nil) + _ marshal.Marshallable = (*XTRedirectTarget)(nil) + _ marshal.Marshallable = (*XTStandardTarget)(nil) + _ marshal.Marshallable = (*XTTCP)(nil) + _ marshal.Marshallable = (*XTUDP)(nil) +) + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOCallback) SizeBytes() int { + return 64 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOCallback) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Data)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Key)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.OpCode)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.ReqPrio)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Buf)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Bytes)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Reserved2)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.ResFD)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOCallback) UnmarshalBytes(src []byte) []byte { + i.Data = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Key = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + i.OpCode = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.ReqPrio = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Buf = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Bytes = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Offset = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Reserved2 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.ResFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOCallback) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOCallback) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOCallback) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOCallback) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOCallback) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOCallback) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOCallback) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOCallback) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOEvent) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOEvent) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Data)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Obj)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Result)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Result2)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOEvent) UnmarshalBytes(src []byte) []byte { + i.Data = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Obj = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Result = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Result2 = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOEvent) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOEvent) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOEvent) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOEvent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOEvent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOEvent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOEvent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOEvent) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (b *BPFInstruction) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (b *BPFInstruction) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(b.OpCode)) + dst = dst[2:] + dst[0] = byte(b.JumpIfTrue) + dst = dst[1:] + dst[0] = byte(b.JumpIfFalse) + dst = dst[1:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(b.K)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (b *BPFInstruction) UnmarshalBytes(src []byte) []byte { + b.OpCode = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + b.JumpIfTrue = uint8(src[0]) + src = src[1:] + b.JumpIfFalse = uint8(src[0]) + src = src[1:] + b.K = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (b *BPFInstruction) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (b *BPFInstruction) MarshalUnsafe(dst []byte) []byte { + size := b.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(b), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (b *BPFInstruction) UnmarshalUnsafe(src []byte) []byte { + size := b.SizeBytes() + gohacks.Memmove(unsafe.Pointer(b), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (b *BPFInstruction) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(b))) + hdr.Len = b.SizeBytes() + hdr.Cap = b.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that b + // must live until the use above. + runtime.KeepAlive(b) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (b *BPFInstruction) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return b.CopyOutN(cc, addr, b.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (b *BPFInstruction) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(b))) + hdr.Len = b.SizeBytes() + hdr.Cap = b.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that b + // must live until the use above. + runtime.KeepAlive(b) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (b *BPFInstruction) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return b.CopyInN(cc, addr, b.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (b *BPFInstruction) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(b))) + hdr.Len = b.SizeBytes() + hdr.Cap = b.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that b + // must live until the use above. + runtime.KeepAlive(b) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyBPFInstructionSliceIn copies in a slice of BPFInstruction objects from the task's memory. +func CopyBPFInstructionSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []BPFInstruction) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*BPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyBPFInstructionSliceOut copies a slice of BPFInstruction objects to the task's memory. +func CopyBPFInstructionSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []BPFInstruction) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*BPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeBPFInstructionSlice is like BPFInstruction.MarshalUnsafe, but for a []BPFInstruction. +func MarshalUnsafeBPFInstructionSlice(src []BPFInstruction, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*BPFInstruction)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeBPFInstructionSlice is like BPFInstruction.UnmarshalUnsafe, but for a []BPFInstruction. +func UnmarshalUnsafeBPFInstructionSlice(dst []BPFInstruction, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*BPFInstruction)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *CapUserData) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *CapUserData) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Effective)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Permitted)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Inheritable)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *CapUserData) UnmarshalBytes(src []byte) []byte { + c.Effective = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.Permitted = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.Inheritable = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *CapUserData) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *CapUserData) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *CapUserData) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *CapUserData) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *CapUserData) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *CapUserData) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *CapUserData) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *CapUserData) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyCapUserDataSliceIn copies in a slice of CapUserData objects from the task's memory. +func CopyCapUserDataSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []CapUserData) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*CapUserData)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyCapUserDataSliceOut copies a slice of CapUserData objects to the task's memory. +func CopyCapUserDataSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []CapUserData) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*CapUserData)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeCapUserDataSlice is like CapUserData.MarshalUnsafe, but for a []CapUserData. +func MarshalUnsafeCapUserDataSlice(src []CapUserData, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*CapUserData)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeCapUserDataSlice is like CapUserData.UnmarshalUnsafe, but for a []CapUserData. +func UnmarshalUnsafeCapUserDataSlice(dst []CapUserData, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*CapUserData)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *CapUserHeader) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *CapUserHeader) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Version)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Pid)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *CapUserHeader) UnmarshalBytes(src []byte) []byte { + c.Version = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.Pid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *CapUserHeader) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *CapUserHeader) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *CapUserHeader) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *CapUserHeader) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *CapUserHeader) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *CapUserHeader) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *CapUserHeader) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *CapUserHeader) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *VfsCapData) SizeBytes() int { + return 20 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *VfsCapData) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.MagicEtc)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.PermittedLo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.InheritableLo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.PermittedHi)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.InheritableHi)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *VfsCapData) UnmarshalBytes(src []byte) []byte { + c.MagicEtc = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.PermittedLo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.InheritableLo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.PermittedHi = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.InheritableHi = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *VfsCapData) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *VfsCapData) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *VfsCapData) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *VfsCapData) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *VfsCapData) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *VfsCapData) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *VfsCapData) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *VfsCapData) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *VfsNsCapData) SizeBytes() int { + return 4 + + (*VfsCapData)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *VfsNsCapData) MarshalBytes(dst []byte) []byte { + dst = c.VfsCapData.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.RootID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *VfsNsCapData) UnmarshalBytes(src []byte) []byte { + src = c.VfsCapData.UnmarshalUnsafe(src) + c.RootID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *VfsNsCapData) Packed() bool { + return c.VfsCapData.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *VfsNsCapData) MarshalUnsafe(dst []byte) []byte { + if c.VfsCapData.Packed() { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] + } + // Type VfsNsCapData doesn't have a packed layout in memory, fallback to MarshalBytes. + return c.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *VfsNsCapData) UnmarshalUnsafe(src []byte) []byte { + if c.VfsCapData.Packed() { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type VfsNsCapData doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return c.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *VfsNsCapData) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !c.VfsCapData.Packed() { + // Type VfsNsCapData doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(c.SizeBytes()) // escapes: okay. + c.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *VfsNsCapData) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *VfsNsCapData) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !c.VfsCapData.Packed() { + // Type VfsNsCapData doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(c.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + c.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *VfsNsCapData) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *VfsNsCapData) WriteTo(writer io.Writer) (int64, error) { + if !c.VfsCapData.Packed() { + // Type VfsNsCapData doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, c.SizeBytes()) + c.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *CloneArgs) SizeBytes() int { + return 88 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *CloneArgs) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.Flags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.Pidfd)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.ChildTID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.ParentTID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.ExitSignal)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.Stack)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.StackSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.TLS)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.SetTID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.SetTIDSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.Cgroup)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *CloneArgs) UnmarshalBytes(src []byte) []byte { + c.Flags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.Pidfd = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.ChildTID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.ParentTID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.ExitSignal = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.Stack = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.StackSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.TLS = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.SetTID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.SetTIDSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.Cgroup = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *CloneArgs) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *CloneArgs) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *CloneArgs) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *CloneArgs) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *CloneArgs) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *CloneArgs) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *CloneArgs) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *CloneArgs) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *ElfHeader64) SizeBytes() int { + return 48 + + 1*16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *ElfHeader64) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < 16; idx++ { + dst[0] = byte(e.Ident[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Type)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Machine)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Version)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Entry)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Phoff)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Shoff)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Ehsize)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Phentsize)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Phnum)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Shentsize)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Shnum)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Shstrndx)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *ElfHeader64) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < 16; idx++ { + e.Ident[idx] = src[0] + src = src[1:] + } + e.Type = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Machine = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Version = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Entry = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Phoff = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Shoff = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Ehsize = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Phentsize = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Phnum = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Shentsize = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Shnum = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Shstrndx = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *ElfHeader64) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *ElfHeader64) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *ElfHeader64) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *ElfHeader64) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *ElfHeader64) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *ElfHeader64) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *ElfHeader64) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *ElfHeader64) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *ElfProg64) SizeBytes() int { + return 56 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *ElfProg64) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Type)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Off)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Vaddr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Paddr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Filesz)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Memsz)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Align)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *ElfProg64) UnmarshalBytes(src []byte) []byte { + e.Type = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Off = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Vaddr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Paddr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Filesz = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Memsz = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Align = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *ElfProg64) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *ElfProg64) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *ElfProg64) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *ElfProg64) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *ElfProg64) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *ElfProg64) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *ElfProg64) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *ElfProg64) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *ElfSection64) SizeBytes() int { + return 64 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *ElfSection64) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Name)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Type)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Flags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Addr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Off)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Link)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Info)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Addralign)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(e.Entsize)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *ElfSection64) UnmarshalBytes(src []byte) []byte { + e.Name = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Type = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Flags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Addr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Off = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Link = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Info = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Addralign = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + e.Entsize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *ElfSection64) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *ElfSection64) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *ElfSection64) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *ElfSection64) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *ElfSection64) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *ElfSection64) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *ElfSection64) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *ElfSection64) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockErrCMsgIPv4) SizeBytes() int { + return 0 + + (*SockExtendedErr)(nil).SizeBytes() + + (*SockAddrInet)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockErrCMsgIPv4) MarshalBytes(dst []byte) []byte { + dst = s.SockExtendedErr.MarshalUnsafe(dst) + dst = s.Offender.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockErrCMsgIPv4) UnmarshalBytes(src []byte) []byte { + src = s.SockExtendedErr.UnmarshalUnsafe(src) + src = s.Offender.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockErrCMsgIPv4) Packed() bool { + return s.Offender.Packed() && s.SockExtendedErr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockErrCMsgIPv4) MarshalUnsafe(dst []byte) []byte { + if s.Offender.Packed() && s.SockExtendedErr.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SockErrCMsgIPv4 doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockErrCMsgIPv4) UnmarshalUnsafe(src []byte) []byte { + if s.Offender.Packed() && s.SockExtendedErr.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SockErrCMsgIPv4 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockErrCMsgIPv4) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Offender.Packed() && s.SockExtendedErr.Packed() { + // Type SockErrCMsgIPv4 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockErrCMsgIPv4) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockErrCMsgIPv4) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Offender.Packed() && s.SockExtendedErr.Packed() { + // Type SockErrCMsgIPv4 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockErrCMsgIPv4) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockErrCMsgIPv4) WriteTo(writer io.Writer) (int64, error) { + if !s.Offender.Packed() && s.SockExtendedErr.Packed() { + // Type SockErrCMsgIPv4 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockErrCMsgIPv6) SizeBytes() int { + return 0 + + (*SockExtendedErr)(nil).SizeBytes() + + (*SockAddrInet6)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockErrCMsgIPv6) MarshalBytes(dst []byte) []byte { + dst = s.SockExtendedErr.MarshalUnsafe(dst) + dst = s.Offender.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockErrCMsgIPv6) UnmarshalBytes(src []byte) []byte { + src = s.SockExtendedErr.UnmarshalUnsafe(src) + src = s.Offender.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockErrCMsgIPv6) Packed() bool { + return s.Offender.Packed() && s.SockExtendedErr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockErrCMsgIPv6) MarshalUnsafe(dst []byte) []byte { + if s.Offender.Packed() && s.SockExtendedErr.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SockErrCMsgIPv6 doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockErrCMsgIPv6) UnmarshalUnsafe(src []byte) []byte { + if s.Offender.Packed() && s.SockExtendedErr.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SockErrCMsgIPv6 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockErrCMsgIPv6) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Offender.Packed() && s.SockExtendedErr.Packed() { + // Type SockErrCMsgIPv6 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockErrCMsgIPv6) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockErrCMsgIPv6) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Offender.Packed() && s.SockExtendedErr.Packed() { + // Type SockErrCMsgIPv6 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockErrCMsgIPv6) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockErrCMsgIPv6) WriteTo(writer io.Writer) (int64, error) { + if !s.Offender.Packed() && s.SockExtendedErr.Packed() { + // Type SockErrCMsgIPv6 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockExtendedErr) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockExtendedErr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Errno)) + dst = dst[4:] + dst[0] = byte(s.Origin) + dst = dst[1:] + dst[0] = byte(s.Type) + dst = dst[1:] + dst[0] = byte(s.Code) + dst = dst[1:] + dst[0] = byte(s.Pad) + dst = dst[1:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Info)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Data)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockExtendedErr) UnmarshalBytes(src []byte) []byte { + s.Errno = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Origin = uint8(src[0]) + src = src[1:] + s.Type = uint8(src[0]) + src = src[1:] + s.Code = uint8(src[0]) + src = src[1:] + s.Pad = uint8(src[0]) + src = src[1:] + s.Info = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Data = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockExtendedErr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockExtendedErr) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockExtendedErr) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockExtendedErr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockExtendedErr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockExtendedErr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockExtendedErr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockExtendedErr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FOwnerEx) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FOwnerEx) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Type)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.PID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FOwnerEx) UnmarshalBytes(src []byte) []byte { + f.Type = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.PID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FOwnerEx) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FOwnerEx) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FOwnerEx) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FOwnerEx) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FOwnerEx) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FOwnerEx) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FOwnerEx) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FOwnerEx) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *Flock) SizeBytes() int { + return 24 + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *Flock) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(f.Type)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(f.Whence)) + dst = dst[2:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Start)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Len)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.PID)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *Flock) UnmarshalBytes(src []byte) []byte { + f.Type = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + f.Whence = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([4]byte(f._), src[:sizeof(byte)*4]) + src = src[1*(4):] + f.Start = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Len = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.PID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(f._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *Flock) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *Flock) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *Flock) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *Flock) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *Flock) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *Flock) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *Flock) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *Flock) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (m *FileMode) SizeBytes() int { + return 2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (m *FileMode) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(*m)) + return dst[2:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (m *FileMode) UnmarshalBytes(src []byte) []byte { + *m = FileMode(uint16(hostarch.ByteOrder.Uint16(src[:2]))) + return src[2:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (m *FileMode) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (m *FileMode) MarshalUnsafe(dst []byte) []byte { + size := m.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(m), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (m *FileMode) UnmarshalUnsafe(src []byte) []byte { + size := m.SizeBytes() + gohacks.Memmove(unsafe.Pointer(m), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (m *FileMode) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (m *FileMode) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyOutN(cc, addr, m.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (m *FileMode) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (m *FileMode) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyInN(cc, addr, m.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (m *FileMode) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Statx) SizeBytes() int { + return 80 + + (*StatxTimestamp)(nil).SizeBytes() + + (*StatxTimestamp)(nil).SizeBytes() + + (*StatxTimestamp)(nil).SizeBytes() + + (*StatxTimestamp)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Statx) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Mask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Blksize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Attributes)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Nlink)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.GID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Mode)) + dst = dst[2:] + // Padding: dst[:sizeof(uint16)] ~= uint16(0) + dst = dst[2:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Ino)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Blocks)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.AttributesMask)) + dst = dst[8:] + dst = s.Atime.MarshalUnsafe(dst) + dst = s.Btime.MarshalUnsafe(dst) + dst = s.Ctime.MarshalUnsafe(dst) + dst = s.Mtime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.RdevMajor)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.RdevMinor)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.DevMajor)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.DevMinor)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Statx) UnmarshalBytes(src []byte) []byte { + s.Mask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Blksize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Attributes = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Nlink = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Mode = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: var _ uint16 ~= src[:sizeof(uint16)] + src = src[2:] + s.Ino = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Blocks = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.AttributesMask = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = s.Atime.UnmarshalUnsafe(src) + src = s.Btime.UnmarshalUnsafe(src) + src = s.Ctime.UnmarshalUnsafe(src) + src = s.Mtime.UnmarshalUnsafe(src) + s.RdevMajor = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.RdevMinor = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.DevMajor = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.DevMinor = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Statx) Packed() bool { + return s.Atime.Packed() && s.Btime.Packed() && s.Ctime.Packed() && s.Mtime.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Statx) MarshalUnsafe(dst []byte) []byte { + if s.Atime.Packed() && s.Btime.Packed() && s.Ctime.Packed() && s.Mtime.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type Statx doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Statx) UnmarshalUnsafe(src []byte) []byte { + if s.Atime.Packed() && s.Btime.Packed() && s.Ctime.Packed() && s.Mtime.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Statx doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Statx) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Atime.Packed() && s.Btime.Packed() && s.Ctime.Packed() && s.Mtime.Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Statx) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Statx) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Atime.Packed() && s.Btime.Packed() && s.Ctime.Packed() && s.Mtime.Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Statx) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Statx) WriteTo(writer io.Writer) (int64, error) { + if !s.Atime.Packed() && s.Btime.Packed() && s.Ctime.Packed() && s.Mtime.Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (s *Statx) CheckedMarshal(dst []byte) ([]byte, bool) { + if s.SizeBytes() > len(dst) { + return dst, false + } + return s.MarshalUnsafe(dst), true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (s *Statx) CheckedUnmarshal(src []byte) ([]byte, bool) { + if s.SizeBytes() > len(src) { + return src, false + } + return s.UnmarshalUnsafe(src), true +} + +// CopyStatxSliceIn copies in a slice of Statx objects from the task's memory. +func CopyStatxSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []Statx) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Statx)(nil).SizeBytes() + + if !dst[0].Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(size * count) + length, err := cc.CopyInBytes(addr, buf) + + // Unmarshal as much as possible, even on error. First handle full objects. + limit := length / size + for idx := 0; idx < limit; idx++ { + buf = dst[idx].UnmarshalBytes(buf) + } + + // Handle any final partial object. buf is guaranteed to be long enough for the + // final element, but may not contain valid data for the entire range. This may + // result in unmarshalling zero values for some parts of the object. + if length%size != 0 { + dst[limit].UnmarshalBytes(buf) + } + + return length, err + } + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyStatxSliceOut copies a slice of Statx objects to the task's memory. +func CopyStatxSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []Statx) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Statx)(nil).SizeBytes() + + if !src[0].Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(size * count) + curBuf := buf + for idx := 0; idx < count; idx++ { + curBuf = src[idx].MarshalBytes(curBuf) + } + return cc.CopyOutBytes(addr, buf) + } + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeStatxSlice is like Statx.MarshalUnsafe, but for a []Statx. +func MarshalUnsafeStatxSlice(src []Statx, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + if !src[0].Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to MarshalBytes. + for idx := 0; idx < count; idx++ { + dst = src[idx].MarshalBytes(dst) + } + return dst + } + + size := (*Statx)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeStatxSlice is like Statx.UnmarshalUnsafe, but for a []Statx. +func UnmarshalUnsafeStatxSlice(dst []Statx, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + if !dst[0].Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to UnmarshalBytes. + for idx := 0; idx < count; idx++ { + src = dst[idx].UnmarshalBytes(src) + } + return src + } + + size := (*Statx)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Statfs) SizeBytes() int { + return 80 + + 4*2 + + 8*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Statfs) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Type)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.BlockSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Blocks)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.BlocksFree)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.BlocksAvailable)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Files)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.FilesFree)) + dst = dst[8:] + for idx := 0; idx < 2; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.FSID[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.NameLength)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.FragmentSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Flags)) + dst = dst[8:] + for idx := 0; idx < 4; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Spare[idx])) + dst = dst[8:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Statfs) UnmarshalBytes(src []byte) []byte { + s.Type = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.BlockSize = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Blocks = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.BlocksFree = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.BlocksAvailable = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Files = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.FilesFree = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < 2; idx++ { + s.FSID[idx] = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + s.NameLength = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.FragmentSize = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Flags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < 4; idx++ { + s.Spare[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Statfs) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Statfs) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Statfs) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Statfs) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Statfs) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Statfs) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Statfs) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Statfs) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *CString) Packed() bool { + // Type CString is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *CString) MarshalUnsafe(dst []byte) []byte { + // Type CString doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *CString) UnmarshalUnsafe(src []byte) []byte { + // Type CString doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (s *CString) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type CString doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *CString) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (s *CString) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type CString doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *CString) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *CString) WriteTo(writer io.Writer) (int64, error) { + // Type CString doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEAccessIn) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEAccessIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Mask)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEAccessIn) UnmarshalBytes(src []byte) []byte { + f.Mask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEAccessIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEAccessIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEAccessIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEAccessIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEAccessIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEAccessIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEAccessIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEAccessIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (a *FUSEAttr) SizeBytes() int { + return 88 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (a *FUSEAttr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Ino)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Blocks)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Atime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Mtime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Ctime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AtimeNsec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.MtimeNsec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.CtimeNsec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Mode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Nlink)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.GID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Rdev)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.BlkSize)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (a *FUSEAttr) UnmarshalBytes(src []byte) []byte { + a.Ino = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Blocks = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Atime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Mtime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Ctime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.AtimeNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.MtimeNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.CtimeNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.Nlink = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.Rdev = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.BlkSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (a *FUSEAttr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (a *FUSEAttr) MarshalUnsafe(dst []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(a), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (a *FUSEAttr) UnmarshalUnsafe(src []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(a), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (a *FUSEAttr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (a *FUSEAttr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyOutN(cc, addr, a.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (a *FUSEAttr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (a *FUSEAttr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyInN(cc, addr, a.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (a *FUSEAttr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEAttrOut) SizeBytes() int { + return 16 + + (*FUSEAttr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEAttrOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.AttrValid)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.AttrValidNsec)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + dst = f.Attr.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEAttrOut) UnmarshalBytes(src []byte) []byte { + f.AttrValid = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.AttrValidNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + src = f.Attr.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEAttrOut) Packed() bool { + return f.Attr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEAttrOut) MarshalUnsafe(dst []byte) []byte { + if f.Attr.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] + } + // Type FUSEAttrOut doesn't have a packed layout in memory, fallback to MarshalBytes. + return f.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEAttrOut) UnmarshalUnsafe(src []byte) []byte { + if f.Attr.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type FUSEAttrOut doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return f.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEAttrOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Attr.Packed() { + // Type FUSEAttrOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + f.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEAttrOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEAttrOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Attr.Packed() { + // Type FUSEAttrOut doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + f.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEAttrOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEAttrOut) WriteTo(writer io.Writer) (int64, error) { + if !f.Attr.Packed() { + // Type FUSEAttrOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, f.SizeBytes()) + f.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSECreateIn) Packed() bool { + // Type FUSECreateIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSECreateIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSECreateIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSECreateIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSECreateIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSECreateIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSECreateIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSECreateIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSECreateIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSECreateIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSECreateIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSECreateIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSECreateIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSECreateMeta) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSECreateMeta) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Mode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Umask)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSECreateMeta) UnmarshalBytes(src []byte) []byte { + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Umask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSECreateMeta) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSECreateMeta) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSECreateMeta) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSECreateMeta) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSECreateMeta) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSECreateMeta) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSECreateMeta) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSECreateMeta) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSECreateOut) SizeBytes() int { + return 0 + + (*FUSEEntryOut)(nil).SizeBytes() + + (*FUSEOpenOut)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSECreateOut) MarshalBytes(dst []byte) []byte { + dst = f.FUSEEntryOut.MarshalUnsafe(dst) + dst = f.FUSEOpenOut.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSECreateOut) UnmarshalBytes(src []byte) []byte { + src = f.FUSEEntryOut.UnmarshalUnsafe(src) + src = f.FUSEOpenOut.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSECreateOut) Packed() bool { + return f.FUSEEntryOut.Packed() && f.FUSEOpenOut.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSECreateOut) MarshalUnsafe(dst []byte) []byte { + if f.FUSEEntryOut.Packed() && f.FUSEOpenOut.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] + } + // Type FUSECreateOut doesn't have a packed layout in memory, fallback to MarshalBytes. + return f.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSECreateOut) UnmarshalUnsafe(src []byte) []byte { + if f.FUSEEntryOut.Packed() && f.FUSEOpenOut.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type FUSECreateOut doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return f.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSECreateOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.FUSEEntryOut.Packed() && f.FUSEOpenOut.Packed() { + // Type FUSECreateOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + f.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSECreateOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSECreateOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.FUSEEntryOut.Packed() && f.FUSEOpenOut.Packed() { + // Type FUSECreateOut doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + f.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSECreateOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSECreateOut) WriteTo(writer io.Writer) (int64, error) { + if !f.FUSEEntryOut.Packed() && f.FUSEOpenOut.Packed() { + // Type FUSECreateOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, f.SizeBytes()) + f.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEDirent) Packed() bool { + // Type FUSEDirent is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEDirent) MarshalUnsafe(dst []byte) []byte { + // Type FUSEDirent doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEDirent) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEDirent doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEDirent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEDirent doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEDirent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEDirent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEDirent doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEDirent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEDirent) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEDirent doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEDirentMeta) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEDirentMeta) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Ino)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Off)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.NameLen)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Type)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEDirentMeta) UnmarshalBytes(src []byte) []byte { + f.Ino = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Off = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.NameLen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Type = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEDirentMeta) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEDirentMeta) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEDirentMeta) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEDirentMeta) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEDirentMeta) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEDirentMeta) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEDirentMeta) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEDirentMeta) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEDirents) Packed() bool { + // Type FUSEDirents is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEDirents) MarshalUnsafe(dst []byte) []byte { + // Type FUSEDirents doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEDirents) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEDirents doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEDirents) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEDirents doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEDirents) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEDirents) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEDirents doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEDirents) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEDirents) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEDirents doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEEmptyIn) Packed() bool { + // Type FUSEEmptyIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEEmptyIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSEEmptyIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEEmptyIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEEmptyIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEEmptyIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEEmptyIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEEmptyIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEEmptyIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEEmptyIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEEmptyIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEEmptyIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEEmptyIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEEntryOut) SizeBytes() int { + return 40 + + (*FUSEAttr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEEntryOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.NodeID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Generation)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.EntryValid)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.AttrValid)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.EntryValidNSec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.AttrValidNSec)) + dst = dst[4:] + dst = f.Attr.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEEntryOut) UnmarshalBytes(src []byte) []byte { + f.NodeID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Generation = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.EntryValid = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.AttrValid = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.EntryValidNSec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.AttrValidNSec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = f.Attr.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEEntryOut) Packed() bool { + return f.Attr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEEntryOut) MarshalUnsafe(dst []byte) []byte { + if f.Attr.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] + } + // Type FUSEEntryOut doesn't have a packed layout in memory, fallback to MarshalBytes. + return f.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEEntryOut) UnmarshalUnsafe(src []byte) []byte { + if f.Attr.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type FUSEEntryOut doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return f.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEEntryOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Attr.Packed() { + // Type FUSEEntryOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + f.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEEntryOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEEntryOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Attr.Packed() { + // Type FUSEEntryOut doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + f.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEEntryOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEEntryOut) WriteTo(writer io.Writer) (int64, error) { + if !f.Attr.Packed() { + // Type FUSEEntryOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, f.SizeBytes()) + f.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEFallocateIn) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEFallocateIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Mode)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEFallocateIn) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEFallocateIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEFallocateIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEFallocateIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEFallocateIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEFallocateIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEFallocateIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEFallocateIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEFallocateIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEFlushIn) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEFlushIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.LockOwner)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEFlushIn) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + f.LockOwner = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEFlushIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEFlushIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEFlushIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEFlushIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEFlushIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEFlushIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEFlushIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEFlushIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEFsyncIn) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEFsyncIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.FsyncFlags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEFsyncIn) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.FsyncFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEFsyncIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEFsyncIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEFsyncIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEFsyncIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEFsyncIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEFsyncIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEFsyncIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEFsyncIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEGetAttrIn) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEGetAttrIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.GetAttrFlags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEGetAttrIn) UnmarshalBytes(src []byte) []byte { + f.GetAttrFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEGetAttrIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEGetAttrIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEGetAttrIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEGetAttrIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEGetAttrIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEGetAttrIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEGetAttrIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEGetAttrIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEHeaderIn) SizeBytes() int { + return 28 + + (*FUSEOpcode)(nil).SizeBytes() + + (*FUSEOpID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEHeaderIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Len)) + dst = dst[4:] + dst = f.Opcode.MarshalUnsafe(dst) + dst = f.Unique.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.NodeID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.GID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.PID)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEHeaderIn) UnmarshalBytes(src []byte) []byte { + f.Len = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = f.Opcode.UnmarshalUnsafe(src) + src = f.Unique.UnmarshalUnsafe(src) + f.NodeID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.PID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEHeaderIn) Packed() bool { + return f.Opcode.Packed() && f.Unique.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEHeaderIn) MarshalUnsafe(dst []byte) []byte { + if f.Opcode.Packed() && f.Unique.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] + } + // Type FUSEHeaderIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return f.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEHeaderIn) UnmarshalUnsafe(src []byte) []byte { + if f.Opcode.Packed() && f.Unique.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type FUSEHeaderIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return f.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEHeaderIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Opcode.Packed() && f.Unique.Packed() { + // Type FUSEHeaderIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + f.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEHeaderIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEHeaderIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Opcode.Packed() && f.Unique.Packed() { + // Type FUSEHeaderIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + f.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEHeaderIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEHeaderIn) WriteTo(writer io.Writer) (int64, error) { + if !f.Opcode.Packed() && f.Unique.Packed() { + // Type FUSEHeaderIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, f.SizeBytes()) + f.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEHeaderOut) SizeBytes() int { + return 8 + + (*FUSEOpID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEHeaderOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Len)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Error)) + dst = dst[4:] + dst = f.Unique.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEHeaderOut) UnmarshalBytes(src []byte) []byte { + f.Len = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Error = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = f.Unique.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEHeaderOut) Packed() bool { + return f.Unique.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEHeaderOut) MarshalUnsafe(dst []byte) []byte { + if f.Unique.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] + } + // Type FUSEHeaderOut doesn't have a packed layout in memory, fallback to MarshalBytes. + return f.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEHeaderOut) UnmarshalUnsafe(src []byte) []byte { + if f.Unique.Packed() { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type FUSEHeaderOut doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return f.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEHeaderOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Unique.Packed() { + // Type FUSEHeaderOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + f.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEHeaderOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEHeaderOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !f.Unique.Packed() { + // Type FUSEHeaderOut doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(f.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + f.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEHeaderOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEHeaderOut) WriteTo(writer io.Writer) (int64, error) { + if !f.Unique.Packed() { + // Type FUSEHeaderOut doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, f.SizeBytes()) + f.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEInitIn) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEInitIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Major)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Minor)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.MaxReadahead)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEInitIn) UnmarshalBytes(src []byte) []byte { + f.Major = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Minor = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.MaxReadahead = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEInitIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEInitIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEInitIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEInitIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEInitIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEInitIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEInitIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEInitIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEInitOut) SizeBytes() int { + return 32 + + 4*8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEInitOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Major)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Minor)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.MaxReadahead)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(f.MaxBackground)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(f.CongestionThreshold)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.MaxWrite)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.TimeGran)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(f.MaxPages)) + dst = dst[2:] + // Padding: dst[:sizeof(uint16)] ~= uint16(0) + dst = dst[2:] + // Padding: dst[:sizeof(uint32)*8] ~= [8]uint32{0} + dst = dst[4*(8):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEInitOut) UnmarshalBytes(src []byte) []byte { + f.Major = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Minor = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.MaxReadahead = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.MaxBackground = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + f.CongestionThreshold = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + f.MaxWrite = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.TimeGran = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.MaxPages = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: var _ uint16 ~= src[:sizeof(uint16)] + src = src[2:] + // Padding: ~ copy([8]uint32(f._), src[:sizeof(uint32)*8]) + src = src[4*(8):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEInitOut) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEInitOut) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEInitOut) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEInitOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEInitOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEInitOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEInitOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEInitOut) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSELinkIn) Packed() bool { + // Type FUSELinkIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSELinkIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSELinkIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSELinkIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSELinkIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSELinkIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSELinkIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSELinkIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSELinkIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSELinkIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSELinkIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSELinkIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSELinkIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSELookupIn) Packed() bool { + // Type FUSELookupIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSELookupIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSELookupIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSELookupIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSELookupIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSELookupIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSELookupIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSELookupIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSELookupIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSELookupIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSELookupIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSELookupIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSELookupIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEMkdirIn) Packed() bool { + // Type FUSEMkdirIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEMkdirIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSEMkdirIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEMkdirIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEMkdirIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEMkdirIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEMkdirIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEMkdirIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEMkdirIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEMkdirIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEMkdirIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEMkdirIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEMkdirIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEMkdirMeta) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEMkdirMeta) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Mode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Umask)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEMkdirMeta) UnmarshalBytes(src []byte) []byte { + f.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Umask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEMkdirMeta) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEMkdirMeta) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEMkdirMeta) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEMkdirMeta) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEMkdirMeta) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEMkdirMeta) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEMkdirMeta) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEMkdirMeta) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEMknodIn) Packed() bool { + // Type FUSEMknodIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEMknodIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSEMknodIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEMknodIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEMknodIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEMknodIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEMknodIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEMknodIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEMknodIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEMknodIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEMknodIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEMknodIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEMknodIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEMknodMeta) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEMknodMeta) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Mode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Rdev)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Umask)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEMknodMeta) UnmarshalBytes(src []byte) []byte { + f.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Rdev = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Umask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEMknodMeta) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEMknodMeta) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEMknodMeta) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEMknodMeta) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEMknodMeta) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEMknodMeta) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEMknodMeta) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEMknodMeta) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (f *FUSEOpID) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEOpID) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*f)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEOpID) UnmarshalBytes(src []byte) []byte { + *f = FUSEOpID(uint64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEOpID) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEOpID) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEOpID) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEOpID) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEOpID) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEOpID) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEOpID) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEOpID) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (f *FUSEOpcode) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEOpcode) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*f)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEOpcode) UnmarshalBytes(src []byte) []byte { + *f = FUSEOpcode(uint32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEOpcode) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEOpcode) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEOpcode) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEOpcode) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEOpcode) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEOpcode) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEOpcode) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEOpcode) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEOpenIn) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEOpenIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEOpenIn) UnmarshalBytes(src []byte) []byte { + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEOpenIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEOpenIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEOpenIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEOpenIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEOpenIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEOpenIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEOpenIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEOpenIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEOpenOut) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEOpenOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.OpenFlag)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEOpenOut) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.OpenFlag = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEOpenOut) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEOpenOut) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEOpenOut) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEOpenOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEOpenOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEOpenOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEOpenOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEOpenOut) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEReadIn) SizeBytes() int { + return 40 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEReadIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.ReadFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.LockOwner)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEReadIn) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.ReadFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.LockOwner = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEReadIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEReadIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEReadIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEReadIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEReadIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEReadIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEReadIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEReadIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEReleaseIn) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEReleaseIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.ReleaseFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.LockOwner)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEReleaseIn) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.ReleaseFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.LockOwner = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEReleaseIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEReleaseIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEReleaseIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEReleaseIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEReleaseIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEReleaseIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEReleaseIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEReleaseIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSERenameIn) Packed() bool { + // Type FUSERenameIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSERenameIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSERenameIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSERenameIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSERenameIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSERenameIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSERenameIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSERenameIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSERenameIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSERenameIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSERenameIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSERenameIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSERenameIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSERmDirIn) Packed() bool { + // Type FUSERmDirIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSERmDirIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSERmDirIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSERmDirIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSERmDirIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSERmDirIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSERmDirIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSERmDirIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSERmDirIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSERmDirIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSERmDirIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSERmDirIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSERmDirIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSESetAttrIn) SizeBytes() int { + return 88 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSESetAttrIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Valid)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.LockOwner)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Atime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Mtime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Ctime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.AtimeNsec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.MtimeNsec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.CtimeNsec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Mode)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.GID)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSESetAttrIn) UnmarshalBytes(src []byte) []byte { + f.Valid = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.LockOwner = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Atime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Mtime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Ctime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.AtimeNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.MtimeNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.CtimeNsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + f.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSESetAttrIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSESetAttrIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSESetAttrIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSESetAttrIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSESetAttrIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSESetAttrIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSESetAttrIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSESetAttrIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEStatfsOut) SizeBytes() int { + return 56 + + 4*6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEStatfsOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Blocks)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.BlocksFree)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.BlocksAvailable)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Files)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.FilesFree)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.BlockSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.NameLength)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.FragmentSize)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + for idx := 0; idx < 6; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Spare[idx])) + dst = dst[4:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEStatfsOut) UnmarshalBytes(src []byte) []byte { + f.Blocks = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.BlocksFree = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.BlocksAvailable = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Files = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.FilesFree = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.BlockSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.NameLength = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.FragmentSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + for idx := 0; idx < 6; idx++ { + f.Spare[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEStatfsOut) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEStatfsOut) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEStatfsOut) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEStatfsOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEStatfsOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEStatfsOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEStatfsOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEStatfsOut) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSESymlinkIn) Packed() bool { + // Type FUSESymlinkIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSESymlinkIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSESymlinkIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSESymlinkIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSESymlinkIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSESymlinkIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSESymlinkIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSESymlinkIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSESymlinkIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSESymlinkIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSESymlinkIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSESymlinkIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSESymlinkIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEUnlinkIn) Packed() bool { + // Type FUSEUnlinkIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEUnlinkIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSEUnlinkIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEUnlinkIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEUnlinkIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEUnlinkIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEUnlinkIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEUnlinkIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEUnlinkIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEUnlinkIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEUnlinkIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEUnlinkIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEUnlinkIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEWriteIn) SizeBytes() int { + return 40 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEWriteIn) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Fh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.WriteFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(f.LockOwner)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEWriteIn) UnmarshalBytes(src []byte) []byte { + f.Fh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.WriteFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.LockOwner = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEWriteIn) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEWriteIn) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEWriteIn) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEWriteIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEWriteIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEWriteIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEWriteIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEWriteIn) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEWriteOut) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEWriteOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Size)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEWriteOut) UnmarshalBytes(src []byte) []byte { + f.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEWriteOut) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEWriteOut) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEWriteOut) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEWriteOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEWriteOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEWriteOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEWriteOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEWriteOut) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEWritePayloadIn) Packed() bool { + // Type FUSEWritePayloadIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEWritePayloadIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSEWritePayloadIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEWritePayloadIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEWritePayloadIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEWritePayloadIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEWritePayloadIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEWritePayloadIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEWritePayloadIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEWritePayloadIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEWritePayloadIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEWritePayloadIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEWritePayloadIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RobustListHead) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RobustListHead) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.List)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.FutexOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.ListOpPending)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RobustListHead) UnmarshalBytes(src []byte) []byte { + r.List = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.FutexOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.ListOpPending = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RobustListHead) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RobustListHead) MarshalUnsafe(dst []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RobustListHead) UnmarshalUnsafe(src []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RobustListHead) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RobustListHead) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RobustListHead) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RobustListHead) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RobustListHead) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOCqRingOffsets) SizeBytes() int { + return 40 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOCqRingOffsets) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Head)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Tail)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.RingMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.RingEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Overflow)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Cqes)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Resv1)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Resv2)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOCqRingOffsets) UnmarshalBytes(src []byte) []byte { + i.Head = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Tail = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.RingMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.RingEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Overflow = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Cqes = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Resv1 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Resv2 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOCqRingOffsets) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOCqRingOffsets) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOCqRingOffsets) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOCqRingOffsets) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOCqRingOffsets) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOCqRingOffsets) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOCqRingOffsets) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOCqRingOffsets) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *IORingIndex) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IORingIndex) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*i)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IORingIndex) UnmarshalBytes(src []byte) []byte { + *i = IORingIndex(uint32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IORingIndex) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IORingIndex) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IORingIndex) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IORingIndex) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IORingIndex) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IORingIndex) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IORingIndex) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IORingIndex) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IORings) SizeBytes() int { + return 32 + + (*IOUring)(nil).SizeBytes() + + (*IOUring)(nil).SizeBytes() + + 1*32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IORings) MarshalBytes(dst []byte) []byte { + dst = i.Sq.MarshalUnsafe(dst) + dst = i.Cq.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.SqRingMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CqRingMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.SqRingEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CqRingEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.sqDropped)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.sqFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.cqFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CqOverflow)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*32] ~= [32]byte{0} + dst = dst[1*(32):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IORings) UnmarshalBytes(src []byte) []byte { + src = i.Sq.UnmarshalUnsafe(src) + src = i.Cq.UnmarshalUnsafe(src) + i.SqRingMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.CqRingMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.SqRingEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.CqRingEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.sqDropped = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.sqFlags = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.cqFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.CqOverflow = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([32]byte(i._), src[:sizeof(byte)*32]) + src = src[1*(32):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IORings) Packed() bool { + return i.Cq.Packed() && i.Sq.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IORings) MarshalUnsafe(dst []byte) []byte { + if i.Cq.Packed() && i.Sq.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IORings doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IORings) UnmarshalUnsafe(src []byte) []byte { + if i.Cq.Packed() && i.Sq.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IORings doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IORings) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Cq.Packed() && i.Sq.Packed() { + // Type IORings doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IORings) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IORings) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Cq.Packed() && i.Sq.Packed() { + // Type IORings doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IORings) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IORings) WriteTo(writer io.Writer) (int64, error) { + if !i.Cq.Packed() && i.Sq.Packed() { + // Type IORings doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOSqRingOffsets) SizeBytes() int { + return 40 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOSqRingOffsets) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Head)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Tail)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.RingMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.RingEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Dropped)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Array)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Resv1)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Resv2)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOSqRingOffsets) UnmarshalBytes(src []byte) []byte { + i.Head = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Tail = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.RingMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.RingEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Dropped = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Array = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Resv1 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Resv2 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOSqRingOffsets) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOSqRingOffsets) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOSqRingOffsets) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOSqRingOffsets) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOSqRingOffsets) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOSqRingOffsets) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOSqRingOffsets) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOSqRingOffsets) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOUring) SizeBytes() int { + return 8 + + 1*60 + + 1*60 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOUring) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Head)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*60] ~= [60]byte{0} + dst = dst[1*(60):] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Tail)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*60] ~= [60]byte{0} + dst = dst[1*(60):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOUring) UnmarshalBytes(src []byte) []byte { + i.Head = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([60]byte(i._), src[:sizeof(byte)*60]) + src = src[1*(60):] + i.Tail = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([60]byte(i._), src[:sizeof(byte)*60]) + src = src[1*(60):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOUring) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOUring) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOUring) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOUring) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOUring) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOUring) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOUring) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOUring) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOUringCqe) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOUringCqe) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.UserData)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Res)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOUringCqe) UnmarshalBytes(src []byte) []byte { + i.UserData = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Res = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOUringCqe) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOUringCqe) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOUringCqe) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOUringCqe) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOUringCqe) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOUringCqe) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOUringCqe) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOUringCqe) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOUringParams) SizeBytes() int { + return 28 + + 4*3 + + (*IOSqRingOffsets)(nil).SizeBytes() + + (*IOCqRingOffsets)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOUringParams) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.SqEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CqEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.SqThreadCPU)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.SqThreadIdle)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Features)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.WqFd)) + dst = dst[4:] + for idx := 0; idx < 3; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Resv[idx])) + dst = dst[4:] + } + dst = i.SqOff.MarshalUnsafe(dst) + dst = i.CqOff.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOUringParams) UnmarshalBytes(src []byte) []byte { + i.SqEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.CqEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.SqThreadCPU = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.SqThreadIdle = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Features = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.WqFd = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 3; idx++ { + i.Resv[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + src = i.SqOff.UnmarshalUnsafe(src) + src = i.CqOff.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOUringParams) Packed() bool { + return i.CqOff.Packed() && i.SqOff.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOUringParams) MarshalUnsafe(dst []byte) []byte { + if i.CqOff.Packed() && i.SqOff.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IOUringParams doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOUringParams) UnmarshalUnsafe(src []byte) []byte { + if i.CqOff.Packed() && i.SqOff.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IOUringParams doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOUringParams) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.CqOff.Packed() && i.SqOff.Packed() { + // Type IOUringParams doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOUringParams) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOUringParams) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.CqOff.Packed() && i.SqOff.Packed() { + // Type IOUringParams doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOUringParams) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOUringParams) WriteTo(writer io.Writer) (int64, error) { + if !i.CqOff.Packed() && i.SqOff.Packed() { + // Type IOUringParams doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IOUringSqe) SizeBytes() int { + return 64 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IOUringSqe) MarshalBytes(dst []byte) []byte { + dst[0] = byte(i.Opcode) + dst = dst[1:] + dst[0] = byte(i.Flags) + dst = dst[1:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.IoPrio)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Fd)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.OffOrAddrOrCmdOp)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.AddrOrSpliceOff)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Len)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.specialFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.UserData)) + dst = dst[8:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.BufIndexOrGroup)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.personality)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.spliceFDOrFileIndex)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.addr3)) + dst = dst[8:] + // Padding: dst[:sizeof(uint64)] ~= uint64(0) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IOUringSqe) UnmarshalBytes(src []byte) []byte { + i.Opcode = uint8(src[0]) + src = src[1:] + i.Flags = uint8(src[0]) + src = src[1:] + i.IoPrio = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.Fd = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.OffOrAddrOrCmdOp = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.AddrOrSpliceOff = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.Len = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.specialFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.UserData = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.BufIndexOrGroup = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.personality = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.spliceFDOrFileIndex = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.addr3 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + // Padding: var _ uint64 ~= src[:sizeof(uint64)] + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IOUringSqe) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IOUringSqe) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IOUringSqe) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IOUringSqe) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IOUringSqe) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IOUringSqe) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IOUringSqe) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IOUringSqe) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPCPerm) SizeBytes() int { + return 48 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPCPerm) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Key)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.GID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CUID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CGID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.Mode)) + dst = dst[2:] + // Padding: dst[:sizeof(uint16)] ~= uint16(0) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.Seq)) + dst = dst[2:] + // Padding: dst[:sizeof(uint16)] ~= uint16(0) + dst = dst[2:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.unused1)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.unused2)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPCPerm) UnmarshalBytes(src []byte) []byte { + i.Key = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.CUID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.CGID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Mode = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: var _ uint16 ~= src[:sizeof(uint16)] + src = src[2:] + i.Seq = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: var _ uint16 ~= src[:sizeof(uint16)] + src = src[2:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + i.unused1 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.unused2 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPCPerm) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPCPerm) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPCPerm) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPCPerm) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPCPerm) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPCPerm) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPCPerm) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPCPerm) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Sysinfo) SizeBytes() int { + return 78 + + 8*3 + + 1*6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Sysinfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Uptime)) + dst = dst[8:] + for idx := 0; idx < 3; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Loads[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.TotalRAM)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.FreeRAM)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SharedRAM)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.BufferRAM)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.TotalSwap)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.FreeSwap)) + dst = dst[8:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Procs)) + dst = dst[2:] + // Padding: dst[:sizeof(byte)*6] ~= [6]byte{0} + dst = dst[1*(6):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.TotalHigh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.FreeHigh)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Unit)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Sysinfo) UnmarshalBytes(src []byte) []byte { + s.Uptime = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < 3; idx++ { + s.Loads[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + s.TotalRAM = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.FreeRAM = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SharedRAM = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.BufferRAM = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.TotalSwap = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.FreeSwap = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Procs = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([6]byte(s._), src[:sizeof(byte)*6]) + src = src[1*(6):] + s.TotalHigh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.FreeHigh = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Unit = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Sysinfo) Packed() bool { + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Sysinfo) MarshalUnsafe(dst []byte) []byte { + // Type Sysinfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Sysinfo) UnmarshalUnsafe(src []byte) []byte { + // Type Sysinfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Sysinfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type Sysinfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Sysinfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Sysinfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type Sysinfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Sysinfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Sysinfo) WriteTo(writer io.Writer) (int64, error) { + // Type Sysinfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (n *NumaPolicy) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NumaPolicy) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*n)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NumaPolicy) UnmarshalBytes(src []byte) []byte { + *n = NumaPolicy(int32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NumaPolicy) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NumaPolicy) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NumaPolicy) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NumaPolicy) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NumaPolicy) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NumaPolicy) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NumaPolicy) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NumaPolicy) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (m *MqAttr) SizeBytes() int { + return 32 + + 8*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (m *MqAttr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MqFlags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MqMaxmsg)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MqMsgsize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MqCurmsgs)) + dst = dst[8:] + // Padding: dst[:sizeof(int64)*4] ~= [4]int64{0} + dst = dst[8*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (m *MqAttr) UnmarshalBytes(src []byte) []byte { + m.MqFlags = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.MqMaxmsg = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.MqMsgsize = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.MqCurmsgs = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + // Padding: ~ copy([4]int64(m._), src[:sizeof(int64)*4]) + src = src[8*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (m *MqAttr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (m *MqAttr) MarshalUnsafe(dst []byte) []byte { + size := m.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(m), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (m *MqAttr) UnmarshalUnsafe(src []byte) []byte { + size := m.SizeBytes() + gohacks.Memmove(unsafe.Pointer(m), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (m *MqAttr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (m *MqAttr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyOutN(cc, addr, m.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (m *MqAttr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (m *MqAttr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyInN(cc, addr, m.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (m *MqAttr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (b *MsgBuf) Packed() bool { + // Type MsgBuf is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (b *MsgBuf) MarshalUnsafe(dst []byte) []byte { + // Type MsgBuf doesn't have a packed layout in memory, fallback to MarshalBytes. + return b.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (b *MsgBuf) UnmarshalUnsafe(src []byte) []byte { + // Type MsgBuf doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return b.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (b *MsgBuf) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type MsgBuf doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(b.SizeBytes()) // escapes: okay. + b.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (b *MsgBuf) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return b.CopyOutN(cc, addr, b.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (b *MsgBuf) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type MsgBuf doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(b.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + b.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (b *MsgBuf) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return b.CopyInN(cc, addr, b.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (b *MsgBuf) WriteTo(writer io.Writer) (int64, error) { + // Type MsgBuf doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, b.SizeBytes()) + b.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (m *MsgInfo) SizeBytes() int { + return 30 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (m *MsgInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgPool)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgMap)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgMax)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgMnb)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgMni)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgSsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgTql)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(m.MsgSeg)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (m *MsgInfo) UnmarshalBytes(src []byte) []byte { + m.MsgPool = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgMap = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgMax = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgMnb = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgMni = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgSsz = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgTql = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgSeg = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (m *MsgInfo) Packed() bool { + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (m *MsgInfo) MarshalUnsafe(dst []byte) []byte { + // Type MsgInfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return m.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (m *MsgInfo) UnmarshalUnsafe(src []byte) []byte { + // Type MsgInfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return m.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (m *MsgInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type MsgInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(m.SizeBytes()) // escapes: okay. + m.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (m *MsgInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyOutN(cc, addr, m.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (m *MsgInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type MsgInfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(m.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + m.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (m *MsgInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyInN(cc, addr, m.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (m *MsgInfo) WriteTo(writer io.Writer) (int64, error) { + // Type MsgInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, m.SizeBytes()) + m.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (m *MsqidDS) SizeBytes() int { + return 48 + + (*IPCPerm)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (m *MsqidDS) MarshalBytes(dst []byte) []byte { + dst = m.MsgPerm.MarshalUnsafe(dst) + dst = m.MsgStime.MarshalUnsafe(dst) + dst = m.MsgRtime.MarshalUnsafe(dst) + dst = m.MsgCtime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MsgCbytes)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MsgQnum)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.MsgQbytes)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgLspid)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(m.MsgLrpid)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.unused4)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(m.unused5)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (m *MsqidDS) UnmarshalBytes(src []byte) []byte { + src = m.MsgPerm.UnmarshalUnsafe(src) + src = m.MsgStime.UnmarshalUnsafe(src) + src = m.MsgRtime.UnmarshalUnsafe(src) + src = m.MsgCtime.UnmarshalUnsafe(src) + m.MsgCbytes = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.MsgQnum = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.MsgQbytes = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.MsgLspid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.MsgLrpid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + m.unused4 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + m.unused5 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (m *MsqidDS) Packed() bool { + return m.MsgCtime.Packed() && m.MsgPerm.Packed() && m.MsgRtime.Packed() && m.MsgStime.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (m *MsqidDS) MarshalUnsafe(dst []byte) []byte { + if m.MsgCtime.Packed() && m.MsgPerm.Packed() && m.MsgRtime.Packed() && m.MsgStime.Packed() { + size := m.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(m), uintptr(size)) + return dst[size:] + } + // Type MsqidDS doesn't have a packed layout in memory, fallback to MarshalBytes. + return m.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (m *MsqidDS) UnmarshalUnsafe(src []byte) []byte { + if m.MsgCtime.Packed() && m.MsgPerm.Packed() && m.MsgRtime.Packed() && m.MsgStime.Packed() { + size := m.SizeBytes() + gohacks.Memmove(unsafe.Pointer(m), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type MsqidDS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return m.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (m *MsqidDS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !m.MsgCtime.Packed() && m.MsgPerm.Packed() && m.MsgRtime.Packed() && m.MsgStime.Packed() { + // Type MsqidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(m.SizeBytes()) // escapes: okay. + m.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (m *MsqidDS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyOutN(cc, addr, m.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (m *MsqidDS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !m.MsgCtime.Packed() && m.MsgPerm.Packed() && m.MsgRtime.Packed() && m.MsgStime.Packed() { + // Type MsqidDS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(m.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + m.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (m *MsqidDS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return m.CopyInN(cc, addr, m.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (m *MsqidDS) WriteTo(writer io.Writer) (int64, error) { + if !m.MsgCtime.Packed() && m.MsgPerm.Packed() && m.MsgRtime.Packed() && m.MsgStime.Packed() { + // Type MsqidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, m.SizeBytes()) + m.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(m))) + hdr.Len = m.SizeBytes() + hdr.Cap = m.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that m + // must live until the use above. + runtime.KeepAlive(m) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (e *EthtoolCmd) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *EthtoolCmd) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*e)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *EthtoolCmd) UnmarshalBytes(src []byte) []byte { + *e = EthtoolCmd(uint32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *EthtoolCmd) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *EthtoolCmd) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *EthtoolCmd) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *EthtoolCmd) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *EthtoolCmd) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *EthtoolCmd) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *EthtoolCmd) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *EthtoolCmd) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *EthtoolGFeatures) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *EthtoolGFeatures) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Cmd)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Size)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *EthtoolGFeatures) UnmarshalBytes(src []byte) []byte { + e.Cmd = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *EthtoolGFeatures) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *EthtoolGFeatures) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *EthtoolGFeatures) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *EthtoolGFeatures) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *EthtoolGFeatures) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *EthtoolGFeatures) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *EthtoolGFeatures) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *EthtoolGFeatures) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *EthtoolGetFeaturesBlock) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *EthtoolGetFeaturesBlock) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Available)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Requested)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Active)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.NeverChanged)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *EthtoolGetFeaturesBlock) UnmarshalBytes(src []byte) []byte { + e.Available = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Requested = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.Active = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + e.NeverChanged = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *EthtoolGetFeaturesBlock) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *EthtoolGetFeaturesBlock) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *EthtoolGetFeaturesBlock) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *EthtoolGetFeaturesBlock) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *EthtoolGetFeaturesBlock) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *EthtoolGetFeaturesBlock) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *EthtoolGetFeaturesBlock) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *EthtoolGetFeaturesBlock) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IFConf) SizeBytes() int { + return 12 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IFConf) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Len)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Ptr)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IFConf) UnmarshalBytes(src []byte) []byte { + i.Len = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(i._), src[:sizeof(byte)*4]) + src = src[1*(4):] + i.Ptr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IFConf) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IFConf) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IFConf) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IFConf) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IFConf) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IFConf) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IFConf) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IFConf) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (ifr *IFReq) SizeBytes() int { + return 0 + + 1*IFNAMSIZ + + 1*24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (ifr *IFReq) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(ifr.IFName[idx]) + dst = dst[1:] + } + for idx := 0; idx < 24; idx++ { + dst[0] = byte(ifr.Data[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (ifr *IFReq) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < IFNAMSIZ; idx++ { + ifr.IFName[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < 24; idx++ { + ifr.Data[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (ifr *IFReq) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (ifr *IFReq) MarshalUnsafe(dst []byte) []byte { + size := ifr.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(ifr), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (ifr *IFReq) UnmarshalUnsafe(src []byte) []byte { + size := ifr.SizeBytes() + gohacks.Memmove(unsafe.Pointer(ifr), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (ifr *IFReq) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(ifr))) + hdr.Len = ifr.SizeBytes() + hdr.Cap = ifr.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that ifr + // must live until the use above. + runtime.KeepAlive(ifr) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (ifr *IFReq) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ifr.CopyOutN(cc, addr, ifr.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (ifr *IFReq) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(ifr))) + hdr.Len = ifr.SizeBytes() + hdr.Cap = ifr.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that ifr + // must live until the use above. + runtime.KeepAlive(ifr) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (ifr *IFReq) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ifr.CopyInN(cc, addr, ifr.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (ifr *IFReq) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(ifr))) + hdr.Len = ifr.SizeBytes() + hdr.Cap = ifr.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that ifr + // must live until the use above. + runtime.KeepAlive(ifr) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (en *ErrorName) SizeBytes() int { + return 1 * XT_FUNCTION_MAXNAMELEN +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (en *ErrorName) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < XT_FUNCTION_MAXNAMELEN; idx++ { + dst[0] = byte(en[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (en *ErrorName) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < XT_FUNCTION_MAXNAMELEN; idx++ { + en[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (en *ErrorName) Packed() bool { + // Array newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (en *ErrorName) MarshalUnsafe(dst []byte) []byte { + size := en.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&en[0]), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (en *ErrorName) UnmarshalUnsafe(src []byte) []byte { + size := en.SizeBytes() + gohacks.Memmove(unsafe.Pointer(en), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (en *ErrorName) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(en))) + hdr.Len = en.SizeBytes() + hdr.Cap = en.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that en + // must live until the use above. + runtime.KeepAlive(en) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (en *ErrorName) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return en.CopyOutN(cc, addr, en.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (en *ErrorName) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(en))) + hdr.Len = en.SizeBytes() + hdr.Cap = en.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that en + // must live until the use above. + runtime.KeepAlive(en) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (en *ErrorName) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return en.CopyInN(cc, addr, en.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (en *ErrorName) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(en))) + hdr.Len = en.SizeBytes() + hdr.Cap = en.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that en + // must live until the use above. + runtime.KeepAlive(en) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (en *ExtensionName) SizeBytes() int { + return 1 * XT_EXTENSION_MAXNAMELEN +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (en *ExtensionName) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < XT_EXTENSION_MAXNAMELEN; idx++ { + dst[0] = byte(en[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (en *ExtensionName) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < XT_EXTENSION_MAXNAMELEN; idx++ { + en[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (en *ExtensionName) Packed() bool { + // Array newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (en *ExtensionName) MarshalUnsafe(dst []byte) []byte { + size := en.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&en[0]), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (en *ExtensionName) UnmarshalUnsafe(src []byte) []byte { + size := en.SizeBytes() + gohacks.Memmove(unsafe.Pointer(en), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (en *ExtensionName) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(en))) + hdr.Len = en.SizeBytes() + hdr.Cap = en.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that en + // must live until the use above. + runtime.KeepAlive(en) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (en *ExtensionName) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return en.CopyOutN(cc, addr, en.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (en *ExtensionName) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(en))) + hdr.Len = en.SizeBytes() + hdr.Cap = en.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that en + // must live until the use above. + runtime.KeepAlive(en) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (en *ExtensionName) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return en.CopyInN(cc, addr, en.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (en *ExtensionName) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(en))) + hdr.Len = en.SizeBytes() + hdr.Cap = en.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that en + // must live until the use above. + runtime.KeepAlive(en) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTEntry) SizeBytes() int { + return 12 + + (*IPTIP)(nil).SizeBytes() + + (*XTCounters)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTEntry) MarshalBytes(dst []byte) []byte { + dst = i.IP.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NFCache)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.TargetOffset)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.NextOffset)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Comeback)) + dst = dst[4:] + dst = i.Counters.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTEntry) UnmarshalBytes(src []byte) []byte { + src = i.IP.UnmarshalUnsafe(src) + i.NFCache = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.TargetOffset = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.NextOffset = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.Comeback = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = i.Counters.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTEntry) Packed() bool { + return i.Counters.Packed() && i.IP.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTEntry) MarshalUnsafe(dst []byte) []byte { + if i.Counters.Packed() && i.IP.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IPTEntry doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTEntry) UnmarshalUnsafe(src []byte) []byte { + if i.Counters.Packed() && i.IP.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IPTEntry doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTEntry) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Counters.Packed() && i.IP.Packed() { + // Type IPTEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTEntry) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTEntry) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Counters.Packed() && i.IP.Packed() { + // Type IPTEntry doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTEntry) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTEntry) WriteTo(writer io.Writer) (int64, error) { + if !i.Counters.Packed() && i.IP.Packed() { + // Type IPTEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTGetEntries) SizeBytes() int { + return 4 + + (*TableName)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTGetEntries) MarshalBytes(dst []byte) []byte { + dst = i.Name.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Size)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTGetEntries) UnmarshalBytes(src []byte) []byte { + src = i.Name.UnmarshalUnsafe(src) + i.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(i._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTGetEntries) Packed() bool { + return i.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTGetEntries) MarshalUnsafe(dst []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IPTGetEntries doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTGetEntries) UnmarshalUnsafe(src []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IPTGetEntries doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTGetEntries) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IPTGetEntries doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTGetEntries) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTGetEntries) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IPTGetEntries doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTGetEntries) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTGetEntries) WriteTo(writer io.Writer) (int64, error) { + if !i.Name.Packed() { + // Type IPTGetEntries doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTGetinfo) SizeBytes() int { + return 12 + + (*TableName)(nil).SizeBytes() + + 4*NF_INET_NUMHOOKS + + 4*NF_INET_NUMHOOKS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTGetinfo) MarshalBytes(dst []byte) []byte { + dst = i.Name.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.ValidHooks)) + dst = dst[4:] + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.HookEntry[idx])) + dst = dst[4:] + } + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Underflow[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NumEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Size)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTGetinfo) UnmarshalBytes(src []byte) []byte { + src = i.Name.UnmarshalUnsafe(src) + i.ValidHooks = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + i.HookEntry[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + i.Underflow[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + i.NumEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTGetinfo) Packed() bool { + return i.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTGetinfo) MarshalUnsafe(dst []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IPTGetinfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTGetinfo) UnmarshalUnsafe(src []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IPTGetinfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTGetinfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IPTGetinfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTGetinfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTGetinfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IPTGetinfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTGetinfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTGetinfo) WriteTo(writer io.Writer) (int64, error) { + if !i.Name.Packed() { + // Type IPTGetinfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTIP) SizeBytes() int { + return 4 + + (*InetAddr)(nil).SizeBytes() + + (*InetAddr)(nil).SizeBytes() + + (*InetAddr)(nil).SizeBytes() + + (*InetAddr)(nil).SizeBytes() + + 1*IFNAMSIZ + + 1*IFNAMSIZ + + 1*IFNAMSIZ + + 1*IFNAMSIZ +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTIP) MarshalBytes(dst []byte) []byte { + dst = i.Src.MarshalUnsafe(dst) + dst = i.Dst.MarshalUnsafe(dst) + dst = i.SrcMask.MarshalUnsafe(dst) + dst = i.DstMask.MarshalUnsafe(dst) + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.InputInterface[idx]) + dst = dst[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.OutputInterface[idx]) + dst = dst[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.InputInterfaceMask[idx]) + dst = dst[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.OutputInterfaceMask[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.Protocol)) + dst = dst[2:] + dst[0] = byte(i.Flags) + dst = dst[1:] + dst[0] = byte(i.InverseFlags) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTIP) UnmarshalBytes(src []byte) []byte { + src = i.Src.UnmarshalUnsafe(src) + src = i.Dst.UnmarshalUnsafe(src) + src = i.SrcMask.UnmarshalUnsafe(src) + src = i.DstMask.UnmarshalUnsafe(src) + for idx := 0; idx < IFNAMSIZ; idx++ { + i.InputInterface[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + i.OutputInterface[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + i.InputInterfaceMask[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + i.OutputInterfaceMask[idx] = src[0] + src = src[1:] + } + i.Protocol = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.Flags = uint8(src[0]) + src = src[1:] + i.InverseFlags = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTIP) Packed() bool { + return i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTIP) MarshalUnsafe(dst []byte) []byte { + if i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IPTIP doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTIP) UnmarshalUnsafe(src []byte) []byte { + if i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IPTIP doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTIP) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + // Type IPTIP doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTIP) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTIP) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + // Type IPTIP doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTIP) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTIP) WriteTo(writer io.Writer) (int64, error) { + if !i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + // Type IPTIP doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTOwnerInfo) SizeBytes() int { + return 18 + + 1*16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTOwnerInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.GID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.PID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.SID)) + dst = dst[4:] + for idx := 0; idx < 16; idx++ { + dst[0] = byte(i.Comm[idx]) + dst = dst[1:] + } + dst[0] = byte(i.Match) + dst = dst[1:] + dst[0] = byte(i.Invert) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTOwnerInfo) UnmarshalBytes(src []byte) []byte { + i.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.PID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.SID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 16; idx++ { + i.Comm[idx] = src[0] + src = src[1:] + } + i.Match = uint8(src[0]) + src = src[1:] + i.Invert = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTOwnerInfo) Packed() bool { + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTOwnerInfo) MarshalUnsafe(dst []byte) []byte { + // Type IPTOwnerInfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTOwnerInfo) UnmarshalUnsafe(src []byte) []byte { + // Type IPTOwnerInfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTOwnerInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type IPTOwnerInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTOwnerInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTOwnerInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type IPTOwnerInfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTOwnerInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTOwnerInfo) WriteTo(writer io.Writer) (int64, error) { + // Type IPTOwnerInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTReplace) SizeBytes() int { + return 24 + + (*TableName)(nil).SizeBytes() + + 4*NF_INET_NUMHOOKS + + 4*NF_INET_NUMHOOKS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTReplace) MarshalBytes(dst []byte) []byte { + dst = i.Name.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.ValidHooks)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NumEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Size)) + dst = dst[4:] + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.HookEntry[idx])) + dst = dst[4:] + } + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Underflow[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NumCounters)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Counters)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTReplace) UnmarshalBytes(src []byte) []byte { + src = i.Name.UnmarshalUnsafe(src) + i.ValidHooks = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.NumEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + i.HookEntry[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + i.Underflow[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + i.NumCounters = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Counters = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTReplace) Packed() bool { + return i.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTReplace) MarshalUnsafe(dst []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IPTReplace doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTReplace) UnmarshalUnsafe(src []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IPTReplace doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTReplace) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IPTReplace doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTReplace) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTReplace) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IPTReplace doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTReplace) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTReplace) WriteTo(writer io.Writer) (int64, error) { + if !i.Name.Packed() { + // Type IPTReplace doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (ke *KernelIPTEntry) Packed() bool { + // Type KernelIPTEntry is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (ke *KernelIPTEntry) MarshalUnsafe(dst []byte) []byte { + // Type KernelIPTEntry doesn't have a packed layout in memory, fallback to MarshalBytes. + return ke.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (ke *KernelIPTEntry) UnmarshalUnsafe(src []byte) []byte { + // Type KernelIPTEntry doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return ke.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (ke *KernelIPTEntry) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIPTEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + ke.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (ke *KernelIPTEntry) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyOutN(cc, addr, ke.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (ke *KernelIPTEntry) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIPTEntry doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + ke.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (ke *KernelIPTEntry) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyInN(cc, addr, ke.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (ke *KernelIPTEntry) WriteTo(writer io.Writer) (int64, error) { + // Type KernelIPTEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, ke.SizeBytes()) + ke.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (ke *KernelIPTGetEntries) Packed() bool { + // Type KernelIPTGetEntries is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (ke *KernelIPTGetEntries) MarshalUnsafe(dst []byte) []byte { + // Type KernelIPTGetEntries doesn't have a packed layout in memory, fallback to MarshalBytes. + return ke.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (ke *KernelIPTGetEntries) UnmarshalUnsafe(src []byte) []byte { + // Type KernelIPTGetEntries doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return ke.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (ke *KernelIPTGetEntries) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIPTGetEntries doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + ke.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (ke *KernelIPTGetEntries) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyOutN(cc, addr, ke.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (ke *KernelIPTGetEntries) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIPTGetEntries doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + ke.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (ke *KernelIPTGetEntries) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyInN(cc, addr, ke.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (ke *KernelIPTGetEntries) WriteTo(writer io.Writer) (int64, error) { + // Type KernelIPTGetEntries doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, ke.SizeBytes()) + ke.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NfNATIPV4MultiRangeCompat) SizeBytes() int { + return 4 + + (*NfNATIPV4Range)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NfNATIPV4MultiRangeCompat) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RangeSize)) + dst = dst[4:] + dst = n.RangeIPV4.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NfNATIPV4MultiRangeCompat) UnmarshalBytes(src []byte) []byte { + n.RangeSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.RangeIPV4.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NfNATIPV4MultiRangeCompat) Packed() bool { + return n.RangeIPV4.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NfNATIPV4MultiRangeCompat) MarshalUnsafe(dst []byte) []byte { + if n.RangeIPV4.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NfNATIPV4MultiRangeCompat doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NfNATIPV4MultiRangeCompat) UnmarshalUnsafe(src []byte) []byte { + if n.RangeIPV4.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NfNATIPV4MultiRangeCompat doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NfNATIPV4MultiRangeCompat) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.RangeIPV4.Packed() { + // Type NfNATIPV4MultiRangeCompat doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NfNATIPV4MultiRangeCompat) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NfNATIPV4MultiRangeCompat) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.RangeIPV4.Packed() { + // Type NfNATIPV4MultiRangeCompat doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NfNATIPV4MultiRangeCompat) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NfNATIPV4MultiRangeCompat) WriteTo(writer io.Writer) (int64, error) { + if !n.RangeIPV4.Packed() { + // Type NfNATIPV4MultiRangeCompat doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NfNATIPV4Range) SizeBytes() int { + return 8 + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NfNATIPV4Range) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.MinIP[idx]) + dst = dst[1:] + } + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.MaxIP[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.MinPort)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.MaxPort)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NfNATIPV4Range) UnmarshalBytes(src []byte) []byte { + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.MinIP[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < 4; idx++ { + n.MaxIP[idx] = src[0] + src = src[1:] + } + n.MinPort = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.MaxPort = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NfNATIPV4Range) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NfNATIPV4Range) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NfNATIPV4Range) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NfNATIPV4Range) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NfNATIPV4Range) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NfNATIPV4Range) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NfNATIPV4Range) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NfNATIPV4Range) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (tn *TableName) SizeBytes() int { + return 1 * XT_TABLE_MAXNAMELEN +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (tn *TableName) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < XT_TABLE_MAXNAMELEN; idx++ { + dst[0] = byte(tn[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (tn *TableName) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < XT_TABLE_MAXNAMELEN; idx++ { + tn[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (tn *TableName) Packed() bool { + // Array newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (tn *TableName) MarshalUnsafe(dst []byte) []byte { + size := tn.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&tn[0]), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (tn *TableName) UnmarshalUnsafe(src []byte) []byte { + size := tn.SizeBytes() + gohacks.Memmove(unsafe.Pointer(tn), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (tn *TableName) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(tn))) + hdr.Len = tn.SizeBytes() + hdr.Cap = tn.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that tn + // must live until the use above. + runtime.KeepAlive(tn) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (tn *TableName) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return tn.CopyOutN(cc, addr, tn.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (tn *TableName) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(tn))) + hdr.Len = tn.SizeBytes() + hdr.Cap = tn.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that tn + // must live until the use above. + runtime.KeepAlive(tn) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (tn *TableName) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return tn.CopyInN(cc, addr, tn.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (tn *TableName) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(tn))) + hdr.Len = tn.SizeBytes() + hdr.Cap = tn.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that tn + // must live until the use above. + runtime.KeepAlive(tn) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTCounters) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTCounters) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(x.Pcnt)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(x.Bcnt)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTCounters) UnmarshalBytes(src []byte) []byte { + x.Pcnt = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + x.Bcnt = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTCounters) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTCounters) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTCounters) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTCounters) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTCounters) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTCounters) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTCounters) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTCounters) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTEntryMatch) SizeBytes() int { + return 3 + + (*ExtensionName)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTEntryMatch) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.MatchSize)) + dst = dst[2:] + dst = x.Name.MarshalUnsafe(dst) + dst[0] = byte(x.Revision) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTEntryMatch) UnmarshalBytes(src []byte) []byte { + x.MatchSize = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + src = x.Name.UnmarshalUnsafe(src) + x.Revision = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTEntryMatch) Packed() bool { + return x.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTEntryMatch) MarshalUnsafe(dst []byte) []byte { + if x.Name.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTEntryMatch doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTEntryMatch) UnmarshalUnsafe(src []byte) []byte { + if x.Name.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTEntryMatch doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTEntryMatch) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() { + // Type XTEntryMatch doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTEntryMatch) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTEntryMatch) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() { + // Type XTEntryMatch doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTEntryMatch) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTEntryMatch) WriteTo(writer io.Writer) (int64, error) { + if !x.Name.Packed() { + // Type XTEntryMatch doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTEntryTarget) SizeBytes() int { + return 3 + + (*ExtensionName)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTEntryTarget) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.TargetSize)) + dst = dst[2:] + dst = x.Name.MarshalUnsafe(dst) + dst[0] = byte(x.Revision) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTEntryTarget) UnmarshalBytes(src []byte) []byte { + x.TargetSize = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + src = x.Name.UnmarshalUnsafe(src) + x.Revision = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTEntryTarget) Packed() bool { + return x.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTEntryTarget) MarshalUnsafe(dst []byte) []byte { + if x.Name.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTEntryTarget doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTEntryTarget) UnmarshalUnsafe(src []byte) []byte { + if x.Name.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTEntryTarget doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTEntryTarget) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() { + // Type XTEntryTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTEntryTarget) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTEntryTarget) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() { + // Type XTEntryTarget doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTEntryTarget) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTEntryTarget) WriteTo(writer io.Writer) (int64, error) { + if !x.Name.Packed() { + // Type XTEntryTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTErrorTarget) SizeBytes() int { + return 0 + + (*XTEntryTarget)(nil).SizeBytes() + + (*ErrorName)(nil).SizeBytes() + + 1*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTErrorTarget) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + dst = x.Name.MarshalUnsafe(dst) + // Padding: dst[:sizeof(byte)*2] ~= [2]byte{0} + dst = dst[1*(2):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTErrorTarget) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + src = x.Name.UnmarshalUnsafe(src) + // Padding: ~ copy([2]byte(x._), src[:sizeof(byte)*2]) + src = src[1*(2):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTErrorTarget) Packed() bool { + return x.Name.Packed() && x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTErrorTarget) MarshalUnsafe(dst []byte) []byte { + if x.Name.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTErrorTarget doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTErrorTarget) UnmarshalUnsafe(src []byte) []byte { + if x.Name.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTErrorTarget doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTErrorTarget) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() && x.Target.Packed() { + // Type XTErrorTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTErrorTarget) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTErrorTarget) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() && x.Target.Packed() { + // Type XTErrorTarget doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTErrorTarget) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTErrorTarget) WriteTo(writer io.Writer) (int64, error) { + if !x.Name.Packed() && x.Target.Packed() { + // Type XTErrorTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTGetRevision) SizeBytes() int { + return 1 + + (*ExtensionName)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTGetRevision) MarshalBytes(dst []byte) []byte { + dst = x.Name.MarshalUnsafe(dst) + dst[0] = byte(x.Revision) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTGetRevision) UnmarshalBytes(src []byte) []byte { + src = x.Name.UnmarshalUnsafe(src) + x.Revision = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTGetRevision) Packed() bool { + return x.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTGetRevision) MarshalUnsafe(dst []byte) []byte { + if x.Name.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTGetRevision doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTGetRevision) UnmarshalUnsafe(src []byte) []byte { + if x.Name.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTGetRevision doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTGetRevision) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() { + // Type XTGetRevision doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTGetRevision) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTGetRevision) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Name.Packed() { + // Type XTGetRevision doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTGetRevision) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTGetRevision) WriteTo(writer io.Writer) (int64, error) { + if !x.Name.Packed() { + // Type XTGetRevision doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTMultiport) SizeBytes() int { + return 2 + + 2*XT_MULTI_PORTS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTMultiport) MarshalBytes(dst []byte) []byte { + dst[0] = byte(x.Flags) + dst = dst[1:] + dst[0] = byte(x.Count) + dst = dst[1:] + for idx := 0; idx < XT_MULTI_PORTS; idx++ { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.Ports[idx])) + dst = dst[2:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTMultiport) UnmarshalBytes(src []byte) []byte { + x.Flags = uint8(src[0]) + src = src[1:] + x.Count = uint8(src[0]) + src = src[1:] + for idx := 0; idx < XT_MULTI_PORTS; idx++ { + x.Ports[idx] = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTMultiport) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTMultiport) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTMultiport) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTMultiport) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTMultiport) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTMultiport) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTMultiport) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTMultiport) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTMultiportV1) SizeBytes() int { + return 3 + + 2*XT_MULTI_PORTS + + 1*XT_MULTI_PORTS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTMultiportV1) MarshalBytes(dst []byte) []byte { + dst[0] = byte(x.Flags) + dst = dst[1:] + dst[0] = byte(x.Count) + dst = dst[1:] + for idx := 0; idx < XT_MULTI_PORTS; idx++ { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.Ports[idx])) + dst = dst[2:] + } + for idx := 0; idx < XT_MULTI_PORTS; idx++ { + dst[0] = byte(x.Pflags[idx]) + dst = dst[1:] + } + dst[0] = byte(x.Invert) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTMultiportV1) UnmarshalBytes(src []byte) []byte { + x.Flags = uint8(src[0]) + src = src[1:] + x.Count = uint8(src[0]) + src = src[1:] + for idx := 0; idx < XT_MULTI_PORTS; idx++ { + x.Ports[idx] = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + } + for idx := 0; idx < XT_MULTI_PORTS; idx++ { + x.Pflags[idx] = uint8(src[0]) + src = src[1:] + } + x.Invert = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTMultiportV1) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTMultiportV1) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTMultiportV1) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTMultiportV1) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTMultiportV1) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTMultiportV1) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTMultiportV1) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTMultiportV1) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTNATTargetV0) SizeBytes() int { + return 0 + + (*XTEntryTarget)(nil).SizeBytes() + + (*NfNATIPV4MultiRangeCompat)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTNATTargetV0) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + dst = x.NfRange.MarshalUnsafe(dst) + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTNATTargetV0) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + src = x.NfRange.UnmarshalUnsafe(src) + // Padding: ~ copy([4]byte(x._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTNATTargetV0) Packed() bool { + return x.NfRange.Packed() && x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTNATTargetV0) MarshalUnsafe(dst []byte) []byte { + if x.NfRange.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTNATTargetV0 doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTNATTargetV0) UnmarshalUnsafe(src []byte) []byte { + if x.NfRange.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTNATTargetV0 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTNATTargetV0) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.NfRange.Packed() && x.Target.Packed() { + // Type XTNATTargetV0 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTNATTargetV0) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTNATTargetV0) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.NfRange.Packed() && x.Target.Packed() { + // Type XTNATTargetV0 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTNATTargetV0) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTNATTargetV0) WriteTo(writer io.Writer) (int64, error) { + if !x.NfRange.Packed() && x.Target.Packed() { + // Type XTNATTargetV0 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTNATTargetV1) SizeBytes() int { + return 0 + + (*XTEntryTarget)(nil).SizeBytes() + + (*NFNATRange)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTNATTargetV1) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + dst = x.Range.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTNATTargetV1) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + src = x.Range.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTNATTargetV1) Packed() bool { + return x.Range.Packed() && x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTNATTargetV1) MarshalUnsafe(dst []byte) []byte { + if x.Range.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTNATTargetV1 doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTNATTargetV1) UnmarshalUnsafe(src []byte) []byte { + if x.Range.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTNATTargetV1 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTNATTargetV1) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Range.Packed() && x.Target.Packed() { + // Type XTNATTargetV1 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTNATTargetV1) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTNATTargetV1) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Range.Packed() && x.Target.Packed() { + // Type XTNATTargetV1 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTNATTargetV1) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTNATTargetV1) WriteTo(writer io.Writer) (int64, error) { + if !x.Range.Packed() && x.Target.Packed() { + // Type XTNATTargetV1 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTNATTargetV2) SizeBytes() int { + return 0 + + (*XTEntryTarget)(nil).SizeBytes() + + (*NFNATRange2)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTNATTargetV2) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + dst = x.Range.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTNATTargetV2) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + src = x.Range.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTNATTargetV2) Packed() bool { + return x.Range.Packed() && x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTNATTargetV2) MarshalUnsafe(dst []byte) []byte { + if x.Range.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTNATTargetV2 doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTNATTargetV2) UnmarshalUnsafe(src []byte) []byte { + if x.Range.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTNATTargetV2 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTNATTargetV2) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Range.Packed() && x.Target.Packed() { + // Type XTNATTargetV2 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTNATTargetV2) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTNATTargetV2) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Range.Packed() && x.Target.Packed() { + // Type XTNATTargetV2 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTNATTargetV2) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTNATTargetV2) WriteTo(writer io.Writer) (int64, error) { + if !x.Range.Packed() && x.Target.Packed() { + // Type XTNATTargetV2 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTOwnerMatchInfo) SizeBytes() int { + return 18 + + 1*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTOwnerMatchInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.UIDMin)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.UIDMax)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.GIDMin)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.GIDMax)) + dst = dst[4:] + dst[0] = byte(x.Match) + dst = dst[1:] + dst[0] = byte(x.Invert) + dst = dst[1:] + // Padding: dst[:sizeof(byte)*2] ~= [2]byte{0} + dst = dst[1*(2):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTOwnerMatchInfo) UnmarshalBytes(src []byte) []byte { + x.UIDMin = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.UIDMax = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.GIDMin = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.GIDMax = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.Match = uint8(src[0]) + src = src[1:] + x.Invert = uint8(src[0]) + src = src[1:] + // Padding: ~ copy([2]byte(x._), src[:sizeof(byte)*2]) + src = src[1*(2):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTOwnerMatchInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTOwnerMatchInfo) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTOwnerMatchInfo) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTOwnerMatchInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTOwnerMatchInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTOwnerMatchInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTOwnerMatchInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTOwnerMatchInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTRedirectTarget) SizeBytes() int { + return 0 + + (*XTEntryTarget)(nil).SizeBytes() + + (*NfNATIPV4MultiRangeCompat)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTRedirectTarget) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + dst = x.NfRange.MarshalUnsafe(dst) + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTRedirectTarget) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + src = x.NfRange.UnmarshalUnsafe(src) + // Padding: ~ copy([4]byte(x._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTRedirectTarget) Packed() bool { + return x.NfRange.Packed() && x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTRedirectTarget) MarshalUnsafe(dst []byte) []byte { + if x.NfRange.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTRedirectTarget doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTRedirectTarget) UnmarshalUnsafe(src []byte) []byte { + if x.NfRange.Packed() && x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTRedirectTarget doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTRedirectTarget) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.NfRange.Packed() && x.Target.Packed() { + // Type XTRedirectTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTRedirectTarget) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTRedirectTarget) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.NfRange.Packed() && x.Target.Packed() { + // Type XTRedirectTarget doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTRedirectTarget) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTRedirectTarget) WriteTo(writer io.Writer) (int64, error) { + if !x.NfRange.Packed() && x.Target.Packed() { + // Type XTRedirectTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTStandardTarget) SizeBytes() int { + return 4 + + (*XTEntryTarget)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTStandardTarget) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.Verdict)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTStandardTarget) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + x.Verdict = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(x._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTStandardTarget) Packed() bool { + return x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTStandardTarget) MarshalUnsafe(dst []byte) []byte { + if x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTStandardTarget doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTStandardTarget) UnmarshalUnsafe(src []byte) []byte { + if x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTStandardTarget doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTStandardTarget) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Target.Packed() { + // Type XTStandardTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTStandardTarget) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTStandardTarget) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Target.Packed() { + // Type XTStandardTarget doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTStandardTarget) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTStandardTarget) WriteTo(writer io.Writer) (int64, error) { + if !x.Target.Packed() { + // Type XTStandardTarget doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTTCP) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTTCP) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.SourcePortStart)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.SourcePortEnd)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.DestinationPortStart)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.DestinationPortEnd)) + dst = dst[2:] + dst[0] = byte(x.Option) + dst = dst[1:] + dst[0] = byte(x.FlagMask) + dst = dst[1:] + dst[0] = byte(x.FlagCompare) + dst = dst[1:] + dst[0] = byte(x.InverseFlags) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTTCP) UnmarshalBytes(src []byte) []byte { + x.SourcePortStart = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.SourcePortEnd = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.DestinationPortStart = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.DestinationPortEnd = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.Option = uint8(src[0]) + src = src[1:] + x.FlagMask = uint8(src[0]) + src = src[1:] + x.FlagCompare = uint8(src[0]) + src = src[1:] + x.InverseFlags = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTTCP) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTTCP) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTTCP) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTTCP) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTTCP) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTTCP) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTTCP) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTTCP) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTUDP) SizeBytes() int { + return 10 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTUDP) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.SourcePortStart)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.SourcePortEnd)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.DestinationPortStart)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.DestinationPortEnd)) + dst = dst[2:] + dst[0] = byte(x.InverseFlags) + dst = dst[1:] + // Padding: dst[:sizeof(uint8)] ~= uint8(0) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTUDP) UnmarshalBytes(src []byte) []byte { + x.SourcePortStart = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.SourcePortEnd = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.DestinationPortStart = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.DestinationPortEnd = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.InverseFlags = uint8(src[0]) + src = src[1:] + // Padding: var _ uint8 ~= src[:sizeof(uint8)] + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTUDP) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTUDP) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTUDP) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTUDP) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTUDP) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTUDP) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTUDP) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTUDP) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IP6TEntry) SizeBytes() int { + return 12 + + (*IP6TIP)(nil).SizeBytes() + + 1*4 + + (*XTCounters)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IP6TEntry) MarshalBytes(dst []byte) []byte { + dst = i.IPv6.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NFCache)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.TargetOffset)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.NextOffset)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Comeback)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + dst = i.Counters.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IP6TEntry) UnmarshalBytes(src []byte) []byte { + src = i.IPv6.UnmarshalUnsafe(src) + i.NFCache = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.TargetOffset = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.NextOffset = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.Comeback = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(i._), src[:sizeof(byte)*4]) + src = src[1*(4):] + src = i.Counters.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IP6TEntry) Packed() bool { + return i.Counters.Packed() && i.IPv6.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IP6TEntry) MarshalUnsafe(dst []byte) []byte { + if i.Counters.Packed() && i.IPv6.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IP6TEntry doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IP6TEntry) UnmarshalUnsafe(src []byte) []byte { + if i.Counters.Packed() && i.IPv6.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IP6TEntry doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IP6TEntry) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Counters.Packed() && i.IPv6.Packed() { + // Type IP6TEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IP6TEntry) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IP6TEntry) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Counters.Packed() && i.IPv6.Packed() { + // Type IP6TEntry doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IP6TEntry) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IP6TEntry) WriteTo(writer io.Writer) (int64, error) { + if !i.Counters.Packed() && i.IPv6.Packed() { + // Type IP6TEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IP6TIP) SizeBytes() int { + return 5 + + (*Inet6Addr)(nil).SizeBytes() + + (*Inet6Addr)(nil).SizeBytes() + + (*Inet6Addr)(nil).SizeBytes() + + (*Inet6Addr)(nil).SizeBytes() + + 1*IFNAMSIZ + + 1*IFNAMSIZ + + 1*IFNAMSIZ + + 1*IFNAMSIZ + + 1*3 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IP6TIP) MarshalBytes(dst []byte) []byte { + dst = i.Src.MarshalUnsafe(dst) + dst = i.Dst.MarshalUnsafe(dst) + dst = i.SrcMask.MarshalUnsafe(dst) + dst = i.DstMask.MarshalUnsafe(dst) + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.InputInterface[idx]) + dst = dst[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.OutputInterface[idx]) + dst = dst[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.InputInterfaceMask[idx]) + dst = dst[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + dst[0] = byte(i.OutputInterfaceMask[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.Protocol)) + dst = dst[2:] + dst[0] = byte(i.TOS) + dst = dst[1:] + dst[0] = byte(i.Flags) + dst = dst[1:] + dst[0] = byte(i.InverseFlags) + dst = dst[1:] + // Padding: dst[:sizeof(byte)*3] ~= [3]byte{0} + dst = dst[1*(3):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IP6TIP) UnmarshalBytes(src []byte) []byte { + src = i.Src.UnmarshalUnsafe(src) + src = i.Dst.UnmarshalUnsafe(src) + src = i.SrcMask.UnmarshalUnsafe(src) + src = i.DstMask.UnmarshalUnsafe(src) + for idx := 0; idx < IFNAMSIZ; idx++ { + i.InputInterface[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + i.OutputInterface[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + i.InputInterfaceMask[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < IFNAMSIZ; idx++ { + i.OutputInterfaceMask[idx] = src[0] + src = src[1:] + } + i.Protocol = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.TOS = uint8(src[0]) + src = src[1:] + i.Flags = uint8(src[0]) + src = src[1:] + i.InverseFlags = uint8(src[0]) + src = src[1:] + // Padding: ~ copy([3]byte(i._), src[:sizeof(byte)*3]) + src = src[1*(3):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IP6TIP) Packed() bool { + return i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IP6TIP) MarshalUnsafe(dst []byte) []byte { + if i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IP6TIP doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IP6TIP) UnmarshalUnsafe(src []byte) []byte { + if i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IP6TIP doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IP6TIP) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + // Type IP6TIP doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IP6TIP) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IP6TIP) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + // Type IP6TIP doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IP6TIP) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IP6TIP) WriteTo(writer io.Writer) (int64, error) { + if !i.Dst.Packed() && i.DstMask.Packed() && i.Src.Packed() && i.SrcMask.Packed() { + // Type IP6TIP doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IP6TReplace) SizeBytes() int { + return 24 + + (*TableName)(nil).SizeBytes() + + 4*NF_INET_NUMHOOKS + + 4*NF_INET_NUMHOOKS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IP6TReplace) MarshalBytes(dst []byte) []byte { + dst = i.Name.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.ValidHooks)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NumEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Size)) + dst = dst[4:] + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.HookEntry[idx])) + dst = dst[4:] + } + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Underflow[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.NumCounters)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.Counters)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IP6TReplace) UnmarshalBytes(src []byte) []byte { + src = i.Name.UnmarshalUnsafe(src) + i.ValidHooks = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.NumEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + i.HookEntry[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < NF_INET_NUMHOOKS; idx++ { + i.Underflow[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + i.NumCounters = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Counters = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IP6TReplace) Packed() bool { + return i.Name.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IP6TReplace) MarshalUnsafe(dst []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IP6TReplace doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IP6TReplace) UnmarshalUnsafe(src []byte) []byte { + if i.Name.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IP6TReplace doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IP6TReplace) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IP6TReplace doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IP6TReplace) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IP6TReplace) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Name.Packed() { + // Type IP6TReplace doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IP6TReplace) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IP6TReplace) WriteTo(writer io.Writer) (int64, error) { + if !i.Name.Packed() { + // Type IP6TReplace doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (ke *KernelIP6TEntry) Packed() bool { + // Type KernelIP6TEntry is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (ke *KernelIP6TEntry) MarshalUnsafe(dst []byte) []byte { + // Type KernelIP6TEntry doesn't have a packed layout in memory, fallback to MarshalBytes. + return ke.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (ke *KernelIP6TEntry) UnmarshalUnsafe(src []byte) []byte { + // Type KernelIP6TEntry doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return ke.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (ke *KernelIP6TEntry) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIP6TEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + ke.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (ke *KernelIP6TEntry) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyOutN(cc, addr, ke.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (ke *KernelIP6TEntry) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIP6TEntry doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + ke.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (ke *KernelIP6TEntry) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyInN(cc, addr, ke.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (ke *KernelIP6TEntry) WriteTo(writer io.Writer) (int64, error) { + // Type KernelIP6TEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, ke.SizeBytes()) + ke.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (ke *KernelIP6TGetEntries) Packed() bool { + // Type KernelIP6TGetEntries is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (ke *KernelIP6TGetEntries) MarshalUnsafe(dst []byte) []byte { + // Type KernelIP6TGetEntries doesn't have a packed layout in memory, fallback to MarshalBytes. + return ke.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (ke *KernelIP6TGetEntries) UnmarshalUnsafe(src []byte) []byte { + // Type KernelIP6TGetEntries doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return ke.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (ke *KernelIP6TGetEntries) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIP6TGetEntries doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + ke.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (ke *KernelIP6TGetEntries) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyOutN(cc, addr, ke.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (ke *KernelIP6TGetEntries) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type KernelIP6TGetEntries doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(ke.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + ke.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (ke *KernelIP6TGetEntries) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ke.CopyInN(cc, addr, ke.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (ke *KernelIP6TGetEntries) WriteTo(writer io.Writer) (int64, error) { + // Type KernelIP6TGetEntries doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, ke.SizeBytes()) + ke.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NFNATRange) SizeBytes() int { + return 8 + + (*Inet6Addr)(nil).SizeBytes() + + (*Inet6Addr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NFNATRange) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + dst = n.MinAddr.MarshalUnsafe(dst) + dst = n.MaxAddr.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.MinProto)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.MaxProto)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NFNATRange) UnmarshalBytes(src []byte) []byte { + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.MinAddr.UnmarshalUnsafe(src) + src = n.MaxAddr.UnmarshalUnsafe(src) + n.MinProto = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.MaxProto = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NFNATRange) Packed() bool { + return n.MaxAddr.Packed() && n.MinAddr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NFNATRange) MarshalUnsafe(dst []byte) []byte { + if n.MaxAddr.Packed() && n.MinAddr.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NFNATRange doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NFNATRange) UnmarshalUnsafe(src []byte) []byte { + if n.MaxAddr.Packed() && n.MinAddr.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NFNATRange doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NFNATRange) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.MaxAddr.Packed() && n.MinAddr.Packed() { + // Type NFNATRange doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NFNATRange) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NFNATRange) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.MaxAddr.Packed() && n.MinAddr.Packed() { + // Type NFNATRange doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NFNATRange) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NFNATRange) WriteTo(writer io.Writer) (int64, error) { + if !n.MaxAddr.Packed() && n.MinAddr.Packed() { + // Type NFNATRange doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NFNATRange2) SizeBytes() int { + return 10 + + (*Inet6Addr)(nil).SizeBytes() + + (*Inet6Addr)(nil).SizeBytes() + + 1*6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NFNATRange2) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + dst = n.MinAddr.MarshalUnsafe(dst) + dst = n.MaxAddr.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.MinProto)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.MaxProto)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.BaseProto)) + dst = dst[2:] + // Padding: dst[:sizeof(byte)*6] ~= [6]byte{0} + dst = dst[1*(6):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NFNATRange2) UnmarshalBytes(src []byte) []byte { + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.MinAddr.UnmarshalUnsafe(src) + src = n.MaxAddr.UnmarshalUnsafe(src) + n.MinProto = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.MaxProto = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.BaseProto = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([6]byte(n._), src[:sizeof(byte)*6]) + src = src[1*(6):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NFNATRange2) Packed() bool { + return n.MaxAddr.Packed() && n.MinAddr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NFNATRange2) MarshalUnsafe(dst []byte) []byte { + if n.MaxAddr.Packed() && n.MinAddr.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NFNATRange2 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NFNATRange2) UnmarshalUnsafe(src []byte) []byte { + if n.MaxAddr.Packed() && n.MinAddr.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NFNATRange2 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NFNATRange2) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.MaxAddr.Packed() && n.MinAddr.Packed() { + // Type NFNATRange2 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NFNATRange2) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NFNATRange2) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.MaxAddr.Packed() && n.MinAddr.Packed() { + // Type NFNATRange2 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NFNATRange2) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NFNATRange2) WriteTo(writer io.Writer) (int64, error) { + if !n.MaxAddr.Packed() && n.MinAddr.Packed() { + // Type NFNATRange2 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NetlinkAttrHeader) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NetlinkAttrHeader) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.Length)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.Type)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NetlinkAttrHeader) UnmarshalBytes(src []byte) []byte { + n.Length = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.Type = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NetlinkAttrHeader) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NetlinkAttrHeader) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NetlinkAttrHeader) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NetlinkAttrHeader) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NetlinkAttrHeader) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NetlinkAttrHeader) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NetlinkAttrHeader) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NetlinkAttrHeader) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NetlinkErrorMessage) SizeBytes() int { + return 4 + + (*NetlinkMessageHeader)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NetlinkErrorMessage) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Error)) + dst = dst[4:] + dst = n.Header.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NetlinkErrorMessage) UnmarshalBytes(src []byte) []byte { + n.Error = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.Header.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NetlinkErrorMessage) Packed() bool { + return n.Header.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NetlinkErrorMessage) MarshalUnsafe(dst []byte) []byte { + if n.Header.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NetlinkErrorMessage doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NetlinkErrorMessage) UnmarshalUnsafe(src []byte) []byte { + if n.Header.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NetlinkErrorMessage doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NetlinkErrorMessage) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Header.Packed() { + // Type NetlinkErrorMessage doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NetlinkErrorMessage) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NetlinkErrorMessage) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Header.Packed() { + // Type NetlinkErrorMessage doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NetlinkErrorMessage) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NetlinkErrorMessage) WriteTo(writer io.Writer) (int64, error) { + if !n.Header.Packed() { + // Type NetlinkErrorMessage doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NetlinkMessageHeader) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NetlinkMessageHeader) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Length)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.Type)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.Flags)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Seq)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.PortID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NetlinkMessageHeader) UnmarshalBytes(src []byte) []byte { + n.Length = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Type = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.Flags = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + n.Seq = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.PortID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NetlinkMessageHeader) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NetlinkMessageHeader) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NetlinkMessageHeader) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NetlinkMessageHeader) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NetlinkMessageHeader) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NetlinkMessageHeader) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NetlinkMessageHeader) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NetlinkMessageHeader) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockAddrNetlink) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockAddrNetlink) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Family)) + dst = dst[2:] + // Padding: dst[:sizeof(uint16)] ~= uint16(0) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.PortID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Groups)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockAddrNetlink) UnmarshalBytes(src []byte) []byte { + s.Family = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: var _ uint16 ~= src[:sizeof(uint16)] + src = src[2:] + s.PortID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Groups = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockAddrNetlink) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockAddrNetlink) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockAddrNetlink) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockAddrNetlink) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockAddrNetlink) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockAddrNetlink) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockAddrNetlink) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockAddrNetlink) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NetFilterGenMsg) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NetFilterGenMsg) MarshalBytes(dst []byte) []byte { + dst[0] = byte(n.Family) + dst = dst[1:] + dst[0] = byte(n.Version) + dst = dst[1:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.ResourceID)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NetFilterGenMsg) UnmarshalBytes(src []byte) []byte { + n.Family = uint8(src[0]) + src = src[1:] + n.Version = uint8(src[0]) + src = src[1:] + n.ResourceID = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NetFilterGenMsg) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NetFilterGenMsg) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NetFilterGenMsg) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NetFilterGenMsg) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NetFilterGenMsg) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NetFilterGenMsg) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NetFilterGenMsg) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NetFilterGenMsg) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *InterfaceAddrMessage) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *InterfaceAddrMessage) MarshalBytes(dst []byte) []byte { + dst[0] = byte(i.Family) + dst = dst[1:] + dst[0] = byte(i.PrefixLen) + dst = dst[1:] + dst[0] = byte(i.Flags) + dst = dst[1:] + dst[0] = byte(i.Scope) + dst = dst[1:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Index)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *InterfaceAddrMessage) UnmarshalBytes(src []byte) []byte { + i.Family = uint8(src[0]) + src = src[1:] + i.PrefixLen = uint8(src[0]) + src = src[1:] + i.Flags = uint8(src[0]) + src = src[1:] + i.Scope = uint8(src[0]) + src = src[1:] + i.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *InterfaceAddrMessage) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *InterfaceAddrMessage) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *InterfaceAddrMessage) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *InterfaceAddrMessage) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *InterfaceAddrMessage) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *InterfaceAddrMessage) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *InterfaceAddrMessage) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *InterfaceAddrMessage) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *InterfaceInfoMessage) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *InterfaceInfoMessage) MarshalBytes(dst []byte) []byte { + dst[0] = byte(i.Family) + dst = dst[1:] + // Padding: dst[:sizeof(uint8)] ~= uint8(0) + dst = dst[1:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.Type)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Change)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *InterfaceInfoMessage) UnmarshalBytes(src []byte) []byte { + i.Family = uint8(src[0]) + src = src[1:] + // Padding: var _ uint8 ~= src[:sizeof(uint8)] + src = src[1:] + i.Type = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + i.Index = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.Change = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *InterfaceInfoMessage) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *InterfaceInfoMessage) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *InterfaceInfoMessage) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *InterfaceInfoMessage) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *InterfaceInfoMessage) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *InterfaceInfoMessage) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *InterfaceInfoMessage) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *InterfaceInfoMessage) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RouteMessage) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RouteMessage) MarshalBytes(dst []byte) []byte { + dst[0] = byte(r.Family) + dst = dst[1:] + dst[0] = byte(r.DstLen) + dst = dst[1:] + dst[0] = byte(r.SrcLen) + dst = dst[1:] + dst[0] = byte(r.TOS) + dst = dst[1:] + dst[0] = byte(r.Table) + dst = dst[1:] + dst[0] = byte(r.Protocol) + dst = dst[1:] + dst[0] = byte(r.Scope) + dst = dst[1:] + dst[0] = byte(r.Type) + dst = dst[1:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RouteMessage) UnmarshalBytes(src []byte) []byte { + r.Family = uint8(src[0]) + src = src[1:] + r.DstLen = uint8(src[0]) + src = src[1:] + r.SrcLen = uint8(src[0]) + src = src[1:] + r.TOS = uint8(src[0]) + src = src[1:] + r.Table = uint8(src[0]) + src = src[1:] + r.Protocol = uint8(src[0]) + src = src[1:] + r.Scope = uint8(src[0]) + src = src[1:] + r.Type = uint8(src[0]) + src = src[1:] + r.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RouteMessage) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RouteMessage) MarshalUnsafe(dst []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RouteMessage) UnmarshalUnsafe(src []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RouteMessage) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RouteMessage) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RouteMessage) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RouteMessage) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RouteMessage) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RtAttr) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RtAttr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(r.Len)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(r.Type)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RtAttr) UnmarshalBytes(src []byte) []byte { + r.Len = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + r.Type = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RtAttr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RtAttr) MarshalUnsafe(dst []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RtAttr) UnmarshalUnsafe(src []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RtAttr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RtAttr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RtAttr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RtAttr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RtAttr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *PollFD) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *PollFD) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.Events)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.REvents)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *PollFD) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Events = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + p.REvents = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *PollFD) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *PollFD) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *PollFD) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *PollFD) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *PollFD) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *PollFD) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *PollFD) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *PollFD) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyPollFDSliceIn copies in a slice of PollFD objects from the task's memory. +func CopyPollFDSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []PollFD) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*PollFD)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyPollFDSliceOut copies a slice of PollFD objects to the task's memory. +func CopyPollFDSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []PollFD) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*PollFD)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafePollFDSlice is like PollFD.MarshalUnsafe, but for a []PollFD. +func MarshalUnsafePollFDSlice(src []PollFD, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*PollFD)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafePollFDSlice is like PollFD.UnmarshalUnsafe, but for a []PollFD. +func UnmarshalUnsafePollFDSlice(dst []PollFD, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*PollFD)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RSeqCriticalSection) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RSeqCriticalSection) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Version)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.Start)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.PostCommitOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.Abort)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RSeqCriticalSection) UnmarshalBytes(src []byte) []byte { + r.Version = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + r.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + r.Start = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.PostCommitOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.Abort = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RSeqCriticalSection) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RSeqCriticalSection) MarshalUnsafe(dst []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RSeqCriticalSection) UnmarshalUnsafe(src []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RSeqCriticalSection) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RSeqCriticalSection) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RSeqCriticalSection) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RSeqCriticalSection) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RSeqCriticalSection) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *Rusage) SizeBytes() int { + return 112 + + (*Timeval)(nil).SizeBytes() + + (*Timeval)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *Rusage) MarshalBytes(dst []byte) []byte { + dst = r.UTime.MarshalUnsafe(dst) + dst = r.STime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.MaxRSS)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.IXRSS)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.IDRSS)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.ISRSS)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.MinFlt)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.MajFlt)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.NSwap)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.InBlock)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.OuBlock)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.MsgSnd)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.MsgRcv)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.NSignals)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.NVCSw)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(r.NIvCSw)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *Rusage) UnmarshalBytes(src []byte) []byte { + src = r.UTime.UnmarshalUnsafe(src) + src = r.STime.UnmarshalUnsafe(src) + r.MaxRSS = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.IXRSS = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.IDRSS = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.ISRSS = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.MinFlt = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.MajFlt = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.NSwap = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.InBlock = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.OuBlock = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.MsgSnd = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.MsgRcv = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.NSignals = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.NVCSw = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + r.NIvCSw = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *Rusage) Packed() bool { + return r.STime.Packed() && r.UTime.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *Rusage) MarshalUnsafe(dst []byte) []byte { + if r.STime.Packed() && r.UTime.Packed() { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] + } + // Type Rusage doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *Rusage) UnmarshalUnsafe(src []byte) []byte { + if r.STime.Packed() && r.UTime.Packed() { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Rusage doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *Rusage) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !r.STime.Packed() && r.UTime.Packed() { + // Type Rusage doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *Rusage) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *Rusage) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !r.STime.Packed() && r.UTime.Packed() { + // Type Rusage doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *Rusage) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *Rusage) WriteTo(writer io.Writer) (int64, error) { + if !r.STime.Packed() && r.UTime.Packed() { + // Type Rusage doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (sd *SeccompData) SizeBytes() int { + return 16 + + 8*6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (sd *SeccompData) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(sd.Nr)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(sd.Arch)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(sd.InstructionPointer)) + dst = dst[8:] + for idx := 0; idx < 6; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(sd.Args[idx])) + dst = dst[8:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (sd *SeccompData) UnmarshalBytes(src []byte) []byte { + sd.Nr = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + sd.Arch = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + sd.InstructionPointer = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < 6; idx++ { + sd.Args[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (sd *SeccompData) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (sd *SeccompData) MarshalUnsafe(dst []byte) []byte { + size := sd.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(sd), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (sd *SeccompData) UnmarshalUnsafe(src []byte) []byte { + size := sd.SizeBytes() + gohacks.Memmove(unsafe.Pointer(sd), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (sd *SeccompData) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(sd))) + hdr.Len = sd.SizeBytes() + hdr.Cap = sd.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that sd + // must live until the use above. + runtime.KeepAlive(sd) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (sd *SeccompData) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return sd.CopyOutN(cc, addr, sd.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (sd *SeccompData) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(sd))) + hdr.Len = sd.SizeBytes() + hdr.Cap = sd.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that sd + // must live until the use above. + runtime.KeepAlive(sd) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (sd *SeccompData) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return sd.CopyInN(cc, addr, sd.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (sd *SeccompData) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(sd))) + hdr.Len = sd.SizeBytes() + hdr.Cap = sd.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that sd + // must live until the use above. + runtime.KeepAlive(sd) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SeccompNotif) SizeBytes() int { + return 16 + + (*SeccompData)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SeccompNotif) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Pid)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Flags)) + dst = dst[4:] + dst = s.Data.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SeccompNotif) UnmarshalBytes(src []byte) []byte { + s.ID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Pid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = s.Data.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SeccompNotif) Packed() bool { + return s.Data.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SeccompNotif) MarshalUnsafe(dst []byte) []byte { + if s.Data.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SeccompNotif doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SeccompNotif) UnmarshalUnsafe(src []byte) []byte { + if s.Data.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SeccompNotif doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SeccompNotif) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Data.Packed() { + // Type SeccompNotif doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SeccompNotif) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SeccompNotif) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Data.Packed() { + // Type SeccompNotif doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SeccompNotif) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SeccompNotif) WriteTo(writer io.Writer) (int64, error) { + if !s.Data.Packed() { + // Type SeccompNotif doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SeccompNotifResp) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SeccompNotifResp) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Val)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Error)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SeccompNotifResp) UnmarshalBytes(src []byte) []byte { + s.ID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Val = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Error = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SeccompNotifResp) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SeccompNotifResp) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SeccompNotifResp) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SeccompNotifResp) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SeccompNotifResp) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SeccompNotifResp) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SeccompNotifResp) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SeccompNotifResp) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SeccompNotifSizes) SizeBytes() int { + return 6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SeccompNotifSizes) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Notif)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Notif_resp)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Data)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SeccompNotifSizes) UnmarshalBytes(src []byte) []byte { + s.Notif = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.Notif_resp = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.Data = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SeccompNotifSizes) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SeccompNotifSizes) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SeccompNotifSizes) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SeccompNotifSizes) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SeccompNotifSizes) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SeccompNotifSizes) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SeccompNotifSizes) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SeccompNotifSizes) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SemInfo) SizeBytes() int { + return 40 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SemInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemMap)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemMni)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemMns)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemMnu)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemMsl)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemOpm)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemUme)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemUsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemVmx)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SemAem)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SemInfo) UnmarshalBytes(src []byte) []byte { + s.SemMap = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemMni = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemMns = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemMnu = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemMsl = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemOpm = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemUme = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemUsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemVmx = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SemAem = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SemInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SemInfo) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SemInfo) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SemInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SemInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SemInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SemInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SemInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Sembuf) SizeBytes() int { + return 6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Sembuf) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.SemNum)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.SemOp)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.SemFlg)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Sembuf) UnmarshalBytes(src []byte) []byte { + s.SemNum = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.SemOp = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.SemFlg = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Sembuf) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Sembuf) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Sembuf) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Sembuf) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Sembuf) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Sembuf) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Sembuf) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Sembuf) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopySembufSliceIn copies in a slice of Sembuf objects from the task's memory. +func CopySembufSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []Sembuf) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Sembuf)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopySembufSliceOut copies a slice of Sembuf objects to the task's memory. +func CopySembufSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []Sembuf) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Sembuf)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeSembufSlice is like Sembuf.MarshalUnsafe, but for a []Sembuf. +func MarshalUnsafeSembufSlice(src []Sembuf, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*Sembuf)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeSembufSlice is like Sembuf.UnmarshalUnsafe, but for a []Sembuf. +func UnmarshalUnsafeSembufSlice(dst []Sembuf, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*Sembuf)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *ShmInfo) SizeBytes() int { + return 44 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *ShmInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.UsedIDs)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmTot)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmRss)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmSwp)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SwapAttempts)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SwapSuccesses)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *ShmInfo) UnmarshalBytes(src []byte) []byte { + s.UsedIDs = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(s._), src[:sizeof(byte)*4]) + src = src[1*(4):] + s.ShmTot = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.ShmRss = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.ShmSwp = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SwapAttempts = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SwapSuccesses = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *ShmInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *ShmInfo) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *ShmInfo) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *ShmInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *ShmInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *ShmInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *ShmInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *ShmInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *ShmParams) SizeBytes() int { + return 40 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *ShmParams) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmMax)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmMin)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmMni)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmSeg)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmAll)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *ShmParams) UnmarshalBytes(src []byte) []byte { + s.ShmMax = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.ShmMin = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.ShmMni = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.ShmSeg = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.ShmAll = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *ShmParams) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *ShmParams) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *ShmParams) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *ShmParams) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *ShmParams) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *ShmParams) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *ShmParams) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *ShmParams) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *ShmidDS) SizeBytes() int { + return 40 + + (*IPCPerm)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *ShmidDS) MarshalBytes(dst []byte) []byte { + dst = s.ShmPerm.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmSegsz)) + dst = dst[8:] + dst = s.ShmAtime.MarshalUnsafe(dst) + dst = s.ShmDtime.MarshalUnsafe(dst) + dst = s.ShmCtime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.ShmCpid)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.ShmLpid)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.ShmNattach)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Unused4)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Unused5)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *ShmidDS) UnmarshalBytes(src []byte) []byte { + src = s.ShmPerm.UnmarshalUnsafe(src) + s.ShmSegsz = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = s.ShmAtime.UnmarshalUnsafe(src) + src = s.ShmDtime.UnmarshalUnsafe(src) + src = s.ShmCtime.UnmarshalUnsafe(src) + s.ShmCpid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.ShmLpid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.ShmNattach = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Unused4 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Unused5 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *ShmidDS) Packed() bool { + return s.ShmAtime.Packed() && s.ShmCtime.Packed() && s.ShmDtime.Packed() && s.ShmPerm.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *ShmidDS) MarshalUnsafe(dst []byte) []byte { + if s.ShmAtime.Packed() && s.ShmCtime.Packed() && s.ShmDtime.Packed() && s.ShmPerm.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type ShmidDS doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *ShmidDS) UnmarshalUnsafe(src []byte) []byte { + if s.ShmAtime.Packed() && s.ShmCtime.Packed() && s.ShmDtime.Packed() && s.ShmPerm.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type ShmidDS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *ShmidDS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.ShmAtime.Packed() && s.ShmCtime.Packed() && s.ShmDtime.Packed() && s.ShmPerm.Packed() { + // Type ShmidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *ShmidDS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *ShmidDS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.ShmAtime.Packed() && s.ShmCtime.Packed() && s.ShmDtime.Packed() && s.ShmPerm.Packed() { + // Type ShmidDS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *ShmidDS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *ShmidDS) WriteTo(writer io.Writer) (int64, error) { + if !s.ShmAtime.Packed() && s.ShmCtime.Packed() && s.ShmDtime.Packed() && s.ShmPerm.Packed() { + // Type ShmidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SigAction) SizeBytes() int { + return 24 + + (*SignalSet)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SigAction) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Handler)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Flags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Restorer)) + dst = dst[8:] + dst = s.Mask.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SigAction) UnmarshalBytes(src []byte) []byte { + s.Handler = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Flags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Restorer = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = s.Mask.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SigAction) Packed() bool { + return s.Mask.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SigAction) MarshalUnsafe(dst []byte) []byte { + if s.Mask.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SigAction doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SigAction) UnmarshalUnsafe(src []byte) []byte { + if s.Mask.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SigAction doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SigAction) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Mask.Packed() { + // Type SigAction doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SigAction) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SigAction) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Mask.Packed() { + // Type SigAction doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SigAction) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SigAction) WriteTo(writer io.Writer) (int64, error) { + if !s.Mask.Packed() { + // Type SigAction doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Sigevent) SizeBytes() int { + return 20 + + 1*44 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Sigevent) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Value)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Signo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Notify)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Tid)) + dst = dst[4:] + for idx := 0; idx < 44; idx++ { + dst[0] = byte(s.UnRemainder[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Sigevent) UnmarshalBytes(src []byte) []byte { + s.Value = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Signo = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Notify = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Tid = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 44; idx++ { + s.UnRemainder[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Sigevent) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Sigevent) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Sigevent) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Sigevent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Sigevent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Sigevent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Sigevent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Sigevent) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SignalInfo) SizeBytes() int { + return 16 + + 1*(128-16) +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SignalInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Signo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Errno)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Code)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + for idx := 0; idx < (128 - 16); idx++ { + dst[0] = byte(s.Fields[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SignalInfo) UnmarshalBytes(src []byte) []byte { + s.Signo = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Errno = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Code = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + for idx := 0; idx < (128 - 16); idx++ { + s.Fields[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SignalInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SignalInfo) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SignalInfo) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SignalInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SignalInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SignalInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SignalInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SignalInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (s *SignalSet) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SignalSet) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*s)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SignalSet) UnmarshalBytes(src []byte) []byte { + *s = SignalSet(uint64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SignalSet) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SignalSet) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SignalSet) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SignalSet) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SignalSet) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SignalSet) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SignalSet) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SignalSet) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SignalStack) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SignalStack) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Addr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Flags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Size)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SignalStack) UnmarshalBytes(src []byte) []byte { + s.Addr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + s.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SignalStack) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SignalStack) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SignalStack) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SignalStack) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SignalStack) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SignalStack) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SignalStack) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SignalStack) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SignalfdSiginfo) SizeBytes() int { + return 82 + + 1*48 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SignalfdSiginfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Signo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Errno)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Code)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.PID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.TID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Band)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Overrun)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.TrapNo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Int)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Ptr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.UTime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.STime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Addr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.AddrLSB)) + dst = dst[2:] + // Padding: dst[:sizeof(uint8)*48] ~= [48]uint8{0} + dst = dst[1*(48):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SignalfdSiginfo) UnmarshalBytes(src []byte) []byte { + s.Signo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Errno = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Code = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.PID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.TID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Band = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Overrun = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.TrapNo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Status = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Int = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Ptr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.UTime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.STime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Addr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.AddrLSB = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([48]uint8(s._), src[:sizeof(uint8)*48]) + src = src[1*(48):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SignalfdSiginfo) Packed() bool { + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SignalfdSiginfo) MarshalUnsafe(dst []byte) []byte { + // Type SignalfdSiginfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SignalfdSiginfo) UnmarshalUnsafe(src []byte) []byte { + // Type SignalfdSiginfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SignalfdSiginfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type SignalfdSiginfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SignalfdSiginfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SignalfdSiginfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type SignalfdSiginfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SignalfdSiginfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SignalfdSiginfo) WriteTo(writer io.Writer) (int64, error) { + // Type SignalfdSiginfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *ControlMessageCredentials) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *ControlMessageCredentials) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.PID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.GID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *ControlMessageCredentials) UnmarshalBytes(src []byte) []byte { + c.PID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *ControlMessageCredentials) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *ControlMessageCredentials) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *ControlMessageCredentials) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *ControlMessageCredentials) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *ControlMessageCredentials) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *ControlMessageCredentials) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *ControlMessageCredentials) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *ControlMessageCredentials) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *ControlMessageHeader) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *ControlMessageHeader) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(c.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Level)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.Type)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *ControlMessageHeader) UnmarshalBytes(src []byte) []byte { + c.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + c.Level = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + c.Type = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *ControlMessageHeader) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *ControlMessageHeader) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *ControlMessageHeader) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *ControlMessageHeader) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *ControlMessageHeader) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *ControlMessageHeader) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *ControlMessageHeader) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *ControlMessageHeader) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *ControlMessageIPPacketInfo) SizeBytes() int { + return 4 + + (*InetAddr)(nil).SizeBytes() + + (*InetAddr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *ControlMessageIPPacketInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.NIC)) + dst = dst[4:] + dst = c.LocalAddr.MarshalUnsafe(dst) + dst = c.DestinationAddr.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *ControlMessageIPPacketInfo) UnmarshalBytes(src []byte) []byte { + c.NIC = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = c.LocalAddr.UnmarshalUnsafe(src) + src = c.DestinationAddr.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *ControlMessageIPPacketInfo) Packed() bool { + return c.DestinationAddr.Packed() && c.LocalAddr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *ControlMessageIPPacketInfo) MarshalUnsafe(dst []byte) []byte { + if c.DestinationAddr.Packed() && c.LocalAddr.Packed() { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] + } + // Type ControlMessageIPPacketInfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return c.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *ControlMessageIPPacketInfo) UnmarshalUnsafe(src []byte) []byte { + if c.DestinationAddr.Packed() && c.LocalAddr.Packed() { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type ControlMessageIPPacketInfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return c.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *ControlMessageIPPacketInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !c.DestinationAddr.Packed() && c.LocalAddr.Packed() { + // Type ControlMessageIPPacketInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(c.SizeBytes()) // escapes: okay. + c.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *ControlMessageIPPacketInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *ControlMessageIPPacketInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !c.DestinationAddr.Packed() && c.LocalAddr.Packed() { + // Type ControlMessageIPPacketInfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(c.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + c.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *ControlMessageIPPacketInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *ControlMessageIPPacketInfo) WriteTo(writer io.Writer) (int64, error) { + if !c.DestinationAddr.Packed() && c.LocalAddr.Packed() { + // Type ControlMessageIPPacketInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, c.SizeBytes()) + c.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (c *ControlMessageIPv6PacketInfo) SizeBytes() int { + return 4 + + (*Inet6Addr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *ControlMessageIPv6PacketInfo) MarshalBytes(dst []byte) []byte { + dst = c.Addr.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(c.NIC)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *ControlMessageIPv6PacketInfo) UnmarshalBytes(src []byte) []byte { + src = c.Addr.UnmarshalUnsafe(src) + c.NIC = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *ControlMessageIPv6PacketInfo) Packed() bool { + return c.Addr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *ControlMessageIPv6PacketInfo) MarshalUnsafe(dst []byte) []byte { + if c.Addr.Packed() { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] + } + // Type ControlMessageIPv6PacketInfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return c.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *ControlMessageIPv6PacketInfo) UnmarshalUnsafe(src []byte) []byte { + if c.Addr.Packed() { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type ControlMessageIPv6PacketInfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return c.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *ControlMessageIPv6PacketInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !c.Addr.Packed() { + // Type ControlMessageIPv6PacketInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(c.SizeBytes()) // escapes: okay. + c.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *ControlMessageIPv6PacketInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *ControlMessageIPv6PacketInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !c.Addr.Packed() { + // Type ControlMessageIPv6PacketInfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(c.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + c.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *ControlMessageIPv6PacketInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *ControlMessageIPv6PacketInfo) WriteTo(writer io.Writer) (int64, error) { + if !c.Addr.Packed() { + // Type ControlMessageIPv6PacketInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, c.SizeBytes()) + c.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *ICMP6Filter) SizeBytes() int { + return 0 + + 4*8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *ICMP6Filter) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < 8; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.Filter[idx])) + dst = dst[4:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *ICMP6Filter) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < 8; idx++ { + i.Filter[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *ICMP6Filter) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *ICMP6Filter) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *ICMP6Filter) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *ICMP6Filter) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *ICMP6Filter) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *ICMP6Filter) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *ICMP6Filter) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *ICMP6Filter) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *Inet6Addr) SizeBytes() int { + return 1 * 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Inet6Addr) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < 16; idx++ { + dst[0] = byte(i[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Inet6Addr) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < 16; idx++ { + i[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Inet6Addr) Packed() bool { + // Array newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Inet6Addr) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&i[0]), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Inet6Addr) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Inet6Addr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Inet6Addr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Inet6Addr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Inet6Addr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Inet6Addr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *Inet6MulticastRequest) SizeBytes() int { + return 4 + + (*Inet6Addr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Inet6MulticastRequest) MarshalBytes(dst []byte) []byte { + dst = i.MulticastAddr.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.InterfaceIndex)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Inet6MulticastRequest) UnmarshalBytes(src []byte) []byte { + src = i.MulticastAddr.UnmarshalUnsafe(src) + i.InterfaceIndex = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Inet6MulticastRequest) Packed() bool { + return i.MulticastAddr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Inet6MulticastRequest) MarshalUnsafe(dst []byte) []byte { + if i.MulticastAddr.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type Inet6MulticastRequest doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Inet6MulticastRequest) UnmarshalUnsafe(src []byte) []byte { + if i.MulticastAddr.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Inet6MulticastRequest doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Inet6MulticastRequest) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.MulticastAddr.Packed() { + // Type Inet6MulticastRequest doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Inet6MulticastRequest) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Inet6MulticastRequest) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.MulticastAddr.Packed() { + // Type Inet6MulticastRequest doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Inet6MulticastRequest) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Inet6MulticastRequest) WriteTo(writer io.Writer) (int64, error) { + if !i.MulticastAddr.Packed() { + // Type Inet6MulticastRequest doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *InetAddr) SizeBytes() int { + return 1 * 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *InetAddr) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < 4; idx++ { + dst[0] = byte(i[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *InetAddr) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < 4; idx++ { + i[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *InetAddr) Packed() bool { + // Array newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *InetAddr) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&i[0]), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *InetAddr) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *InetAddr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *InetAddr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *InetAddr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *InetAddr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *InetAddr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *InetMulticastRequest) SizeBytes() int { + return 0 + + (*InetAddr)(nil).SizeBytes() + + (*InetAddr)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *InetMulticastRequest) MarshalBytes(dst []byte) []byte { + dst = i.MulticastAddr.MarshalUnsafe(dst) + dst = i.InterfaceAddr.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *InetMulticastRequest) UnmarshalBytes(src []byte) []byte { + src = i.MulticastAddr.UnmarshalUnsafe(src) + src = i.InterfaceAddr.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *InetMulticastRequest) Packed() bool { + return i.InterfaceAddr.Packed() && i.MulticastAddr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *InetMulticastRequest) MarshalUnsafe(dst []byte) []byte { + if i.InterfaceAddr.Packed() && i.MulticastAddr.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type InetMulticastRequest doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *InetMulticastRequest) UnmarshalUnsafe(src []byte) []byte { + if i.InterfaceAddr.Packed() && i.MulticastAddr.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type InetMulticastRequest doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *InetMulticastRequest) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.InterfaceAddr.Packed() && i.MulticastAddr.Packed() { + // Type InetMulticastRequest doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *InetMulticastRequest) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *InetMulticastRequest) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.InterfaceAddr.Packed() && i.MulticastAddr.Packed() { + // Type InetMulticastRequest doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *InetMulticastRequest) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *InetMulticastRequest) WriteTo(writer io.Writer) (int64, error) { + if !i.InterfaceAddr.Packed() && i.MulticastAddr.Packed() { + // Type InetMulticastRequest doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *InetMulticastRequestWithNIC) SizeBytes() int { + return 4 + + (*InetMulticastRequest)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *InetMulticastRequestWithNIC) MarshalBytes(dst []byte) []byte { + dst = i.InetMulticastRequest.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.InterfaceIndex)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *InetMulticastRequestWithNIC) UnmarshalBytes(src []byte) []byte { + src = i.InetMulticastRequest.UnmarshalUnsafe(src) + i.InterfaceIndex = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *InetMulticastRequestWithNIC) Packed() bool { + return i.InetMulticastRequest.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *InetMulticastRequestWithNIC) MarshalUnsafe(dst []byte) []byte { + if i.InetMulticastRequest.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type InetMulticastRequestWithNIC doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *InetMulticastRequestWithNIC) UnmarshalUnsafe(src []byte) []byte { + if i.InetMulticastRequest.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type InetMulticastRequestWithNIC doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *InetMulticastRequestWithNIC) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.InetMulticastRequest.Packed() { + // Type InetMulticastRequestWithNIC doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *InetMulticastRequestWithNIC) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *InetMulticastRequestWithNIC) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.InetMulticastRequest.Packed() { + // Type InetMulticastRequestWithNIC doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *InetMulticastRequestWithNIC) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *InetMulticastRequestWithNIC) WriteTo(writer io.Writer) (int64, error) { + if !i.InetMulticastRequest.Packed() { + // Type InetMulticastRequestWithNIC doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (l *Linger) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (l *Linger) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(l.OnOff)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(l.Linger)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (l *Linger) UnmarshalBytes(src []byte) []byte { + l.OnOff = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + l.Linger = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (l *Linger) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (l *Linger) MarshalUnsafe(dst []byte) []byte { + size := l.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(l), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (l *Linger) UnmarshalUnsafe(src []byte) []byte { + size := l.SizeBytes() + gohacks.Memmove(unsafe.Pointer(l), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (l *Linger) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(l))) + hdr.Len = l.SizeBytes() + hdr.Cap = l.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that l + // must live until the use above. + runtime.KeepAlive(l) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (l *Linger) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return l.CopyOutN(cc, addr, l.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (l *Linger) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(l))) + hdr.Len = l.SizeBytes() + hdr.Cap = l.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that l + // must live until the use above. + runtime.KeepAlive(l) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (l *Linger) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return l.CopyInN(cc, addr, l.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (l *Linger) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(l))) + hdr.Len = l.SizeBytes() + hdr.Cap = l.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that l + // must live until the use above. + runtime.KeepAlive(l) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockAddrInet) SizeBytes() int { + return 4 + + (*InetAddr)(nil).SizeBytes() + + 1*8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockAddrInet) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Family)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Port)) + dst = dst[2:] + dst = s.Addr.MarshalUnsafe(dst) + // Padding: dst[:sizeof(uint8)*8] ~= [8]uint8{0} + dst = dst[1*(8):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockAddrInet) UnmarshalBytes(src []byte) []byte { + s.Family = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.Port = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + src = s.Addr.UnmarshalUnsafe(src) + // Padding: ~ copy([8]uint8(s._), src[:sizeof(uint8)*8]) + src = src[1*(8):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockAddrInet) Packed() bool { + return s.Addr.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockAddrInet) MarshalUnsafe(dst []byte) []byte { + if s.Addr.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SockAddrInet doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockAddrInet) UnmarshalUnsafe(src []byte) []byte { + if s.Addr.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SockAddrInet doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockAddrInet) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Addr.Packed() { + // Type SockAddrInet doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockAddrInet) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockAddrInet) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.Addr.Packed() { + // Type SockAddrInet doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockAddrInet) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockAddrInet) WriteTo(writer io.Writer) (int64, error) { + if !s.Addr.Packed() { + // Type SockAddrInet doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockAddrInet6) SizeBytes() int { + return 12 + + 1*16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockAddrInet6) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Family)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Port)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Flowinfo)) + dst = dst[4:] + for idx := 0; idx < 16; idx++ { + dst[0] = byte(s.Addr[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Scope_id)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockAddrInet6) UnmarshalBytes(src []byte) []byte { + s.Family = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.Port = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.Flowinfo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 16; idx++ { + s.Addr[idx] = src[0] + src = src[1:] + } + s.Scope_id = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockAddrInet6) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockAddrInet6) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockAddrInet6) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockAddrInet6) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockAddrInet6) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockAddrInet6) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockAddrInet6) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockAddrInet6) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockAddrLink) SizeBytes() int { + return 12 + + 1*8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockAddrLink) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Family)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Protocol)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.InterfaceIndex)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.ARPHardwareType)) + dst = dst[2:] + dst[0] = byte(s.PacketType) + dst = dst[1:] + dst[0] = byte(s.HardwareAddrLen) + dst = dst[1:] + for idx := 0; idx < 8; idx++ { + dst[0] = byte(s.HardwareAddr[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockAddrLink) UnmarshalBytes(src []byte) []byte { + s.Family = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.Protocol = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.InterfaceIndex = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.ARPHardwareType = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + s.PacketType = src[0] + src = src[1:] + s.HardwareAddrLen = src[0] + src = src[1:] + for idx := 0; idx < 8; idx++ { + s.HardwareAddr[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockAddrLink) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockAddrLink) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockAddrLink) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockAddrLink) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockAddrLink) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockAddrLink) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockAddrLink) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockAddrLink) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SockAddrUnix) SizeBytes() int { + return 2 + + 1*UnixPathMax +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SockAddrUnix) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(s.Family)) + dst = dst[2:] + for idx := 0; idx < UnixPathMax; idx++ { + dst[0] = byte(s.Path[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SockAddrUnix) UnmarshalBytes(src []byte) []byte { + s.Family = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < UnixPathMax; idx++ { + s.Path[idx] = int8(src[0]) + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SockAddrUnix) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SockAddrUnix) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SockAddrUnix) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SockAddrUnix) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SockAddrUnix) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SockAddrUnix) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SockAddrUnix) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SockAddrUnix) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *TCPInfo) SizeBytes() int { + return 224 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *TCPInfo) MarshalBytes(dst []byte) []byte { + dst[0] = byte(t.State) + dst = dst[1:] + dst[0] = byte(t.CaState) + dst = dst[1:] + dst[0] = byte(t.Retransmits) + dst = dst[1:] + dst[0] = byte(t.Probes) + dst = dst[1:] + dst[0] = byte(t.Backoff) + dst = dst[1:] + dst[0] = byte(t.Options) + dst = dst[1:] + dst[0] = byte(t.WindowScale) + dst = dst[1:] + dst[0] = byte(t.DeliveryRateAppLimited) + dst = dst[1:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RTO)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.ATO)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.SndMss)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RcvMss)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Unacked)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Sacked)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Lost)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Retrans)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Fackets)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.LastDataSent)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.LastAckSent)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.LastDataRecv)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.LastAckRecv)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.PMTU)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RcvSsthresh)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RTT)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RTTVar)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.SndSsthresh)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.SndCwnd)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Advmss)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Reordering)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RcvRTT)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.RcvSpace)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TotalRetrans)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.PacingRate)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.MaxPacingRate)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.BytesAcked)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.BytesReceived)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.SegsOut)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.SegsIn)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.NotSentBytes)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.MinRTT)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.DataSegsIn)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.DataSegsOut)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.DeliveryRate)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.BusyTime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.RwndLimited)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.SndBufLimited)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Delivered)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.DeliveredCE)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.BytesSent)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.BytesRetrans)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.DSACKDups)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.ReordSeen)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *TCPInfo) UnmarshalBytes(src []byte) []byte { + t.State = uint8(src[0]) + src = src[1:] + t.CaState = uint8(src[0]) + src = src[1:] + t.Retransmits = uint8(src[0]) + src = src[1:] + t.Probes = uint8(src[0]) + src = src[1:] + t.Backoff = uint8(src[0]) + src = src[1:] + t.Options = uint8(src[0]) + src = src[1:] + t.WindowScale = uint8(src[0]) + src = src[1:] + t.DeliveryRateAppLimited = uint8(src[0]) + src = src[1:] + t.RTO = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.ATO = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.SndMss = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.RcvMss = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Unacked = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Sacked = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Lost = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Retrans = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Fackets = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LastDataSent = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LastAckSent = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LastDataRecv = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LastAckRecv = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.PMTU = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.RcvSsthresh = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.RTT = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.RTTVar = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.SndSsthresh = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.SndCwnd = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Advmss = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Reordering = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.RcvRTT = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.RcvSpace = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TotalRetrans = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.PacingRate = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.MaxPacingRate = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.BytesAcked = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.BytesReceived = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.SegsOut = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.SegsIn = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.NotSentBytes = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.MinRTT = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.DataSegsIn = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.DataSegsOut = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.DeliveryRate = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.BusyTime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.RwndLimited = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.SndBufLimited = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.Delivered = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.DeliveredCE = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.BytesSent = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.BytesRetrans = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.DSACKDups = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.ReordSeen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *TCPInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *TCPInfo) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *TCPInfo) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *TCPInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *TCPInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *TCPInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *TCPInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *TCPInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *Tpacket2Hdr) SizeBytes() int { + return 28 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *Tpacket2Hdr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpStatus)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpLen)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpSnaplen)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(t.TpMac)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(t.TpNet)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpSec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpNSec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(t.TpVlanTci)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(t.TpVlanTpid)) + dst = dst[2:] + // Padding: dst[:sizeof(uint8)*4] ~= [4]uint8{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *Tpacket2Hdr) UnmarshalBytes(src []byte) []byte { + t.TpStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpLen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpSnaplen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpMac = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + t.TpNet = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + t.TpSec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpNSec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpVlanTci = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + t.TpVlanTpid = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([4]uint8(t._), src[:sizeof(uint8)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *Tpacket2Hdr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *Tpacket2Hdr) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *Tpacket2Hdr) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *Tpacket2Hdr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *Tpacket2Hdr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *Tpacket2Hdr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *Tpacket2Hdr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *Tpacket2Hdr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *TpacketHdr) SizeBytes() int { + return 28 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *TpacketHdr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(t.TpStatus)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpLen)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpSnaplen)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(t.TpMac)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(t.TpNet)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpSec)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpUsec)) + dst = dst[4:] + // Padding: dst[:sizeof(uint8)*4] ~= [4]uint8{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *TpacketHdr) UnmarshalBytes(src []byte) []byte { + t.TpStatus = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + t.TpLen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpSnaplen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpMac = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + t.TpNet = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + t.TpSec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpUsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]uint8(t._), src[:sizeof(uint8)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *TpacketHdr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *TpacketHdr) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *TpacketHdr) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *TpacketHdr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *TpacketHdr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *TpacketHdr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *TpacketHdr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *TpacketHdr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *TpacketReq) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *TpacketReq) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpBlockSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpBlockNr)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpFrameSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.TpFrameNr)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *TpacketReq) UnmarshalBytes(src []byte) []byte { + t.TpBlockSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpBlockNr = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpFrameSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.TpFrameNr = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *TpacketReq) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *TpacketReq) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *TpacketReq) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *TpacketReq) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *TpacketReq) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *TpacketReq) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *TpacketReq) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *TpacketReq) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *TpacketStats) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *TpacketStats) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Packets)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.Dropped)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *TpacketStats) UnmarshalBytes(src []byte) []byte { + t.Packets = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.Dropped = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *TpacketStats) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *TpacketStats) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *TpacketStats) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *TpacketStats) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *TpacketStats) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *TpacketStats) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *TpacketStats) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *TpacketStats) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (c *ClockT) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (c *ClockT) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*c)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (c *ClockT) UnmarshalBytes(src []byte) []byte { + *c = ClockT(int64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (c *ClockT) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (c *ClockT) MarshalUnsafe(dst []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(c), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (c *ClockT) UnmarshalUnsafe(src []byte) []byte { + size := c.SizeBytes() + gohacks.Memmove(unsafe.Pointer(c), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (c *ClockT) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (c *ClockT) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyOutN(cc, addr, c.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (c *ClockT) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (c *ClockT) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return c.CopyInN(cc, addr, c.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (c *ClockT) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(c))) + hdr.Len = c.SizeBytes() + hdr.Cap = c.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that c + // must live until the use above. + runtime.KeepAlive(c) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *ItimerVal) SizeBytes() int { + return 0 + + (*Timeval)(nil).SizeBytes() + + (*Timeval)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *ItimerVal) MarshalBytes(dst []byte) []byte { + dst = i.Interval.MarshalUnsafe(dst) + dst = i.Value.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *ItimerVal) UnmarshalBytes(src []byte) []byte { + src = i.Interval.UnmarshalUnsafe(src) + src = i.Value.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *ItimerVal) Packed() bool { + return i.Interval.Packed() && i.Value.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *ItimerVal) MarshalUnsafe(dst []byte) []byte { + if i.Interval.Packed() && i.Value.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type ItimerVal doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *ItimerVal) UnmarshalUnsafe(src []byte) []byte { + if i.Interval.Packed() && i.Value.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type ItimerVal doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *ItimerVal) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Interval.Packed() && i.Value.Packed() { + // Type ItimerVal doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *ItimerVal) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *ItimerVal) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Interval.Packed() && i.Value.Packed() { + // Type ItimerVal doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *ItimerVal) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *ItimerVal) WriteTo(writer io.Writer) (int64, error) { + if !i.Interval.Packed() && i.Value.Packed() { + // Type ItimerVal doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *Itimerspec) SizeBytes() int { + return 0 + + (*Timespec)(nil).SizeBytes() + + (*Timespec)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Itimerspec) MarshalBytes(dst []byte) []byte { + dst = i.Interval.MarshalUnsafe(dst) + dst = i.Value.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Itimerspec) UnmarshalBytes(src []byte) []byte { + src = i.Interval.UnmarshalUnsafe(src) + src = i.Value.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Itimerspec) Packed() bool { + return i.Interval.Packed() && i.Value.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Itimerspec) MarshalUnsafe(dst []byte) []byte { + if i.Interval.Packed() && i.Value.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type Itimerspec doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Itimerspec) UnmarshalUnsafe(src []byte) []byte { + if i.Interval.Packed() && i.Value.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Itimerspec doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Itimerspec) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Interval.Packed() && i.Value.Packed() { + // Type Itimerspec doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Itimerspec) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Itimerspec) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.Interval.Packed() && i.Value.Packed() { + // Type Itimerspec doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Itimerspec) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Itimerspec) WriteTo(writer io.Writer) (int64, error) { + if !i.Interval.Packed() && i.Value.Packed() { + // Type Itimerspec doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (sxts *StatxTimestamp) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (sxts *StatxTimestamp) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(sxts.Sec)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(sxts.Nsec)) + dst = dst[4:] + // Padding: dst[:sizeof(int32)] ~= int32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (sxts *StatxTimestamp) UnmarshalBytes(src []byte) []byte { + sxts.Sec = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + sxts.Nsec = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ int32 ~= src[:sizeof(int32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (sxts *StatxTimestamp) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (sxts *StatxTimestamp) MarshalUnsafe(dst []byte) []byte { + size := sxts.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(sxts), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (sxts *StatxTimestamp) UnmarshalUnsafe(src []byte) []byte { + size := sxts.SizeBytes() + gohacks.Memmove(unsafe.Pointer(sxts), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (sxts *StatxTimestamp) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(sxts))) + hdr.Len = sxts.SizeBytes() + hdr.Cap = sxts.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that sxts + // must live until the use above. + runtime.KeepAlive(sxts) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (sxts *StatxTimestamp) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return sxts.CopyOutN(cc, addr, sxts.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (sxts *StatxTimestamp) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(sxts))) + hdr.Len = sxts.SizeBytes() + hdr.Cap = sxts.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that sxts + // must live until the use above. + runtime.KeepAlive(sxts) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (sxts *StatxTimestamp) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return sxts.CopyInN(cc, addr, sxts.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (sxts *StatxTimestamp) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(sxts))) + hdr.Len = sxts.SizeBytes() + hdr.Cap = sxts.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that sxts + // must live until the use above. + runtime.KeepAlive(sxts) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (t *TimeT) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *TimeT) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*t)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *TimeT) UnmarshalBytes(src []byte) []byte { + *t = TimeT(int64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *TimeT) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *TimeT) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *TimeT) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *TimeT) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *TimeT) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *TimeT) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *TimeT) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *TimeT) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (t *TimerID) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *TimerID) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*t)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *TimerID) UnmarshalBytes(src []byte) []byte { + *t = TimerID(int32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *TimerID) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *TimerID) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *TimerID) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *TimerID) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *TimerID) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *TimerID) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *TimerID) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *TimerID) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (ts *Timespec) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (ts *Timespec) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(ts.Sec)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(ts.Nsec)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (ts *Timespec) UnmarshalBytes(src []byte) []byte { + ts.Sec = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + ts.Nsec = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (ts *Timespec) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (ts *Timespec) MarshalUnsafe(dst []byte) []byte { + size := ts.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(ts), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (ts *Timespec) UnmarshalUnsafe(src []byte) []byte { + size := ts.SizeBytes() + gohacks.Memmove(unsafe.Pointer(ts), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (ts *Timespec) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(ts))) + hdr.Len = ts.SizeBytes() + hdr.Cap = ts.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that ts + // must live until the use above. + runtime.KeepAlive(ts) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (ts *Timespec) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ts.CopyOutN(cc, addr, ts.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (ts *Timespec) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(ts))) + hdr.Len = ts.SizeBytes() + hdr.Cap = ts.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that ts + // must live until the use above. + runtime.KeepAlive(ts) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (ts *Timespec) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return ts.CopyInN(cc, addr, ts.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (ts *Timespec) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(ts))) + hdr.Len = ts.SizeBytes() + hdr.Cap = ts.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that ts + // must live until the use above. + runtime.KeepAlive(ts) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyTimespecSliceIn copies in a slice of Timespec objects from the task's memory. +func CopyTimespecSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []Timespec) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Timespec)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyTimespecSliceOut copies a slice of Timespec objects to the task's memory. +func CopyTimespecSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []Timespec) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Timespec)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeTimespecSlice is like Timespec.MarshalUnsafe, but for a []Timespec. +func MarshalUnsafeTimespecSlice(src []Timespec, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*Timespec)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeTimespecSlice is like Timespec.UnmarshalUnsafe, but for a []Timespec. +func UnmarshalUnsafeTimespecSlice(dst []Timespec, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*Timespec)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (tv *Timeval) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (tv *Timeval) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(tv.Sec)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(tv.Usec)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (tv *Timeval) UnmarshalBytes(src []byte) []byte { + tv.Sec = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + tv.Usec = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (tv *Timeval) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (tv *Timeval) MarshalUnsafe(dst []byte) []byte { + size := tv.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(tv), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (tv *Timeval) UnmarshalUnsafe(src []byte) []byte { + size := tv.SizeBytes() + gohacks.Memmove(unsafe.Pointer(tv), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (tv *Timeval) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(tv))) + hdr.Len = tv.SizeBytes() + hdr.Cap = tv.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that tv + // must live until the use above. + runtime.KeepAlive(tv) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (tv *Timeval) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return tv.CopyOutN(cc, addr, tv.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (tv *Timeval) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(tv))) + hdr.Len = tv.SizeBytes() + hdr.Cap = tv.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that tv + // must live until the use above. + runtime.KeepAlive(tv) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (tv *Timeval) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return tv.CopyInN(cc, addr, tv.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (tv *Timeval) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(tv))) + hdr.Len = tv.SizeBytes() + hdr.Cap = tv.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that tv + // must live until the use above. + runtime.KeepAlive(tv) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyTimevalSliceIn copies in a slice of Timeval objects from the task's memory. +func CopyTimevalSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []Timeval) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Timeval)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyTimevalSliceOut copies a slice of Timeval objects to the task's memory. +func CopyTimevalSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []Timeval) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Timeval)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeTimevalSlice is like Timeval.MarshalUnsafe, but for a []Timeval. +func MarshalUnsafeTimevalSlice(src []Timeval, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*Timeval)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeTimevalSlice is like Timeval.UnmarshalUnsafe, but for a []Timeval. +func UnmarshalUnsafeTimevalSlice(dst []Timeval, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*Timeval)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *Tms) SizeBytes() int { + return 0 + + (*ClockT)(nil).SizeBytes() + + (*ClockT)(nil).SizeBytes() + + (*ClockT)(nil).SizeBytes() + + (*ClockT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *Tms) MarshalBytes(dst []byte) []byte { + dst = t.UTime.MarshalUnsafe(dst) + dst = t.STime.MarshalUnsafe(dst) + dst = t.CUTime.MarshalUnsafe(dst) + dst = t.CSTime.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *Tms) UnmarshalBytes(src []byte) []byte { + src = t.UTime.UnmarshalUnsafe(src) + src = t.STime.UnmarshalUnsafe(src) + src = t.CUTime.UnmarshalUnsafe(src) + src = t.CSTime.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *Tms) Packed() bool { + return t.CSTime.Packed() && t.CUTime.Packed() && t.STime.Packed() && t.UTime.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *Tms) MarshalUnsafe(dst []byte) []byte { + if t.CSTime.Packed() && t.CUTime.Packed() && t.STime.Packed() && t.UTime.Packed() { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] + } + // Type Tms doesn't have a packed layout in memory, fallback to MarshalBytes. + return t.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *Tms) UnmarshalUnsafe(src []byte) []byte { + if t.CSTime.Packed() && t.CUTime.Packed() && t.STime.Packed() && t.UTime.Packed() { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Tms doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return t.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *Tms) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !t.CSTime.Packed() && t.CUTime.Packed() && t.STime.Packed() && t.UTime.Packed() { + // Type Tms doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(t.SizeBytes()) // escapes: okay. + t.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *Tms) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *Tms) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !t.CSTime.Packed() && t.CUTime.Packed() && t.STime.Packed() && t.UTime.Packed() { + // Type Tms doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(t.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + t.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *Tms) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *Tms) WriteTo(writer io.Writer) (int64, error) { + if !t.CSTime.Packed() && t.CUTime.Packed() && t.STime.Packed() && t.UTime.Packed() { + // Type Tms doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, t.SizeBytes()) + t.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (u *Utime) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *Utime) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(u.Actime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(u.Modtime)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *Utime) UnmarshalBytes(src []byte) []byte { + u.Actime = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + u.Modtime = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *Utime) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *Utime) MarshalUnsafe(dst []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *Utime) UnmarshalUnsafe(src []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *Utime) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *Utime) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *Utime) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *Utime) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *Utime) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *Termios) SizeBytes() int { + return 17 + + 1*NumControlCharacters +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *Termios) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.InputFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.OutputFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.ControlFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.LocalFlags)) + dst = dst[4:] + dst[0] = byte(t.LineDiscipline) + dst = dst[1:] + for idx := 0; idx < NumControlCharacters; idx++ { + dst[0] = byte(t.ControlCharacters[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *Termios) UnmarshalBytes(src []byte) []byte { + t.InputFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.OutputFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.ControlFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LocalFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LineDiscipline = uint8(src[0]) + src = src[1:] + for idx := 0; idx < NumControlCharacters; idx++ { + t.ControlCharacters[idx] = uint8(src[0]) + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *Termios) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *Termios) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *Termios) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *Termios) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *Termios) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *Termios) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *Termios) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *Termios) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (w *WindowSize) SizeBytes() int { + return 4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (w *WindowSize) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Rows)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Cols)) + dst = dst[2:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (w *WindowSize) UnmarshalBytes(src []byte) []byte { + w.Rows = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + w.Cols = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([4]byte(w._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (w *WindowSize) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (w *WindowSize) MarshalUnsafe(dst []byte) []byte { + size := w.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(w), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (w *WindowSize) UnmarshalUnsafe(src []byte) []byte { + size := w.SizeBytes() + gohacks.Memmove(unsafe.Pointer(w), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (w *WindowSize) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) + hdr.Len = w.SizeBytes() + hdr.Cap = w.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that w + // must live until the use above. + runtime.KeepAlive(w) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (w *WindowSize) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return w.CopyOutN(cc, addr, w.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (w *WindowSize) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) + hdr.Len = w.SizeBytes() + hdr.Cap = w.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that w + // must live until the use above. + runtime.KeepAlive(w) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (w *WindowSize) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return w.CopyInN(cc, addr, w.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (w *WindowSize) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) + hdr.Len = w.SizeBytes() + hdr.Cap = w.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that w + // must live until the use above. + runtime.KeepAlive(w) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (w *Winsize) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (w *Winsize) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Row)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Col)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Xpixel)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Ypixel)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (w *Winsize) UnmarshalBytes(src []byte) []byte { + w.Row = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + w.Col = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + w.Xpixel = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + w.Ypixel = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (w *Winsize) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (w *Winsize) MarshalUnsafe(dst []byte) []byte { + size := w.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(w), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (w *Winsize) UnmarshalUnsafe(src []byte) []byte { + size := w.SizeBytes() + gohacks.Memmove(unsafe.Pointer(w), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (w *Winsize) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) + hdr.Len = w.SizeBytes() + hdr.Cap = w.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that w + // must live until the use above. + runtime.KeepAlive(w) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (w *Winsize) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return w.CopyOutN(cc, addr, w.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (w *Winsize) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) + hdr.Len = w.SizeBytes() + hdr.Cap = w.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that w + // must live until the use above. + runtime.KeepAlive(w) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (w *Winsize) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return w.CopyInN(cc, addr, w.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (w *Winsize) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) + hdr.Len = w.SizeBytes() + hdr.Cap = w.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that w + // must live until the use above. + runtime.KeepAlive(w) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (u *UtsName) SizeBytes() int { + return 0 + + 1*(UTSLen+1) + + 1*(UTSLen+1) + + 1*(UTSLen+1) + + 1*(UTSLen+1) + + 1*(UTSLen+1) + + 1*(UTSLen+1) +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *UtsName) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < (UTSLen + 1); idx++ { + dst[0] = byte(u.Sysname[idx]) + dst = dst[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + dst[0] = byte(u.Nodename[idx]) + dst = dst[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + dst[0] = byte(u.Release[idx]) + dst = dst[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + dst[0] = byte(u.Version[idx]) + dst = dst[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + dst[0] = byte(u.Machine[idx]) + dst = dst[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + dst[0] = byte(u.Domainname[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *UtsName) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < (UTSLen + 1); idx++ { + u.Sysname[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + u.Nodename[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + u.Release[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + u.Version[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + u.Machine[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < (UTSLen + 1); idx++ { + u.Domainname[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *UtsName) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *UtsName) MarshalUnsafe(dst []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *UtsName) UnmarshalUnsafe(src []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *UtsName) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *UtsName) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *UtsName) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *UtsName) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *UtsName) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIODeviceInfo) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIODeviceInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.NumRegions)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.NumIrqs)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.CapOffset)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.pad)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIODeviceInfo) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.NumRegions = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.NumIrqs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.CapOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.pad = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIODeviceInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIODeviceInfo) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIODeviceInfo) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIODeviceInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIODeviceInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIODeviceInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIODeviceInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIODeviceInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIOIommuType1DmaMap) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIOIommuType1DmaMap) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.Vaddr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.IOVa)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.Size)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIOIommuType1DmaMap) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Vaddr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + v.IOVa = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + v.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIOIommuType1DmaMap) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIOIommuType1DmaMap) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIOIommuType1DmaMap) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIOIommuType1DmaMap) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIOIommuType1DmaMap) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIOIommuType1DmaMap) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIOIommuType1DmaMap) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIOIommuType1DmaMap) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIOIommuType1DmaUnmap) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIOIommuType1DmaUnmap) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.IOVa)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.Size)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIOIommuType1DmaUnmap) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.IOVa = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + v.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIOIommuType1DmaUnmap) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIOIommuType1DmaUnmap) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIOIommuType1DmaUnmap) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIOIommuType1DmaUnmap) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIOIommuType1DmaUnmap) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIOIommuType1DmaUnmap) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIOIommuType1DmaUnmap) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIOIommuType1DmaUnmap) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIOIrqInfo) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIOIrqInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Count)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIOIrqInfo) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Count = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIOIrqInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIOIrqInfo) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIOIrqInfo) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIOIrqInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIOIrqInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIOIrqInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIOIrqInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIOIrqInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIOIrqSet) SizeBytes() int { + return 20 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIOIrqSet) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Start)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Count)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIOIrqSet) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Start = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Count = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIOIrqSet) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIOIrqSet) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIOIrqSet) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIOIrqSet) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIOIrqSet) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIOIrqSet) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIOIrqSet) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIOIrqSet) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIORegionInfo) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIORegionInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.capOffset)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.Offset)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIORegionInfo) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.capOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + v.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIORegionInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIORegionInfo) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIORegionInfo) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIORegionInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIORegionInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIORegionInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIORegionInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIORegionInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} diff --git a/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go b/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go new file mode 100644 index 0000000..3b47f4b --- /dev/null +++ b/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go @@ -0,0 +1,760 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +// If there are issues with build constraint aggregation, see +// tools/go_marshal/gomarshal/generator.go:writeHeader(). The constraints here +// come from the input set of files used to generate this file. This input set +// is filtered based on pre-defined file suffixes related to build constraints, +// see tools/defs.bzl:calculate_sets(). + +//go:build amd64 && amd64 && amd64 && amd64 && amd64 && amd64 +// +build amd64,amd64,amd64,amd64,amd64,amd64 + +package linux + +import ( + "io" + "reflect" + "runtime" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Marshallable types used by this file. +var ( + _ marshal.Marshallable = (*EpollEvent)(nil) + _ marshal.Marshallable = (*IPCPerm)(nil) + _ marshal.Marshallable = (*PtraceRegs)(nil) + _ marshal.Marshallable = (*SemidDS)(nil) + _ marshal.Marshallable = (*Stat)(nil) + _ marshal.Marshallable = (*TimeT)(nil) + _ marshal.Marshallable = (*Timespec)(nil) +) + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *EpollEvent) SizeBytes() int { + return 4 + + 4*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *EpollEvent) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Events)) + dst = dst[4:] + for idx := 0; idx < 2; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Data[idx])) + dst = dst[4:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *EpollEvent) UnmarshalBytes(src []byte) []byte { + e.Events = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 2; idx++ { + e.Data[idx] = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *EpollEvent) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *EpollEvent) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *EpollEvent) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *EpollEvent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *EpollEvent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *EpollEvent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *EpollEvent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *EpollEvent) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyEpollEventSliceIn copies in a slice of EpollEvent objects from the task's memory. +func CopyEpollEventSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []EpollEvent) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyEpollEventSliceOut copies a slice of EpollEvent objects to the task's memory. +func CopyEpollEventSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []EpollEvent) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeEpollEventSlice is like EpollEvent.MarshalUnsafe, but for a []EpollEvent. +func MarshalUnsafeEpollEventSlice(src []EpollEvent, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*EpollEvent)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeEpollEventSlice is like EpollEvent.UnmarshalUnsafe, but for a []EpollEvent. +func UnmarshalUnsafeEpollEventSlice(dst []EpollEvent, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*EpollEvent)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Stat) SizeBytes() int { + return 72 + + (*Timespec)(nil).SizeBytes() + + (*Timespec)(nil).SizeBytes() + + (*Timespec)(nil).SizeBytes() + + 8*3 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Stat) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Dev)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Ino)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Nlink)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Mode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.GID)) + dst = dst[4:] + // Padding: dst[:sizeof(int32)] ~= int32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Rdev)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Blksize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Blocks)) + dst = dst[8:] + dst = s.ATime.MarshalUnsafe(dst) + dst = s.MTime.MarshalUnsafe(dst) + dst = s.CTime.MarshalUnsafe(dst) + // Padding: dst[:sizeof(int64)*3] ~= [3]int64{0} + dst = dst[8*(3):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Stat) UnmarshalBytes(src []byte) []byte { + s.Dev = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Ino = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Nlink = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ int32 ~= src[:sizeof(int32)] + src = src[4:] + s.Rdev = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Size = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Blksize = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Blocks = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = s.ATime.UnmarshalUnsafe(src) + src = s.MTime.UnmarshalUnsafe(src) + src = s.CTime.UnmarshalUnsafe(src) + // Padding: ~ copy([3]int64(s._), src[:sizeof(int64)*3]) + src = src[8*(3):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Stat) Packed() bool { + return s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Stat) MarshalUnsafe(dst []byte) []byte { + if s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type Stat doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Stat) UnmarshalUnsafe(src []byte) []byte { + if s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Stat doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Stat) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + // Type Stat doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Stat) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Stat) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + // Type Stat doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Stat) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Stat) WriteTo(writer io.Writer) (int64, error) { + if !s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + // Type Stat doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *PtraceRegs) SizeBytes() int { + return 216 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *PtraceRegs) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R15)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R14)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R13)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R12)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rbp)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rbx)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R11)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R10)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R9)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.R8)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rax)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rcx)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rdx)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rsi)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rdi)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Orig_rax)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rip)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Cs)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Eflags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Rsp)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Ss)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Fs_base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Gs_base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Ds)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Es)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Fs)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Gs)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *PtraceRegs) UnmarshalBytes(src []byte) []byte { + p.R15 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R14 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R13 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R12 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rbp = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rbx = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R11 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R10 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R9 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.R8 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rax = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rcx = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rdx = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rsi = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rdi = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Orig_rax = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rip = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Cs = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Eflags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Rsp = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Ss = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Fs_base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Gs_base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Ds = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Es = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Fs = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Gs = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *PtraceRegs) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *PtraceRegs) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *PtraceRegs) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *PtraceRegs) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *PtraceRegs) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *PtraceRegs) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *PtraceRegs) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *PtraceRegs) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SemidDS) SizeBytes() int { + return 40 + + (*IPCPerm)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SemidDS) MarshalBytes(dst []byte) []byte { + dst = s.SemPerm.MarshalUnsafe(dst) + dst = s.SemOTime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.unused1)) + dst = dst[8:] + dst = s.SemCTime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.unused2)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SemNSems)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.unused3)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.unused4)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SemidDS) UnmarshalBytes(src []byte) []byte { + src = s.SemPerm.UnmarshalUnsafe(src) + src = s.SemOTime.UnmarshalUnsafe(src) + s.unused1 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = s.SemCTime.UnmarshalUnsafe(src) + s.unused2 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SemNSems = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.unused3 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.unused4 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SemidDS) Packed() bool { + return s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SemidDS) MarshalUnsafe(dst []byte) []byte { + if s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SemidDS doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SemidDS) UnmarshalUnsafe(src []byte) []byte { + if s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SemidDS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SemidDS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + // Type SemidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SemidDS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SemidDS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + // Type SemidDS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SemidDS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SemidDS) WriteTo(writer io.Writer) (int64, error) { + if !s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + // Type SemidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} diff --git a/pkg/abi/linux/linux_amd64_state_autogen.go b/pkg/abi/linux/linux_amd64_state_autogen.go new file mode 100644 index 0000000..8457806 --- /dev/null +++ b/pkg/abi/linux/linux_amd64_state_autogen.go @@ -0,0 +1,119 @@ +// automatically generated by stateify. + +//go:build amd64 && amd64 && amd64 && amd64 && amd64 && amd64 +// +build amd64,amd64,amd64,amd64,amd64,amd64 + +package linux + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *PtraceRegs) StateTypeName() string { + return "pkg/abi/linux.PtraceRegs" +} + +func (p *PtraceRegs) StateFields() []string { + return []string{ + "R15", + "R14", + "R13", + "R12", + "Rbp", + "Rbx", + "R11", + "R10", + "R9", + "R8", + "Rax", + "Rcx", + "Rdx", + "Rsi", + "Rdi", + "Orig_rax", + "Rip", + "Cs", + "Eflags", + "Rsp", + "Ss", + "Fs_base", + "Gs_base", + "Ds", + "Es", + "Fs", + "Gs", + } +} + +func (p *PtraceRegs) beforeSave() {} + +// +checklocksignore +func (p *PtraceRegs) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.R15) + stateSinkObject.Save(1, &p.R14) + stateSinkObject.Save(2, &p.R13) + stateSinkObject.Save(3, &p.R12) + stateSinkObject.Save(4, &p.Rbp) + stateSinkObject.Save(5, &p.Rbx) + stateSinkObject.Save(6, &p.R11) + stateSinkObject.Save(7, &p.R10) + stateSinkObject.Save(8, &p.R9) + stateSinkObject.Save(9, &p.R8) + stateSinkObject.Save(10, &p.Rax) + stateSinkObject.Save(11, &p.Rcx) + stateSinkObject.Save(12, &p.Rdx) + stateSinkObject.Save(13, &p.Rsi) + stateSinkObject.Save(14, &p.Rdi) + stateSinkObject.Save(15, &p.Orig_rax) + stateSinkObject.Save(16, &p.Rip) + stateSinkObject.Save(17, &p.Cs) + stateSinkObject.Save(18, &p.Eflags) + stateSinkObject.Save(19, &p.Rsp) + stateSinkObject.Save(20, &p.Ss) + stateSinkObject.Save(21, &p.Fs_base) + stateSinkObject.Save(22, &p.Gs_base) + stateSinkObject.Save(23, &p.Ds) + stateSinkObject.Save(24, &p.Es) + stateSinkObject.Save(25, &p.Fs) + stateSinkObject.Save(26, &p.Gs) +} + +func (p *PtraceRegs) afterLoad(context.Context) {} + +// +checklocksignore +func (p *PtraceRegs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.R15) + stateSourceObject.Load(1, &p.R14) + stateSourceObject.Load(2, &p.R13) + stateSourceObject.Load(3, &p.R12) + stateSourceObject.Load(4, &p.Rbp) + stateSourceObject.Load(5, &p.Rbx) + stateSourceObject.Load(6, &p.R11) + stateSourceObject.Load(7, &p.R10) + stateSourceObject.Load(8, &p.R9) + stateSourceObject.Load(9, &p.R8) + stateSourceObject.Load(10, &p.Rax) + stateSourceObject.Load(11, &p.Rcx) + stateSourceObject.Load(12, &p.Rdx) + stateSourceObject.Load(13, &p.Rsi) + stateSourceObject.Load(14, &p.Rdi) + stateSourceObject.Load(15, &p.Orig_rax) + stateSourceObject.Load(16, &p.Rip) + stateSourceObject.Load(17, &p.Cs) + stateSourceObject.Load(18, &p.Eflags) + stateSourceObject.Load(19, &p.Rsp) + stateSourceObject.Load(20, &p.Ss) + stateSourceObject.Load(21, &p.Fs_base) + stateSourceObject.Load(22, &p.Gs_base) + stateSourceObject.Load(23, &p.Ds) + stateSourceObject.Load(24, &p.Es) + stateSourceObject.Load(25, &p.Fs) + stateSourceObject.Load(26, &p.Gs) +} + +func init() { + state.Register((*PtraceRegs)(nil)) +} diff --git a/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go b/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go new file mode 100644 index 0000000..884e946 --- /dev/null +++ b/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go @@ -0,0 +1,673 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +// If there are issues with build constraint aggregation, see +// tools/go_marshal/gomarshal/generator.go:writeHeader(). The constraints here +// come from the input set of files used to generate this file. This input set +// is filtered based on pre-defined file suffixes related to build constraints, +// see tools/defs.bzl:calculate_sets(). + +//go:build arm64 && arm64 && arm64 && arm64 && arm64 +// +build arm64,arm64,arm64,arm64,arm64 + +package linux + +import ( + "io" + "reflect" + "runtime" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Marshallable types used by this file. +var ( + _ marshal.Marshallable = (*EpollEvent)(nil) + _ marshal.Marshallable = (*IPCPerm)(nil) + _ marshal.Marshallable = (*PtraceRegs)(nil) + _ marshal.Marshallable = (*SemidDS)(nil) + _ marshal.Marshallable = (*Stat)(nil) + _ marshal.Marshallable = (*TimeT)(nil) + _ marshal.Marshallable = (*Timespec)(nil) +) + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *EpollEvent) SizeBytes() int { + return 8 + + 4*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *EpollEvent) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Events)) + dst = dst[4:] + // Padding: dst[:sizeof(int32)] ~= int32(0) + dst = dst[4:] + for idx := 0; idx < 2; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Data[idx])) + dst = dst[4:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *EpollEvent) UnmarshalBytes(src []byte) []byte { + e.Events = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ int32 ~= src[:sizeof(int32)] + src = src[4:] + for idx := 0; idx < 2; idx++ { + e.Data[idx] = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *EpollEvent) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *EpollEvent) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *EpollEvent) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *EpollEvent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *EpollEvent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *EpollEvent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *EpollEvent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *EpollEvent) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyEpollEventSliceIn copies in a slice of EpollEvent objects from the task's memory. +func CopyEpollEventSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []EpollEvent) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyEpollEventSliceOut copies a slice of EpollEvent objects to the task's memory. +func CopyEpollEventSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []EpollEvent) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeEpollEventSlice is like EpollEvent.MarshalUnsafe, but for a []EpollEvent. +func MarshalUnsafeEpollEventSlice(src []EpollEvent, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*EpollEvent)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeEpollEventSlice is like EpollEvent.UnmarshalUnsafe, but for a []EpollEvent. +func UnmarshalUnsafeEpollEventSlice(dst []EpollEvent, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*EpollEvent)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *Stat) SizeBytes() int { + return 72 + + (*Timespec)(nil).SizeBytes() + + (*Timespec)(nil).SizeBytes() + + (*Timespec)(nil).SizeBytes() + + 4*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *Stat) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Dev)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Ino)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Mode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Nlink)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.UID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.GID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Rdev)) + dst = dst[8:] + // Padding: dst[:sizeof(uint64)] ~= uint64(0) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Blksize)) + dst = dst[4:] + // Padding: dst[:sizeof(int32)] ~= int32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.Blocks)) + dst = dst[8:] + dst = s.ATime.MarshalUnsafe(dst) + dst = s.MTime.MarshalUnsafe(dst) + dst = s.CTime.MarshalUnsafe(dst) + // Padding: dst[:sizeof(int32)*2] ~= [2]int32{0} + dst = dst[4*(2):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *Stat) UnmarshalBytes(src []byte) []byte { + s.Dev = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Ino = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Mode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Nlink = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.UID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.GID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.Rdev = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + // Padding: var _ uint64 ~= src[:sizeof(uint64)] + src = src[8:] + s.Size = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.Blksize = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ int32 ~= src[:sizeof(int32)] + src = src[4:] + s.Blocks = int64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = s.ATime.UnmarshalUnsafe(src) + src = s.MTime.UnmarshalUnsafe(src) + src = s.CTime.UnmarshalUnsafe(src) + // Padding: ~ copy([2]int32(s._), src[:sizeof(int32)*2]) + src = src[4*(2):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *Stat) Packed() bool { + return s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *Stat) MarshalUnsafe(dst []byte) []byte { + if s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type Stat doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *Stat) UnmarshalUnsafe(src []byte) []byte { + if s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type Stat doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *Stat) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + // Type Stat doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *Stat) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *Stat) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + // Type Stat doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *Stat) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *Stat) WriteTo(writer io.Writer) (int64, error) { + if !s.ATime.Packed() && s.CTime.Packed() && s.MTime.Packed() { + // Type Stat doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *PtraceRegs) SizeBytes() int { + return 24 + + 8*31 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *PtraceRegs) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < 31; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Regs[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Sp)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Pc)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Pstate)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *PtraceRegs) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < 31; idx++ { + p.Regs[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + p.Sp = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Pc = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Pstate = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *PtraceRegs) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *PtraceRegs) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *PtraceRegs) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *PtraceRegs) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *PtraceRegs) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *PtraceRegs) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *PtraceRegs) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *PtraceRegs) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SemidDS) SizeBytes() int { + return 24 + + (*IPCPerm)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() + + (*TimeT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SemidDS) MarshalBytes(dst []byte) []byte { + dst = s.SemPerm.MarshalUnsafe(dst) + dst = s.SemOTime.MarshalUnsafe(dst) + dst = s.SemCTime.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SemNSems)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.unused3)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.unused4)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SemidDS) UnmarshalBytes(src []byte) []byte { + src = s.SemPerm.UnmarshalUnsafe(src) + src = s.SemOTime.UnmarshalUnsafe(src) + src = s.SemCTime.UnmarshalUnsafe(src) + s.SemNSems = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.unused3 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.unused4 = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SemidDS) Packed() bool { + return s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SemidDS) MarshalUnsafe(dst []byte) []byte { + if s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] + } + // Type SemidDS doesn't have a packed layout in memory, fallback to MarshalBytes. + return s.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SemidDS) UnmarshalUnsafe(src []byte) []byte { + if s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type SemidDS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return s.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SemidDS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + // Type SemidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + s.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SemidDS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SemidDS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + // Type SemidDS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(s.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + s.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SemidDS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SemidDS) WriteTo(writer io.Writer) (int64, error) { + if !s.SemCTime.Packed() && s.SemOTime.Packed() && s.SemPerm.Packed() { + // Type SemidDS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, s.SizeBytes()) + s.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} diff --git a/pkg/abi/linux/linux_arm64_state_autogen.go b/pkg/abi/linux/linux_arm64_state_autogen.go new file mode 100644 index 0000000..e90c1a7 --- /dev/null +++ b/pkg/abi/linux/linux_arm64_state_autogen.go @@ -0,0 +1,50 @@ +// automatically generated by stateify. + +//go:build arm64 && arm64 && arm64 && arm64 && arm64 +// +build arm64,arm64,arm64,arm64,arm64 + +package linux + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *PtraceRegs) StateTypeName() string { + return "pkg/abi/linux.PtraceRegs" +} + +func (p *PtraceRegs) StateFields() []string { + return []string{ + "Regs", + "Sp", + "Pc", + "Pstate", + } +} + +func (p *PtraceRegs) beforeSave() {} + +// +checklocksignore +func (p *PtraceRegs) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.Regs) + stateSinkObject.Save(1, &p.Sp) + stateSinkObject.Save(2, &p.Pc) + stateSinkObject.Save(3, &p.Pstate) +} + +func (p *PtraceRegs) afterLoad(context.Context) {} + +// +checklocksignore +func (p *PtraceRegs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.Regs) + stateSourceObject.Load(1, &p.Sp) + stateSourceObject.Load(2, &p.Pc) + stateSourceObject.Load(3, &p.Pstate) +} + +func init() { + state.Register((*PtraceRegs)(nil)) +} diff --git a/pkg/abi/linux/linux_state_autogen.go b/pkg/abi/linux/linux_state_autogen.go new file mode 100644 index 0000000..d212e77 --- /dev/null +++ b/pkg/abi/linux/linux_state_autogen.go @@ -0,0 +1,599 @@ +// automatically generated by stateify. + +package linux + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (i *IOEvent) StateTypeName() string { + return "pkg/abi/linux.IOEvent" +} + +func (i *IOEvent) StateFields() []string { + return []string{ + "Data", + "Obj", + "Result", + "Result2", + } +} + +func (i *IOEvent) beforeSave() {} + +// +checklocksignore +func (i *IOEvent) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Data) + stateSinkObject.Save(1, &i.Obj) + stateSinkObject.Save(2, &i.Result) + stateSinkObject.Save(3, &i.Result2) +} + +func (i *IOEvent) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IOEvent) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Data) + stateSourceObject.Load(1, &i.Obj) + stateSourceObject.Load(2, &i.Result) + stateSourceObject.Load(3, &i.Result2) +} + +func (b *BPFInstruction) StateTypeName() string { + return "pkg/abi/linux.BPFInstruction" +} + +func (b *BPFInstruction) StateFields() []string { + return []string{ + "OpCode", + "JumpIfTrue", + "JumpIfFalse", + "K", + } +} + +func (b *BPFInstruction) beforeSave() {} + +// +checklocksignore +func (b *BPFInstruction) StateSave(stateSinkObject state.Sink) { + b.beforeSave() + stateSinkObject.Save(0, &b.OpCode) + stateSinkObject.Save(1, &b.JumpIfTrue) + stateSinkObject.Save(2, &b.JumpIfFalse) + stateSinkObject.Save(3, &b.K) +} + +func (b *BPFInstruction) afterLoad(context.Context) {} + +// +checklocksignore +func (b *BPFInstruction) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &b.OpCode) + stateSourceObject.Load(1, &b.JumpIfTrue) + stateSourceObject.Load(2, &b.JumpIfFalse) + stateSourceObject.Load(3, &b.K) +} + +func (f *FUSEHeaderIn) StateTypeName() string { + return "pkg/abi/linux.FUSEHeaderIn" +} + +func (f *FUSEHeaderIn) StateFields() []string { + return []string{ + "Len", + "Opcode", + "Unique", + "NodeID", + "UID", + "GID", + "PID", + } +} + +func (f *FUSEHeaderIn) beforeSave() {} + +// +checklocksignore +func (f *FUSEHeaderIn) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.Len) + stateSinkObject.Save(1, &f.Opcode) + stateSinkObject.Save(2, &f.Unique) + stateSinkObject.Save(3, &f.NodeID) + stateSinkObject.Save(4, &f.UID) + stateSinkObject.Save(5, &f.GID) + stateSinkObject.Save(6, &f.PID) +} + +func (f *FUSEHeaderIn) afterLoad(context.Context) {} + +// +checklocksignore +func (f *FUSEHeaderIn) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.Len) + stateSourceObject.Load(1, &f.Opcode) + stateSourceObject.Load(2, &f.Unique) + stateSourceObject.Load(3, &f.NodeID) + stateSourceObject.Load(4, &f.UID) + stateSourceObject.Load(5, &f.GID) + stateSourceObject.Load(6, &f.PID) +} + +func (f *FUSEHeaderOut) StateTypeName() string { + return "pkg/abi/linux.FUSEHeaderOut" +} + +func (f *FUSEHeaderOut) StateFields() []string { + return []string{ + "Len", + "Error", + "Unique", + } +} + +func (f *FUSEHeaderOut) beforeSave() {} + +// +checklocksignore +func (f *FUSEHeaderOut) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.Len) + stateSinkObject.Save(1, &f.Error) + stateSinkObject.Save(2, &f.Unique) +} + +func (f *FUSEHeaderOut) afterLoad(context.Context) {} + +// +checklocksignore +func (f *FUSEHeaderOut) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.Len) + stateSourceObject.Load(1, &f.Error) + stateSourceObject.Load(2, &f.Unique) +} + +func (i *IOUringCqe) StateTypeName() string { + return "pkg/abi/linux.IOUringCqe" +} + +func (i *IOUringCqe) StateFields() []string { + return []string{ + "UserData", + "Res", + "Flags", + } +} + +func (i *IOUringCqe) beforeSave() {} + +// +checklocksignore +func (i *IOUringCqe) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.UserData) + stateSinkObject.Save(1, &i.Res) + stateSinkObject.Save(2, &i.Flags) +} + +func (i *IOUringCqe) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IOUringCqe) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.UserData) + stateSourceObject.Load(1, &i.Res) + stateSourceObject.Load(2, &i.Flags) +} + +func (i *IOUring) StateTypeName() string { + return "pkg/abi/linux.IOUring" +} + +func (i *IOUring) StateFields() []string { + return []string{ + "Head", + "Tail", + } +} + +func (i *IOUring) beforeSave() {} + +// +checklocksignore +func (i *IOUring) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Head) + stateSinkObject.Save(1, &i.Tail) +} + +func (i *IOUring) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IOUring) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Head) + stateSourceObject.Load(1, &i.Tail) +} + +func (i *IORings) StateTypeName() string { + return "pkg/abi/linux.IORings" +} + +func (i *IORings) StateFields() []string { + return []string{ + "Sq", + "Cq", + "SqRingMask", + "CqRingMask", + "SqRingEntries", + "CqRingEntries", + "sqDropped", + "sqFlags", + "cqFlags", + "CqOverflow", + } +} + +func (i *IORings) beforeSave() {} + +// +checklocksignore +func (i *IORings) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Sq) + stateSinkObject.Save(1, &i.Cq) + stateSinkObject.Save(2, &i.SqRingMask) + stateSinkObject.Save(3, &i.CqRingMask) + stateSinkObject.Save(4, &i.SqRingEntries) + stateSinkObject.Save(5, &i.CqRingEntries) + stateSinkObject.Save(6, &i.sqDropped) + stateSinkObject.Save(7, &i.sqFlags) + stateSinkObject.Save(8, &i.cqFlags) + stateSinkObject.Save(9, &i.CqOverflow) +} + +func (i *IORings) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IORings) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Sq) + stateSourceObject.Load(1, &i.Cq) + stateSourceObject.Load(2, &i.SqRingMask) + stateSourceObject.Load(3, &i.CqRingMask) + stateSourceObject.Load(4, &i.SqRingEntries) + stateSourceObject.Load(5, &i.CqRingEntries) + stateSourceObject.Load(6, &i.sqDropped) + stateSourceObject.Load(7, &i.sqFlags) + stateSourceObject.Load(8, &i.cqFlags) + stateSourceObject.Load(9, &i.CqOverflow) +} + +func (i *IOUringSqe) StateTypeName() string { + return "pkg/abi/linux.IOUringSqe" +} + +func (i *IOUringSqe) StateFields() []string { + return []string{ + "Opcode", + "Flags", + "IoPrio", + "Fd", + "OffOrAddrOrCmdOp", + "AddrOrSpliceOff", + "Len", + "specialFlags", + "UserData", + "BufIndexOrGroup", + "personality", + "spliceFDOrFileIndex", + "addr3", + } +} + +func (i *IOUringSqe) beforeSave() {} + +// +checklocksignore +func (i *IOUringSqe) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Opcode) + stateSinkObject.Save(1, &i.Flags) + stateSinkObject.Save(2, &i.IoPrio) + stateSinkObject.Save(3, &i.Fd) + stateSinkObject.Save(4, &i.OffOrAddrOrCmdOp) + stateSinkObject.Save(5, &i.AddrOrSpliceOff) + stateSinkObject.Save(6, &i.Len) + stateSinkObject.Save(7, &i.specialFlags) + stateSinkObject.Save(8, &i.UserData) + stateSinkObject.Save(9, &i.BufIndexOrGroup) + stateSinkObject.Save(10, &i.personality) + stateSinkObject.Save(11, &i.spliceFDOrFileIndex) + stateSinkObject.Save(12, &i.addr3) +} + +func (i *IOUringSqe) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IOUringSqe) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Opcode) + stateSourceObject.Load(1, &i.Flags) + stateSourceObject.Load(2, &i.IoPrio) + stateSourceObject.Load(3, &i.Fd) + stateSourceObject.Load(4, &i.OffOrAddrOrCmdOp) + stateSourceObject.Load(5, &i.AddrOrSpliceOff) + stateSourceObject.Load(6, &i.Len) + stateSourceObject.Load(7, &i.specialFlags) + stateSourceObject.Load(8, &i.UserData) + stateSourceObject.Load(9, &i.BufIndexOrGroup) + stateSourceObject.Load(10, &i.personality) + stateSourceObject.Load(11, &i.spliceFDOrFileIndex) + stateSourceObject.Load(12, &i.addr3) +} + +func (s *SigAction) StateTypeName() string { + return "pkg/abi/linux.SigAction" +} + +func (s *SigAction) StateFields() []string { + return []string{ + "Handler", + "Flags", + "Restorer", + "Mask", + } +} + +func (s *SigAction) beforeSave() {} + +// +checklocksignore +func (s *SigAction) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.Handler) + stateSinkObject.Save(1, &s.Flags) + stateSinkObject.Save(2, &s.Restorer) + stateSinkObject.Save(3, &s.Mask) +} + +func (s *SigAction) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SigAction) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.Handler) + stateSourceObject.Load(1, &s.Flags) + stateSourceObject.Load(2, &s.Restorer) + stateSourceObject.Load(3, &s.Mask) +} + +func (s *SignalStack) StateTypeName() string { + return "pkg/abi/linux.SignalStack" +} + +func (s *SignalStack) StateFields() []string { + return []string{ + "Addr", + "Flags", + "Size", + } +} + +func (s *SignalStack) beforeSave() {} + +// +checklocksignore +func (s *SignalStack) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.Addr) + stateSinkObject.Save(1, &s.Flags) + stateSinkObject.Save(2, &s.Size) +} + +func (s *SignalStack) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SignalStack) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.Addr) + stateSourceObject.Load(1, &s.Flags) + stateSourceObject.Load(2, &s.Size) +} + +func (s *SignalInfo) StateTypeName() string { + return "pkg/abi/linux.SignalInfo" +} + +func (s *SignalInfo) StateFields() []string { + return []string{ + "Signo", + "Errno", + "Code", + "Fields", + } +} + +func (s *SignalInfo) beforeSave() {} + +// +checklocksignore +func (s *SignalInfo) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.Signo) + stateSinkObject.Save(1, &s.Errno) + stateSinkObject.Save(2, &s.Code) + stateSinkObject.Save(3, &s.Fields) +} + +func (s *SignalInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SignalInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.Signo) + stateSourceObject.Load(1, &s.Errno) + stateSourceObject.Load(2, &s.Code) + stateSourceObject.Load(3, &s.Fields) +} + +func (c *ControlMessageIPPacketInfo) StateTypeName() string { + return "pkg/abi/linux.ControlMessageIPPacketInfo" +} + +func (c *ControlMessageIPPacketInfo) StateFields() []string { + return []string{ + "NIC", + "LocalAddr", + "DestinationAddr", + } +} + +func (c *ControlMessageIPPacketInfo) beforeSave() {} + +// +checklocksignore +func (c *ControlMessageIPPacketInfo) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.NIC) + stateSinkObject.Save(1, &c.LocalAddr) + stateSinkObject.Save(2, &c.DestinationAddr) +} + +func (c *ControlMessageIPPacketInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (c *ControlMessageIPPacketInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.NIC) + stateSourceObject.Load(1, &c.LocalAddr) + stateSourceObject.Load(2, &c.DestinationAddr) +} + +func (c *ControlMessageIPv6PacketInfo) StateTypeName() string { + return "pkg/abi/linux.ControlMessageIPv6PacketInfo" +} + +func (c *ControlMessageIPv6PacketInfo) StateFields() []string { + return []string{ + "Addr", + "NIC", + } +} + +func (c *ControlMessageIPv6PacketInfo) beforeSave() {} + +// +checklocksignore +func (c *ControlMessageIPv6PacketInfo) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.Addr) + stateSinkObject.Save(1, &c.NIC) +} + +func (c *ControlMessageIPv6PacketInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (c *ControlMessageIPv6PacketInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.Addr) + stateSourceObject.Load(1, &c.NIC) +} + +func (i *ICMP6Filter) StateTypeName() string { + return "pkg/abi/linux.ICMP6Filter" +} + +func (i *ICMP6Filter) StateFields() []string { + return []string{ + "Filter", + } +} + +func (i *ICMP6Filter) beforeSave() {} + +// +checklocksignore +func (i *ICMP6Filter) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Filter) +} + +func (i *ICMP6Filter) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMP6Filter) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Filter) +} + +func (t *KernelTermios) StateTypeName() string { + return "pkg/abi/linux.KernelTermios" +} + +func (t *KernelTermios) StateFields() []string { + return []string{ + "InputFlags", + "OutputFlags", + "ControlFlags", + "LocalFlags", + "LineDiscipline", + "ControlCharacters", + "InputSpeed", + "OutputSpeed", + } +} + +func (t *KernelTermios) beforeSave() {} + +// +checklocksignore +func (t *KernelTermios) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.InputFlags) + stateSinkObject.Save(1, &t.OutputFlags) + stateSinkObject.Save(2, &t.ControlFlags) + stateSinkObject.Save(3, &t.LocalFlags) + stateSinkObject.Save(4, &t.LineDiscipline) + stateSinkObject.Save(5, &t.ControlCharacters) + stateSinkObject.Save(6, &t.InputSpeed) + stateSinkObject.Save(7, &t.OutputSpeed) +} + +func (t *KernelTermios) afterLoad(context.Context) {} + +// +checklocksignore +func (t *KernelTermios) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.InputFlags) + stateSourceObject.Load(1, &t.OutputFlags) + stateSourceObject.Load(2, &t.ControlFlags) + stateSourceObject.Load(3, &t.LocalFlags) + stateSourceObject.Load(4, &t.LineDiscipline) + stateSourceObject.Load(5, &t.ControlCharacters) + stateSourceObject.Load(6, &t.InputSpeed) + stateSourceObject.Load(7, &t.OutputSpeed) +} + +func (w *WindowSize) StateTypeName() string { + return "pkg/abi/linux.WindowSize" +} + +func (w *WindowSize) StateFields() []string { + return []string{ + "Rows", + "Cols", + } +} + +func (w *WindowSize) beforeSave() {} + +// +checklocksignore +func (w *WindowSize) StateSave(stateSinkObject state.Sink) { + w.beforeSave() + stateSinkObject.Save(0, &w.Rows) + stateSinkObject.Save(1, &w.Cols) +} + +func (w *WindowSize) afterLoad(context.Context) {} + +// +checklocksignore +func (w *WindowSize) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &w.Rows) + stateSourceObject.Load(1, &w.Cols) +} + +func init() { + state.Register((*IOEvent)(nil)) + state.Register((*BPFInstruction)(nil)) + state.Register((*FUSEHeaderIn)(nil)) + state.Register((*FUSEHeaderOut)(nil)) + state.Register((*IOUringCqe)(nil)) + state.Register((*IOUring)(nil)) + state.Register((*IORings)(nil)) + state.Register((*IOUringSqe)(nil)) + state.Register((*SigAction)(nil)) + state.Register((*SignalStack)(nil)) + state.Register((*SignalInfo)(nil)) + state.Register((*ControlMessageIPPacketInfo)(nil)) + state.Register((*ControlMessageIPv6PacketInfo)(nil)) + state.Register((*ICMP6Filter)(nil)) + state.Register((*KernelTermios)(nil)) + state.Register((*WindowSize)(nil)) +} diff --git a/pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go b/pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go new file mode 100644 index 0000000..399c294 --- /dev/null +++ b/pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go @@ -0,0 +1,3 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +package linux diff --git a/pkg/abi/linux/linux_unsafe_state_autogen.go b/pkg/abi/linux/linux_unsafe_state_autogen.go new file mode 100644 index 0000000..bddf5a7 --- /dev/null +++ b/pkg/abi/linux/linux_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package linux diff --git a/pkg/abi/linux/membarrier.go b/pkg/abi/linux/membarrier.go new file mode 100644 index 0000000..4f6021a --- /dev/null +++ b/pkg/abi/linux/membarrier.go @@ -0,0 +1,34 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// membarrier(2) commands, from include/uapi/linux/membarrier.h. +const ( + MEMBARRIER_CMD_QUERY = 0 + MEMBARRIER_CMD_GLOBAL = (1 << 0) + MEMBARRIER_CMD_GLOBAL_EXPEDITED = (1 << 1) + MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED = (1 << 2) + MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3) + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4) + MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 5) + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 6) + MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ = (1 << 7) + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ = (1 << 8) +) + +// membarrier(2) flags, from include/uapi/linux/membarrier.h. +const ( + MEMBARRIER_CMD_FLAG_CPU = (1 << 0) +) diff --git a/pkg/abi/linux/mm.go b/pkg/abi/linux/mm.go new file mode 100644 index 0000000..cad6251 --- /dev/null +++ b/pkg/abi/linux/mm.go @@ -0,0 +1,160 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// Protections for mmap(2). +const ( + PROT_NONE = 0 + PROT_READ = 1 << 0 + PROT_WRITE = 1 << 1 + PROT_EXEC = 1 << 2 + PROT_SEM = 1 << 3 + PROT_GROWSDOWN = 1 << 24 + PROT_GROWSUP = 1 << 25 +) + +// Flags for mmap(2). +const ( + MAP_SHARED = 1 << 0 + MAP_PRIVATE = 1 << 1 + MAP_DROPPABLE = 1 << 3 + MAP_FIXED = 1 << 4 + MAP_ANONYMOUS = 1 << 5 + MAP_32BIT = 1 << 6 // arch/x86/include/uapi/asm/mman.h + MAP_GROWSDOWN = 1 << 8 + MAP_DENYWRITE = 1 << 11 + MAP_EXECUTABLE = 1 << 12 + MAP_LOCKED = 1 << 13 + MAP_NORESERVE = 1 << 14 + MAP_POPULATE = 1 << 15 + MAP_NONBLOCK = 1 << 16 + MAP_STACK = 1 << 17 + MAP_HUGETLB = 1 << 18 +) + +// Flags for mremap(2). +const ( + MREMAP_MAYMOVE = 1 << 0 + MREMAP_FIXED = 1 << 1 +) + +// Flags for mlock2(2). +const ( + MLOCK_ONFAULT = 0x01 +) + +// Flags for mlockall(2). +const ( + MCL_CURRENT = 1 + MCL_FUTURE = 2 + MCL_ONFAULT = 4 +) + +// Advice for madvise(2). +const ( + MADV_NORMAL = 0 + MADV_RANDOM = 1 + MADV_SEQUENTIAL = 2 + MADV_WILLNEED = 3 + MADV_DONTNEED = 4 + MADV_REMOVE = 9 + MADV_DONTFORK = 10 + MADV_DOFORK = 11 + MADV_MERGEABLE = 12 + MADV_UNMERGEABLE = 13 + MADV_HUGEPAGE = 14 + MADV_NOHUGEPAGE = 15 + MADV_DONTDUMP = 16 + MADV_DODUMP = 17 + MADV_HWPOISON = 100 + MADV_SOFT_OFFLINE = 101 + MADV_NOMAJFAULT = 200 + MADV_DONTCHGME = 201 +) + +// Flags for msync(2). +const ( + MS_ASYNC = 1 << 0 + MS_INVALIDATE = 1 << 1 + MS_SYNC = 1 << 2 +) + +// NumaPolicy is the NUMA memory policy for a memory range. See numa(7). +// +// +marshal +type NumaPolicy int32 + +// Policies for get_mempolicy(2)/set_mempolicy(2). +const ( + MPOL_DEFAULT NumaPolicy = 0 + MPOL_PREFERRED NumaPolicy = 1 + MPOL_BIND NumaPolicy = 2 + MPOL_INTERLEAVE NumaPolicy = 3 + MPOL_LOCAL NumaPolicy = 4 + MPOL_MAX NumaPolicy = 5 +) + +// Flags for get_mempolicy(2). +const ( + MPOL_F_NODE = 1 << 0 + MPOL_F_ADDR = 1 << 1 + MPOL_F_MEMS_ALLOWED = 1 << 2 +) + +// Flags for set_mempolicy(2). +const ( + MPOL_F_RELATIVE_NODES = 1 << 14 + MPOL_F_STATIC_NODES = 1 << 15 + + MPOL_MODE_FLAGS = (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES) +) + +// Flags for mbind(2). +const ( + MPOL_MF_STRICT = 1 << 0 + MPOL_MF_MOVE = 1 << 1 + MPOL_MF_MOVE_ALL = 1 << 2 + + MPOL_MF_VALID = MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL +) + +// TaskSize is the address space size. +var TaskSize = func() uintptr { + pageSize := uintptr(unix.Getpagesize()) + for _, s := range feasibleTaskSizes { + // mmap returns ENOMEM if addr is greater than TASK_SIZE, + // otherwise it returns EINVAL, because addr isn't aligned to + // the page size. + _, _, errno := unix.RawSyscall6( + unix.SYS_MMAP, + s-pageSize-1, + 512, + uintptr(unix.PROT_NONE), + uintptr(unix.MAP_ANONYMOUS|unix.MAP_PRIVATE|unix.MAP_FIXED), 0, 0) + if errno == unix.EINVAL { + return s + } + if errno != unix.ENOMEM { + panic(fmt.Sprintf("mmap returned unexpected error: %d", errno)) + } + } + panic("None of the address space sizes could be successfully mmaped") +}() diff --git a/pkg/abi/linux/mm_amd64.go b/pkg/abi/linux/mm_amd64.go new file mode 100644 index 0000000..5faa2fa --- /dev/null +++ b/pkg/abi/linux/mm_amd64.go @@ -0,0 +1,33 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package linux + +// TASK_SIZE can be one of two values, corresponding to 4-level and 5-level +// paging. +// +// The array has to be sorted in decreasing order. +var feasibleTaskSizes = []uintptr{0xfffffffffff000, 0x7ffffffff000} + +// Page fault error codes +const ( + X86_PF_PROT = 1 << iota + X86_PF_WRITE + X86_PF_USER + X86_PF_RSVD + X86_PF_INSTR +) diff --git a/pkg/abi/linux/mm_arm64.go b/pkg/abi/linux/mm_arm64.go new file mode 100644 index 0000000..2d36f86 --- /dev/null +++ b/pkg/abi/linux/mm_arm64.go @@ -0,0 +1,25 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package linux + +// Only 4K page size is supported on arm64. In this case, TASK_SIZE can +// be one of three values, corresponding to 3-level, 4-level and +// 5-level paging. +// +// The array has to be sorted in decreasing order. +var feasibleTaskSizes = []uintptr{1 << 52, 1 << 48, 1 << 39} diff --git a/pkg/abi/linux/mqueue.go b/pkg/abi/linux/mqueue.go new file mode 100644 index 0000000..79b463e --- /dev/null +++ b/pkg/abi/linux/mqueue.go @@ -0,0 +1,55 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Default values for POSIX message queues. Source: +// include/linux/ipc_namespace.h +const ( + DFLT_QUEUESMAX = 256 + MIN_MSGMAX = 1 + DFLT_MSG uint = 10 + DFLT_MSGMAX = 10 + HARD_MSGMAX = 65536 + MIN_MSGSIZEMAX = 128 + DFLT_MSGSIZE uint = 8192 + DFLT_MSGSIZEMAX = 8192 + HARD_MSGSIZEMAX = (16 * 1024 * 1024) +) + +// Maximum values for a message queue. Source: include/uapi/linux/mqueue.h +const ( + MQ_PRIO_MAX = 32768 + MQ_BYTES_MAX = 819200 +) + +// Codes used by mq_notify. Source: include/uapi/linux/mqueue.h +const ( + NOTIFY_NONE = 0 + NOTIFY_WOKENUP = 1 + NOTIFY_REMOVED = 2 + + NOTIFY_COOKIE_LEN = 32 +) + +// MqAttr is equivalent to struct mq_attr. Source: include/uapi/linux/mqueue.h +// +// +marshal +type MqAttr struct { + MqFlags int64 // Message queue flags. + MqMaxmsg int64 // Maximum number of messages. + MqMsgsize int64 // Maximum message size. + MqCurmsgs int64 // Number of messages currently queued. + _ [4]int64 // Ignored for input, zeroed for output. +} diff --git a/pkg/abi/linux/msgqueue.go b/pkg/abi/linux/msgqueue.go new file mode 100644 index 0000000..f283a66 --- /dev/null +++ b/pkg/abi/linux/msgqueue.go @@ -0,0 +1,108 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "github.com/sagernet/gvisor/pkg/marshal/primitive" +) + +// Linux-specific control commands. Source: include/uapi/linux/msg.h +const ( + MSG_STAT = 11 + MSG_INFO = 12 + MSG_STAT_ANY = 13 +) + +// msgrcv(2) options. Source: include/uapi/linux/msg.h +const ( + MSG_NOERROR = 0o10000 // No error if message is too big. + MSG_EXCEPT = 0o20000 // Receive any message except of specified type. + MSG_COPY = 0o40000 // Copy (not remove) all queue messages. +) + +// System-wide limits for message queues. Source: include/uapi/linux/msg.h +const ( + MSGMNI = 32000 // Maximum number of message queue identifiers. + MSGMAX = 8192 // Maximum size of message (bytes). + MSGMNB = 16384 // Default max size of a message queue. +) + +// System-wide limits. Unused. Source: include/uapi/linux/msg.h +const ( + MSGPOOL = (MSGMNI * MSGMNB / 1024) + MSGTQL = MSGMNB + MSGMAP = MSGMNB + MSGSSZ = 16 + + // MSGSEG is simplified due to the inexistance of a ternary operator. + MSGSEG = 0xffff +) + +// MsqidDS is equivalent to struct msqid64_ds. Source: +// include/uapi/asm-generic/shmbuf.h +// +// +marshal +type MsqidDS struct { + MsgPerm IPCPerm // IPC permissions. + MsgStime TimeT // Last msgsnd time. + MsgRtime TimeT // Last msgrcv time. + MsgCtime TimeT // Last change time. + MsgCbytes uint64 // Current number of bytes on the queue. + MsgQnum uint64 // Number of messages in the queue. + MsgQbytes uint64 // Max number of bytes in the queue. + MsgLspid int32 // PID of last msgsnd. + MsgLrpid int32 // PID of last msgrcv. + unused4 uint64 + unused5 uint64 +} + +// MsgBuf is equivalent to struct msgbuf. Source: include/uapi/linux/msg.h +// +// +marshal dynamic +type MsgBuf struct { + Type primitive.Int64 + Text primitive.ByteSlice +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (b *MsgBuf) SizeBytes() int { + return b.Type.SizeBytes() + b.Text.SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (b *MsgBuf) MarshalBytes(dst []byte) []byte { + dst = b.Type.MarshalUnsafe(dst) + return b.Text.MarshalBytes(dst) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (b *MsgBuf) UnmarshalBytes(src []byte) []byte { + src = b.Type.UnmarshalUnsafe(src) + return b.Text.UnmarshalBytes(src) +} + +// MsgInfo is equivalent to struct msginfo. Source: include/uapi/linux/msg.h +// +// +marshal +type MsgInfo struct { + MsgPool int32 + MsgMap int32 + MsgMax int32 + MsgMnb int32 + MsgMni int32 + MsgSsz int32 + MsgTql int32 + MsgSeg uint16 `marshal:"unaligned"` +} diff --git a/pkg/abi/linux/netdevice.go b/pkg/abi/linux/netdevice.go new file mode 100644 index 0000000..924bd88 --- /dev/null +++ b/pkg/abi/linux/netdevice.go @@ -0,0 +1,128 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +const ( + // IFNAMSIZ is the size of the name field for IFReq. + IFNAMSIZ = 16 +) + +// IFReq is an interface request. +// +// +marshal +type IFReq struct { + // IFName is an encoded name, normally null-terminated. This should be + // accessed via the Name and SetName functions. + IFName [IFNAMSIZ]byte + + // Data is the union of the following structures: + // + // struct sockaddr ifr_addr; + // struct sockaddr ifr_dstaddr; + // struct sockaddr ifr_broadaddr; + // struct sockaddr ifr_netmask; + // struct sockaddr ifr_hwaddr; + // short ifr_flags; + // int ifr_ifindex; + // int ifr_metric; + // int ifr_mtu; + // struct ifmap ifr_map; + // char ifr_slave[IFNAMSIZ]; + // char ifr_newname[IFNAMSIZ]; + // char *ifr_data; + Data [24]byte +} + +// Name returns the name. +func (ifr *IFReq) Name() string { + for c := 0; c < len(ifr.IFName); c++ { + if ifr.IFName[c] == 0 { + return string(ifr.IFName[:c]) + } + } + return string(ifr.IFName[:]) +} + +// SetName sets the name. +func (ifr *IFReq) SetName(name string) { + n := copy(ifr.IFName[:], []byte(name)) + clear(ifr.IFName[n:]) +} + +// SizeOfIFReq is the binary size of an IFReq struct (40 bytes). +var SizeOfIFReq = (*IFReq)(nil).SizeBytes() + +// IFMap contains interface hardware parameters. +type IFMap struct { + MemStart uint64 + MemEnd uint64 + BaseAddr int16 + IRQ byte + DMA byte + Port byte + _ [3]byte // Pad to sizeof(struct ifmap). +} + +// IFConf is used to return a list of interfaces and their addresses. See +// netdevice(7) and struct ifconf for more detail on its use. +// +// +marshal +type IFConf struct { + Len int32 + _ [4]byte // Pad to sizeof(struct ifconf). + Ptr uint64 +} + +// SizeOfIFConf is the binary size of an IFConf struct (16 bytes). +var SizeOfIFConf = (*IFConf)(nil).SizeBytes() + +// EthtoolCmd is a marshallable type to be able to easily copyin the +// the command for an SIOCETHTOOL ioctl. +// +// +marshal +type EthtoolCmd uint32 + +const ( + // ETHTOOL_GFEATURES is the command to SIOCETHTOOL to query device + // features. + // See: + ETHTOOL_GFEATURES EthtoolCmd = 0x3a +) + +// EthtoolGFeatures is used to return a list of device features. +// See: +// +// +marshal +type EthtoolGFeatures struct { + Cmd uint32 + Size uint32 +} + +// EthtoolGetFeaturesBlock is used to return state of upto 32 device +// features. +// See: +// +// +marshal +type EthtoolGetFeaturesBlock struct { + Available uint32 + Requested uint32 + Active uint32 + NeverChanged uint32 +} + +const ( + // LOOPBACK_IFINDEX is defined in include/net/flow.h. + LOOPBACK_IFINDEX = 1 +) diff --git a/pkg/abi/linux/netfilter.go b/pkg/abi/linux/netfilter.go new file mode 100644 index 0000000..5033ee4 --- /dev/null +++ b/pkg/abi/linux/netfilter.go @@ -0,0 +1,796 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "github.com/sagernet/gvisor/pkg/marshal" + "github.com/sagernet/gvisor/pkg/marshal/primitive" +) + +// This file contains structures required to support netfilter, specifically +// the iptables tool. + +// Hooks into the network stack. These correspond to values in +// include/uapi/linux/netfilter.h. +const ( + NF_INET_PRE_ROUTING = 0 + NF_INET_LOCAL_IN = 1 + NF_INET_FORWARD = 2 + NF_INET_LOCAL_OUT = 3 + NF_INET_POST_ROUTING = 4 + NF_INET_NUMHOOKS = 5 + NF_INET_INGRESS = NF_INET_NUMHOOKS +) + +const ( + NF_NETDEV_INGRESS = iota + NF_NETDEV_EGRESS + NF_NETDEV_NUMHOOKS +) + +// Protocol families (address families). These correspond to values in +// include/uapi/linux/netfilter.h. +const ( + NFPROTO_UNSPEC = 0 + NFPROTO_INET = 1 + NFPROTO_IPV4 = 2 + NFPROTO_ARP = 3 + NFPROTO_NETDEV = 5 + NFPROTO_BRIDGE = 7 + NFPROTO_IPV6 = 10 +) + +// Verdicts that can be returned by targets. These correspond to values in +// include/uapi/linux/netfilter.h +const ( + NF_DROP = 0 + NF_ACCEPT = 1 + NF_STOLEN = 2 + NF_QUEUE = 3 + NF_REPEAT = 4 + NF_STOP = 5 + NF_MAX_VERDICT = NF_STOP + // NF_RETURN is defined in include/uapi/linux/netfilter/x_tables.h. + NF_RETURN = -NF_REPEAT - 1 +) + +// VerdictStrings maps int verdicts to the strings they represent. It is used +// for debugging. +var VerdictStrings = map[int32]string{ + -NF_DROP - 1: "DROP", + -NF_ACCEPT - 1: "ACCEPT", + -NF_QUEUE - 1: "QUEUE", + NF_RETURN: "RETURN", +} + +// Socket options for SOL_SOCKET. These correspond to values in +// include/uapi/linux/netfilter_ipv4/ip_tables.h. +const ( + IPT_BASE_CTL = 64 + IPT_SO_SET_REPLACE = IPT_BASE_CTL + IPT_SO_SET_ADD_COUNTERS = IPT_BASE_CTL + 1 + IPT_SO_SET_MAX = IPT_SO_SET_ADD_COUNTERS + + IPT_SO_GET_INFO = IPT_BASE_CTL + IPT_SO_GET_ENTRIES = IPT_BASE_CTL + 1 + IPT_SO_GET_REVISION_MATCH = IPT_BASE_CTL + 2 + IPT_SO_GET_REVISION_TARGET = IPT_BASE_CTL + 3 + IPT_SO_GET_MAX = IPT_SO_GET_REVISION_TARGET +) + +// Socket option for SOL_IP. This corresponds to the value in +// include/uapi/linux/netfilter_ipv4.h. +const ( + SO_ORIGINAL_DST = 80 +) + +// Name lengths. These correspond to values in +// include/uapi/linux/netfilter/x_tables.h. +const ( + XT_FUNCTION_MAXNAMELEN = 30 + XT_EXTENSION_MAXNAMELEN = 29 + XT_TABLE_MAXNAMELEN = 32 +) + +// IPTEntry is an iptable rule. It corresponds to struct ipt_entry in +// include/uapi/linux/netfilter_ipv4/ip_tables.h. +// +// +marshal +type IPTEntry struct { + // IP is used to filter packets based on the IP header. + IP IPTIP + + // NFCache relates to kernel-internal caching and isn't used by + // userspace. + NFCache uint32 + + // TargetOffset is the byte offset from the beginning of this IPTEntry + // to the start of the entry's target. + TargetOffset uint16 + + // NextOffset is the byte offset from the beginning of this IPTEntry to + // the start of the next entry. It is thus also the size of the entry. + NextOffset uint16 + + // Comeback is a return pointer. It is not used by userspace. + Comeback uint32 + + // Counters holds the packet and byte counts for this rule. + Counters XTCounters + + // Elems holds the data for all this rule's matches followed by the + // target. It is variable length -- users have to iterate over any + // matches and use TargetOffset and NextOffset to make sense of the + // data. + // + // Elems is omitted here because it would cause IPTEntry to be an extra + // byte larger (see http://www.catb.org/esr/structure-packing/). + // + // Elems [0]byte +} + +// SizeOfIPTEntry is the size of an IPTEntry. +const SizeOfIPTEntry = 112 + +// KernelIPTEntry is identical to IPTEntry, but includes the Elems field. +// +// +marshal dynamic +type KernelIPTEntry struct { + Entry IPTEntry + + // Elems holds the data for all this rule's matches followed by the + // target. It is variable length -- users have to iterate over any + // matches and use TargetOffset and NextOffset to make sense of the + // data. + Elems primitive.ByteSlice +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (ke *KernelIPTEntry) SizeBytes() int { + return ke.Entry.SizeBytes() + ke.Elems.SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (ke *KernelIPTEntry) MarshalBytes(dst []byte) []byte { + dst = ke.Entry.MarshalUnsafe(dst) + return ke.Elems.MarshalBytes(dst) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (ke *KernelIPTEntry) UnmarshalBytes(src []byte) []byte { + src = ke.Entry.UnmarshalUnsafe(src) + return ke.Elems.UnmarshalBytes(src) +} + +var _ marshal.Marshallable = (*KernelIPTEntry)(nil) + +// IPTIP contains information for matching a packet's IP header. +// It corresponds to struct ipt_ip in +// include/uapi/linux/netfilter_ipv4/ip_tables.h. +// +// +marshal +type IPTIP struct { + // Src is the source IP address. + Src InetAddr + + // Dst is the destination IP address. + Dst InetAddr + + // SrcMask is the source IP mask. + SrcMask InetAddr + + // DstMask is the destination IP mask. + DstMask InetAddr + + // InputInterface is the input network interface. + InputInterface [IFNAMSIZ]byte + + // OutputInterface is the output network interface. + OutputInterface [IFNAMSIZ]byte + + // InputInterfaceMask is the input interface mask. + InputInterfaceMask [IFNAMSIZ]byte + + // OuputInterfaceMask is the output interface mask. + OutputInterfaceMask [IFNAMSIZ]byte + + // Protocol is the transport protocol. + Protocol uint16 + + // Flags define matching behavior for the IP header. + Flags uint8 + + // InverseFlags invert the meaning of fields in struct IPTIP. See the + // IPT_INV_* flags. + InverseFlags uint8 +} + +// Flags in IPTIP.InverseFlags. Corresponding constants are in +// include/uapi/linux/netfilter_ipv4/ip_tables.h. +const ( + // Invert the meaning of InputInterface. + IPT_INV_VIA_IN = 0x01 + // Invert the meaning of OutputInterface. + IPT_INV_VIA_OUT = 0x02 + // Unclear what this is, as no references to it exist in the kernel. + IPT_INV_TOS = 0x04 + // Invert the meaning of Src. + IPT_INV_SRCIP = 0x08 + // Invert the meaning of Dst. + IPT_INV_DSTIP = 0x10 + // Invert the meaning of the IPT_F_FRAG flag. + IPT_INV_FRAG = 0x20 + // Invert the meaning of the Protocol field. + IPT_INV_PROTO = 0x40 + // Enable all flags. + IPT_INV_MASK = 0x7F +) + +// SizeOfIPTIP is the size of an IPTIP. +const SizeOfIPTIP = 84 + +// XTCounters holds packet and byte counts for a rule. It corresponds to struct +// xt_counters in include/uapi/linux/netfilter/x_tables.h. +// +// +marshal +type XTCounters struct { + // Pcnt is the packet count. + Pcnt uint64 + + // Bcnt is the byte count. + Bcnt uint64 +} + +// SizeOfXTCounters is the size of an XTCounters. +const SizeOfXTCounters = 16 + +// XTEntryMatch holds a match for a rule. For example, a user using the +// addrtype iptables match extension would put the data for that match into an +// XTEntryMatch. iptables-extensions(8) has a list of possible matches. +// +// XTEntryMatch corresponds to struct xt_entry_match in +// include/uapi/linux/netfilter/x_tables.h. That struct contains a union +// exposing different data to the user and kernel, but this struct holds only +// the user data. +// +// +marshal +type XTEntryMatch struct { + MatchSize uint16 + Name ExtensionName + Revision uint8 + // Data is omitted here because it would cause XTEntryMatch to be an + // extra byte larger (see http://www.catb.org/esr/structure-packing/). + // Data [0]byte +} + +// SizeOfXTEntryMatch is the size of an XTEntryMatch. +const SizeOfXTEntryMatch = 32 + +// KernelXTEntryMatch is identical to XTEntryMatch, but contains +// variable-length Data field. +type KernelXTEntryMatch struct { + XTEntryMatch + Data []byte +} + +// XTGetRevision corresponds to xt_get_revision in +// include/uapi/linux/netfilter/x_tables.h +// +// +marshal +type XTGetRevision struct { + Name ExtensionName + Revision uint8 +} + +// SizeOfXTGetRevision is the size of an XTGetRevision. +const SizeOfXTGetRevision = 30 + +// XTEntryTarget holds a target for a rule. For example, it can specify that +// packets matching the rule should DROP, ACCEPT, or use an extension target. +// iptables-extension(8) has a list of possible targets. +// +// XTEntryTarget corresponds to struct xt_entry_target in +// include/uapi/linux/netfilter/x_tables.h. That struct contains a union +// exposing different data to the user and kernel, but this struct holds only +// the user data. +// +// +marshal +type XTEntryTarget struct { + TargetSize uint16 + Name ExtensionName + Revision uint8 + // Data is omitted here because it would cause XTEntryTarget to be an + // extra byte larger (see http://www.catb.org/esr/structure-packing/). + // Data [0]byte +} + +// SizeOfXTEntryTarget is the size of an XTEntryTarget. +const SizeOfXTEntryTarget = 32 + +// KernelXTEntryTarget is identical to XTEntryTarget, but contains a +// variable-length Data field. +type KernelXTEntryTarget struct { + XTEntryTarget + Data []byte +} + +// XTStandardTarget is a built-in target, one of ACCEPT, DROP, JUMP, QUEUE, +// RETURN, or jump. It corresponds to struct xt_standard_target in +// include/uapi/linux/netfilter/x_tables.h. +// +// +marshal +type XTStandardTarget struct { + Target XTEntryTarget + // A positive verdict indicates a jump, and is the offset from the + // start of the table to jump to. A negative value means one of the + // other built-in targets. + Verdict int32 + _ [4]byte +} + +// SizeOfXTStandardTarget is the size of an XTStandardTarget. +const SizeOfXTStandardTarget = 40 + +// XTErrorTarget triggers an error when reached. It is also used to mark the +// beginning of user-defined chains by putting the name of the chain in +// ErrorName. It corresponds to struct xt_error_target in +// include/uapi/linux/netfilter/x_tables.h. +// +// +marshal +type XTErrorTarget struct { + Target XTEntryTarget + Name ErrorName + _ [2]byte +} + +// SizeOfXTErrorTarget is the size of an XTErrorTarget. +const SizeOfXTErrorTarget = 64 + +// Flag values for NfNATIPV4Range. The values indicate whether to map +// protocol specific part(ports) or IPs. It corresponds to values in +// include/uapi/linux/netfilter/nf_nat.h. +const ( + NF_NAT_RANGE_MAP_IPS = 1 << 0 + NF_NAT_RANGE_PROTO_SPECIFIED = 1 << 1 + NF_NAT_RANGE_PROTO_RANDOM = 1 << 2 + NF_NAT_RANGE_PERSISTENT = 1 << 3 + NF_NAT_RANGE_PROTO_RANDOM_FULLY = 1 << 4 + NF_NAT_RANGE_PROTO_RANDOM_ALL = (NF_NAT_RANGE_PROTO_RANDOM | NF_NAT_RANGE_PROTO_RANDOM_FULLY) + NF_NAT_RANGE_MASK = (NF_NAT_RANGE_MAP_IPS | + NF_NAT_RANGE_PROTO_SPECIFIED | NF_NAT_RANGE_PROTO_RANDOM | + NF_NAT_RANGE_PERSISTENT | NF_NAT_RANGE_PROTO_RANDOM_FULLY) +) + +// NfNATIPV4Range corresponds to struct nf_nat_ipv4_range +// in include/uapi/linux/netfilter/nf_nat.h. The fields are in +// network byte order. +// +// +marshal +type NfNATIPV4Range struct { + Flags uint32 + MinIP [4]byte + MaxIP [4]byte + MinPort uint16 + MaxPort uint16 +} + +// NfNATIPV4MultiRangeCompat corresponds to struct +// nf_nat_ipv4_multi_range_compat in include/uapi/linux/netfilter/nf_nat.h. +// +// +marshal +type NfNATIPV4MultiRangeCompat struct { + RangeSize uint32 + RangeIPV4 NfNATIPV4Range +} + +// XTRedirectTarget triggers a redirect when reached. +// Adding 4 bytes of padding to make the struct 8 byte aligned. +// +// +marshal +type XTRedirectTarget struct { + Target XTEntryTarget + NfRange NfNATIPV4MultiRangeCompat + _ [4]byte +} + +// SizeOfXTRedirectTarget is the size of an XTRedirectTarget. +const SizeOfXTRedirectTarget = 56 + +// XTNATTargetV0 triggers NAT when reached. +// Adding 4 bytes of padding to make the struct 8 byte aligned. +// +// +marshal +type XTNATTargetV0 struct { + Target XTEntryTarget + NfRange NfNATIPV4MultiRangeCompat + _ [4]byte +} + +// SizeOfXTNATTargetV0 is the size of an XTNATTargetV0. +const SizeOfXTNATTargetV0 = 56 + +// XTNATTargetV1 triggers NAT when reached. +// +// +marshal +type XTNATTargetV1 struct { + Target XTEntryTarget + Range NFNATRange +} + +// SizeOfXTNATTargetV1 is the size of an XTNATTargetV1. +const SizeOfXTNATTargetV1 = SizeOfXTEntryTarget + SizeOfNFNATRange + +// XTNATTargetV2 triggers NAT when reached. +// +// +marshal +type XTNATTargetV2 struct { + Target XTEntryTarget + Range NFNATRange2 +} + +// SizeOfXTNATTargetV2 is the size of an XTNATTargetV2. +const SizeOfXTNATTargetV2 = SizeOfXTEntryTarget + SizeOfNFNATRange2 + +// IPTGetinfo is the argument for the IPT_SO_GET_INFO sockopt. It corresponds +// to struct ipt_getinfo in include/uapi/linux/netfilter_ipv4/ip_tables.h. +// +// +marshal +type IPTGetinfo struct { + Name TableName + ValidHooks uint32 + HookEntry [NF_INET_NUMHOOKS]uint32 + Underflow [NF_INET_NUMHOOKS]uint32 + NumEntries uint32 + Size uint32 +} + +// SizeOfIPTGetinfo is the size of an IPTGetinfo. +const SizeOfIPTGetinfo = 84 + +// IPTGetEntries is the argument for the IPT_SO_GET_ENTRIES sockopt. It +// corresponds to struct ipt_get_entries in +// include/uapi/linux/netfilter_ipv4/ip_tables.h. +// +// +marshal +type IPTGetEntries struct { + Name TableName + Size uint32 + _ [4]byte + // Entrytable is omitted here because it would cause IPTGetEntries to + // be an extra byte longer (see + // http://www.catb.org/esr/structure-packing/). + // Entrytable [0]IPTEntry +} + +// SizeOfIPTGetEntries is the size of an IPTGetEntries. +const SizeOfIPTGetEntries = 40 + +// KernelIPTGetEntries is identical to IPTGetEntries, but includes the +// Entrytable field. +// +// +marshal dynamic +type KernelIPTGetEntries struct { + IPTGetEntries + Entrytable []KernelIPTEntry +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (ke *KernelIPTGetEntries) SizeBytes() int { + res := ke.IPTGetEntries.SizeBytes() + for _, entry := range ke.Entrytable { + res += entry.SizeBytes() + } + return res +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (ke *KernelIPTGetEntries) MarshalBytes(dst []byte) []byte { + dst = ke.IPTGetEntries.MarshalUnsafe(dst) + for i := range ke.Entrytable { + dst = ke.Entrytable[i].MarshalBytes(dst) + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (ke *KernelIPTGetEntries) UnmarshalBytes(src []byte) []byte { + src = ke.IPTGetEntries.UnmarshalUnsafe(src) + for i := range ke.Entrytable { + src = ke.Entrytable[i].UnmarshalBytes(src) + } + return src +} + +var _ marshal.Marshallable = (*KernelIPTGetEntries)(nil) + +// IPTReplace is the argument for the IPT_SO_SET_REPLACE sockopt. It +// corresponds to struct ipt_replace in +// include/uapi/linux/netfilter_ipv4/ip_tables.h. +// +// +marshal +type IPTReplace struct { + Name TableName + ValidHooks uint32 + NumEntries uint32 + Size uint32 + HookEntry [NF_INET_NUMHOOKS]uint32 + Underflow [NF_INET_NUMHOOKS]uint32 + NumCounters uint32 + Counters uint64 // This is really a *XTCounters. + // Entries is omitted here because it would cause IPTReplace to be an + // extra byte longer (see http://www.catb.org/esr/structure-packing/). + // Entries [0]IPTEntry +} + +// SizeOfIPTReplace is the size of an IPTReplace. +const SizeOfIPTReplace = 96 + +// ExtensionName holds the name of a netfilter extension. +// +// +marshal +type ExtensionName [XT_EXTENSION_MAXNAMELEN]byte + +// String implements fmt.Stringer. +func (en ExtensionName) String() string { + return goString(en[:]) +} + +// TableName holds the name of a netfilter table. +// +// +marshal +type TableName [XT_TABLE_MAXNAMELEN]byte + +// String implements fmt.Stringer. +func (tn TableName) String() string { + return goString(tn[:]) +} + +// ErrorName holds the name of a netfilter error. These can also hold +// user-defined chains. +// +// +marshal +type ErrorName [XT_FUNCTION_MAXNAMELEN]byte + +// String implements fmt.Stringer. +func (en ErrorName) String() string { + return goString(en[:]) +} + +func goString(cstring []byte) string { + for i, c := range cstring { + if c == 0 { + return string(cstring[:i]) + } + } + return string(cstring) +} + +// XTTCP holds data for matching TCP packets. It corresponds to struct xt_tcp +// in include/uapi/linux/netfilter/xt_tcpudp.h. +// +// +marshal +type XTTCP struct { + // SourcePortStart specifies the inclusive start of the range of source + // ports to which the matcher applies. + SourcePortStart uint16 + + // SourcePortEnd specifies the inclusive end of the range of source ports + // to which the matcher applies. + SourcePortEnd uint16 + + // DestinationPortStart specifies the start of the destination port + // range to which the matcher applies. + DestinationPortStart uint16 + + // DestinationPortEnd specifies the end of the destination port + // range to which the matcher applies. + DestinationPortEnd uint16 + + // Option specifies that a particular TCP option must be set. + Option uint8 + + // FlagMask masks TCP flags when comparing to the FlagCompare byte. It allows + // for specification of which flags are important to the matcher. + FlagMask uint8 + + // FlagCompare, in combination with FlagMask, is used to match only packets + // that have certain flags set. + FlagCompare uint8 + + // InverseFlags flips the meaning of certain fields. See the + // TX_TCP_INV_* flags. + InverseFlags uint8 +} + +// SizeOfXTTCP is the size of an XTTCP. +const SizeOfXTTCP = 12 + +// Flags in XTTCP.InverseFlags. Corresponding constants are in +// include/uapi/linux/netfilter/xt_tcpudp.h. +const ( + // Invert the meaning of SourcePortStart/End. + XT_TCP_INV_SRCPT = 0x01 + // Invert the meaning of DestinationPortStart/End. + XT_TCP_INV_DSTPT = 0x02 + // Invert the meaning of FlagCompare. + XT_TCP_INV_FLAGS = 0x04 + // Invert the meaning of Option. + XT_TCP_INV_OPTION = 0x08 + // Enable all flags. + XT_TCP_INV_MASK = 0x0F +) + +// XTUDP holds data for matching UDP packets. It corresponds to struct xt_udp +// in include/uapi/linux/netfilter/xt_tcpudp.h. +// +// +marshal +type XTUDP struct { + // SourcePortStart is the inclusive start of the range of source ports + // to which the matcher applies. + SourcePortStart uint16 + + // SourcePortEnd is the inclusive end of the range of source ports to + // which the matcher applies. + SourcePortEnd uint16 + + // DestinationPortStart is the inclusive start of the destination port + // range to which the matcher applies. + DestinationPortStart uint16 + + // DestinationPortEnd is the inclusive end of the destination port + // range to which the matcher applies. + DestinationPortEnd uint16 + + // InverseFlags flips the meaning of certain fields. See the + // TX_UDP_INV_* flags. + InverseFlags uint8 + + _ uint8 +} + +// SizeOfXTUDP is the size of an XTUDP. +const SizeOfXTUDP = 10 + +// Flags in XTUDP.InverseFlags. Corresponding constants are in +// include/uapi/linux/netfilter/xt_tcpudp.h. +const ( + // Invert the meaning of SourcePortStart/End. + XT_UDP_INV_SRCPT = 0x01 + // Invert the meaning of DestinationPortStart/End. + XT_UDP_INV_DSTPT = 0x02 + // Enable all flags. + XT_UDP_INV_MASK = 0x03 +) + +// IPTOwnerInfo holds data for matching packets with the owner v0 matcher. It +// corresponds to struct ipt_owner_info in libxt_owner.c of iptables binary. +// +// +marshal +type IPTOwnerInfo struct { + // UID is user id which created the packet. + UID uint32 + + // GID is group id which created the packet. + GID uint32 + + // PID is process id of the process which created the packet. + PID uint32 + + // SID is session id which created the packet. + SID uint32 + + // Comm is the command name which created the packet. + Comm [16]byte + + // Match is used to match UID/GID of the socket. See the + // XT_OWNER_* flags below. + Match uint8 + + // Invert flips the meaning of Match field. + Invert uint8 `marshal:"unaligned"` +} + +// SizeOfIPTOwnerInfo is the size of an IPTOwnerInfo. +const SizeOfIPTOwnerInfo = 34 + +// XTOwnerMatchInfo holds data for matching packets with the owner v1 matcher. +// It corresponds to struct xt_owner_match_info in +// include/uapi/linux/netfilter/xt_owner.h +// +// +marshal +type XTOwnerMatchInfo struct { + UIDMin uint32 + UIDMax uint32 + GIDMin uint32 + GIDMax uint32 + Match uint8 + Invert uint8 + _ [2]byte +} + +// SizeOfXTOwnerMatchInfo is the size of an XTOwnerMatchInfo. +const SizeOfXTOwnerMatchInfo = 20 + +// Flags in IPTOwnerInfo.Match and XTOwnerMatchInfo.Match. Corresponding +// constants are in include/uapi/linux/netfilter/xt_owner.h. +const ( + // Match the UID of the packet. + XT_OWNER_UID = 1 << 0 + // Match the GID of the packet. + XT_OWNER_GID = 1 << 1 + // Match if the socket exists for the packet. Forwarded + // packets do not have an associated socket. + XT_OWNER_SOCKET = 1 << 2 +) + +// XT_MULTI_PORTS is the maximum number of ports that the +// multiport match can handle. +const XT_MULTI_PORTS = 15 + +// Flags in XTMultiport{,V1}.Flags; values from "enum xt_multiport_flags" +// in "include/uapi/linux/netfilter/xt_multiport.h". +const ( + XT_MULTIPORT_SOURCE uint8 = 0x0 // Match against source ports. + XT_MULTIPORT_DESTINATION uint8 = 0x1 // Match against destination ports. + XT_MULTIPORT_EITHER uint8 = 0x2 // Match against either ports. +) + +// XTMultiport holds data for matching packets against a set +// of ports. It corresponds to "struct xt_multiport" defined +// in "include/uapi/linux/netfilter/xt_multiport.h". +// +// +marshal +type XTMultiport struct { + // Flags indicates whether the match applies to + // source ports, destination ports, or either, as + // defined by "enum xt_multiport_flags". + Flags uint8 + + // Count is the number of ports in the "Ports" + // slice that the match will check. It must be + // between 1 and "XT_MULTI_PORTS" (inclusive). + Count uint8 + + // Ports is the set of ports that will be matched. + // Only the first "Count" entries are considered. + Ports [XT_MULTI_PORTS]uint16 +} + +// XTMultiportV1 holds data for matching packets against a set +// of ports. It corresponds to "struct xt_multiport_v1" defined +// in "include/uapi/linux/netfilter/xt_multiport.h". +// +// +marshal +type XTMultiportV1 struct { + // Fields same as "XTMultiport". + Flags uint8 + Count uint8 + Ports [XT_MULTI_PORTS]uint16 + + // Pflags is an array of port-specific flags. Each entry + // in "Pflags" corresponds to the port at the same index + // in "Ports". + Pflags [XT_MULTI_PORTS]uint8 + + // Invert is a flag that, if nonzero, indicates + // that the match result should be inverted. + Invert uint8 +} + +// SizeOfXTMultiport is the size of XTMultiport (in bytes). +const SizeOfXTMultiport = 2 + (XT_MULTI_PORTS * 2) + +// SizeOfXTMultiportV1 is the size of XTMultiportV1 (in bytes). +const SizeOfXTMultiportV1 = SizeOfXTMultiport + XT_MULTI_PORTS + 1 diff --git a/pkg/abi/linux/netfilter_arp.go b/pkg/abi/linux/netfilter_arp.go new file mode 100644 index 0000000..fc681b1 --- /dev/null +++ b/pkg/abi/linux/netfilter_arp.go @@ -0,0 +1,24 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// These constants show the hooks ARP packets can be evaluated at. +// From include/uapi/linux/netfilter_arp.h. +const ( + NF_ARP_IN = iota + NF_ARP_OUT + NF_ARP_FORWARD + NF_ARP_NUMHOOKS +) diff --git a/pkg/abi/linux/netfilter_bridge.go b/pkg/abi/linux/netfilter_bridge.go new file mode 100644 index 0000000..e1ed932 --- /dev/null +++ b/pkg/abi/linux/netfilter_bridge.go @@ -0,0 +1,41 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import "math" + +// Netfilter Bridge Standard Hook Points, from uapi/linux/netfilter_bridge.h. +const ( + NF_BR_PRE_ROUTING = iota + NF_BR_LOCAL_IN + NF_BR_FORWARD + NF_BR_LOCAL_OUT + NF_BR_POST_ROUTING + NF_BR_BROUTING + NF_BR_NUMHOOKS +) + +// Netfilter Bridge Standard Hook Priorities, from +// uapi/linux/netfilter_bridge.h. +const ( + NF_BR_PRI_FIRST = math.MinInt + NF_BR_PRI_NAT_DST_BRIDGED = -300 + NF_BR_PRI_FILTER_BRIDGED = -200 + NF_BR_PRI_BRNF = 0 + NF_BR_PRI_NAT_DST_OTHER = 100 + NF_BR_PRI_FILTER_OTHER = 200 + NF_BR_PRI_NAT_SRC = 300 + NF_BR_PRI_LAST = math.MaxInt +) diff --git a/pkg/abi/linux/netfilter_ipv4.go b/pkg/abi/linux/netfilter_ipv4.go new file mode 100644 index 0000000..0943bcb --- /dev/null +++ b/pkg/abi/linux/netfilter_ipv4.go @@ -0,0 +1,36 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import "math" + +// Netfilter IPv4 Standard Hook Priorities, from uapi/linux/netfilter_ipv4.h. +const ( + NF_IP_PRI_FIRST = math.MinInt + NF_IP_PRI_RAW_BEFORE_DEFRAG = -450 + NF_IP_PRI_CONNTRACK_DEFRAG = -400 + NF_IP_PRI_RAW = -300 + NF_IP_PRI_SELINUX_FIRST = -225 + NF_IP_PRI_CONNTRACK = -200 + NF_IP_PRI_MANGLE = -150 + NF_IP_PRI_NAT_DST = -100 + NF_IP_PRI_FILTER = 0 + NF_IP_PRI_SECURITY = 50 + NF_IP_PRI_NAT_SRC = 100 + NF_IP_PRI_SELINUX_LAST = 225 + NF_IP_PRI_CONNTRACK_HELPER = 300 + NF_IP_PRI_CONNTRACK_CONFIRM = math.MaxInt + NF_IP_PRI_LAST = math.MaxInt +) diff --git a/pkg/abi/linux/netfilter_ipv6.go b/pkg/abi/linux/netfilter_ipv6.go new file mode 100644 index 0000000..691e604 --- /dev/null +++ b/pkg/abi/linux/netfilter_ipv6.go @@ -0,0 +1,314 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "math" + + "github.com/sagernet/gvisor/pkg/marshal" + "github.com/sagernet/gvisor/pkg/marshal/primitive" +) + +// This file contains structures required to support IPv6 netfilter and +// ip6tables. Some constants and structs are equal to their IPv4 analogues, and +// are only distinguished by context (e.g. whether used on an IPv4 of IPv6 +// socket). + +// Netfilter IPv6 Standard Hook Priorities, from uapi/linux/netfilter_ipv6.h. +const ( + NF_IP6_PRI_FIRST = math.MinInt + NF_IP6_PRI_RAW_BEFORE_DEFRAG = -450 + NF_IP6_PRI_CONNTRACK_DEFRAG = -400 + NF_IP6_PRI_RAW = -300 + NF_IP6_PRI_SELINUX_FIRST = -225 + NF_IP6_PRI_CONNTRACK = -200 + NF_IP6_PRI_MANGLE = -150 + NF_IP6_PRI_NAT_DST = -100 + NF_IP6_PRI_FILTER = 0 + NF_IP6_PRI_SECURITY = 50 + NF_IP6_PRI_NAT_SRC = 100 + NF_IP6_PRI_SELINUX_LAST = 225 + NF_IP6_PRI_CONNTRACK_HELPER = 300 + NF_IP6_PRI_LAST = math.MaxInt +) + +// Socket options for SOL_SOCLET. These correspond to values in +// include/uapi/linux/netfilter_ipv6/ip6_tables.h. +const ( + IP6T_BASE_CTL = 64 + IP6T_SO_SET_REPLACE = IPT_BASE_CTL + IP6T_SO_SET_ADD_COUNTERS = IPT_BASE_CTL + 1 + IP6T_SO_SET_MAX = IPT_SO_SET_ADD_COUNTERS + + IP6T_SO_GET_INFO = IPT_BASE_CTL + IP6T_SO_GET_ENTRIES = IPT_BASE_CTL + 1 + IP6T_SO_GET_REVISION_MATCH = IPT_BASE_CTL + 4 + IP6T_SO_GET_REVISION_TARGET = IPT_BASE_CTL + 5 + IP6T_SO_GET_MAX = IP6T_SO_GET_REVISION_TARGET +) + +// IP6T_ORIGINAL_DST is the ip6tables SOL_IPV6 socket option. Corresponds to +// the value in include/uapi/linux/netfilter_ipv6/ip6_tables.h. +const IP6T_ORIGINAL_DST = 80 + +// IP6TReplace is the argument for the IP6T_SO_SET_REPLACE sockopt. It +// corresponds to struct ip6t_replace in +// include/uapi/linux/netfilter_ipv6/ip6_tables.h. +// +// +marshal +type IP6TReplace struct { + Name TableName + ValidHooks uint32 + NumEntries uint32 + Size uint32 + HookEntry [NF_INET_NUMHOOKS]uint32 + Underflow [NF_INET_NUMHOOKS]uint32 + NumCounters uint32 + Counters uint64 // This is really a *XTCounters. + // Entries is omitted here because it would cause IP6TReplace to be an + // extra byte longer (see http://www.catb.org/esr/structure-packing/). + // Entries [0]IP6TEntry +} + +// SizeOfIP6TReplace is the size of an IP6TReplace. +const SizeOfIP6TReplace = 96 + +// KernelIP6TGetEntries is identical to IP6TGetEntries, but includes the +// Entrytable field. +// +// +marshal dynamic +type KernelIP6TGetEntries struct { + IPTGetEntries + Entrytable []KernelIP6TEntry +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (ke *KernelIP6TGetEntries) SizeBytes() int { + res := ke.IPTGetEntries.SizeBytes() + for _, entry := range ke.Entrytable { + res += entry.SizeBytes() + } + return res +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (ke *KernelIP6TGetEntries) MarshalBytes(dst []byte) []byte { + dst = ke.IPTGetEntries.MarshalUnsafe(dst) + for i := range ke.Entrytable { + dst = ke.Entrytable[i].MarshalBytes(dst) + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (ke *KernelIP6TGetEntries) UnmarshalBytes(src []byte) []byte { + src = ke.IPTGetEntries.UnmarshalUnsafe(src) + for i := range ke.Entrytable { + src = ke.Entrytable[i].UnmarshalBytes(src) + } + return src +} + +var _ marshal.Marshallable = (*KernelIP6TGetEntries)(nil) + +// IP6TEntry is an iptables rule. It corresponds to struct ip6t_entry in +// include/uapi/linux/netfilter_ipv6/ip6_tables.h. +// +// +marshal +type IP6TEntry struct { + // IPv6 is used to filter packets based on the IPv6 header. + IPv6 IP6TIP + + // NFCache relates to kernel-internal caching and isn't used by + // userspace. + NFCache uint32 + + // TargetOffset is the byte offset from the beginning of this IPTEntry + // to the start of the entry's target. + TargetOffset uint16 + + // NextOffset is the byte offset from the beginning of this IPTEntry to + // the start of the next entry. It is thus also the size of the entry. + NextOffset uint16 + + // Comeback is a return pointer. It is not used by userspace. + Comeback uint32 + + _ [4]byte + + // Counters holds the packet and byte counts for this rule. + Counters XTCounters + + // Elems holds the data for all this rule's matches followed by the + // target. It is variable length -- users have to iterate over any + // matches and use TargetOffset and NextOffset to make sense of the + // data. + // + // Elems is omitted here because it would cause IPTEntry to be an extra + // byte larger (see http://www.catb.org/esr/structure-packing/). + // + // Elems [0]byte +} + +// SizeOfIP6TEntry is the size of an IP6TEntry. +const SizeOfIP6TEntry = 168 + +// KernelIP6TEntry is identical to IP6TEntry, but includes the Elems field. +// +// +marshal dynamic +type KernelIP6TEntry struct { + Entry IP6TEntry + + // Elems holds the data for all this rule's matches followed by the + // target. It is variable length -- users have to iterate over any + // matches and use TargetOffset and NextOffset to make sense of the + // data. + Elems primitive.ByteSlice +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (ke *KernelIP6TEntry) SizeBytes() int { + return ke.Entry.SizeBytes() + ke.Elems.SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (ke *KernelIP6TEntry) MarshalBytes(dst []byte) []byte { + dst = ke.Entry.MarshalUnsafe(dst) + return ke.Elems.MarshalBytes(dst) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (ke *KernelIP6TEntry) UnmarshalBytes(src []byte) []byte { + src = ke.Entry.UnmarshalUnsafe(src) + return ke.Elems.UnmarshalBytes(src) +} + +var _ marshal.Marshallable = (*KernelIP6TEntry)(nil) + +// IP6TIP contains information for matching a packet's IP header. +// It corresponds to struct ip6t_ip6 in +// include/uapi/linux/netfilter_ipv6/ip6_tables.h. +// +// +marshal +type IP6TIP struct { + // Src is the source IP address. + Src Inet6Addr + + // Dst is the destination IP address. + Dst Inet6Addr + + // SrcMask is the source IP mask. + SrcMask Inet6Addr + + // DstMask is the destination IP mask. + DstMask Inet6Addr + + // InputInterface is the input network interface. + InputInterface [IFNAMSIZ]byte + + // OutputInterface is the output network interface. + OutputInterface [IFNAMSIZ]byte + + // InputInterfaceMask is the input interface mask. + InputInterfaceMask [IFNAMSIZ]byte + + // OuputInterfaceMask is the output interface mask. + OutputInterfaceMask [IFNAMSIZ]byte + + // Protocol is the transport protocol. + Protocol uint16 + + // TOS matches TOS flags when Flags indicates filtering by TOS. + TOS uint8 + + // Flags define matching behavior for the IP header. + Flags uint8 + + // InverseFlags invert the meaning of fields in struct IPTIP. See the + // IP6T_INV_* flags. + InverseFlags uint8 + + // Linux defines in6_addr (Inet6Addr for us) as the union of a + // 16-element byte array and a 4-element 32-bit integer array, so the + // whole struct is 4-byte aligned. + _ [3]byte +} + +// SizeOfIP6TIP is the size of an IP6 header. +const SizeOfIP6TIP = 136 + +// Flags in IP6TIP.Flags. Corresponding constants are in +// include/uapi/linux/netfilter_ipv6/ip6_tables.h. +const ( + // Whether to check the Protocol field. + IP6T_F_PROTO = 0x01 + // Whether to match the TOS field. + IP6T_F_TOS = 0x02 + // Indicates that the jump target is an absolute GOTO, not an offset. + IP6T_F_GOTO = 0x04 + // Enables all flags. + IP6T_F_MASK = 0x07 +) + +// Flags in IP6TIP.InverseFlags. Corresponding constants are in +// include/uapi/linux/netfilter_ipv6/ip6_tables.h. +const ( + // Invert the meaning of InputInterface. + IP6T_INV_VIA_IN = 0x01 + // Invert the meaning of OutputInterface. + IP6T_INV_VIA_OUT = 0x02 + // Invert the meaning of TOS. + IP6T_INV_TOS = 0x04 + // Invert the meaning of Src. + IP6T_INV_SRCIP = 0x08 + // Invert the meaning of Dst. + IP6T_INV_DSTIP = 0x10 + // Invert the meaning of the IPT_F_FRAG flag. + IP6T_INV_FRAG = 0x20 + // Enable all flags. + IP6T_INV_MASK = 0x7F +) + +// NFNATRange corresponds to struct nf_nat_range in +// include/uapi/linux/netfilter/nf_nat.h. +// +// +marshal +type NFNATRange struct { + Flags uint32 + MinAddr Inet6Addr + MaxAddr Inet6Addr + MinProto uint16 // Network byte order. + MaxProto uint16 // Network byte order. +} + +// SizeOfNFNATRange is the size of NFNATRange. +const SizeOfNFNATRange = 40 + +// NFNATRange2 corresponds to struct nf_nat_range2 in +// include/uapi/linux/netfilter/nf_nat.h. +// +// +marshal +type NFNATRange2 struct { + Flags uint32 + MinAddr Inet6Addr + MaxAddr Inet6Addr + MinProto uint16 // Network byte order. + MaxProto uint16 // Network byte order. + BaseProto uint16 // Network byte order. + _ [6]byte +} + +// SizeOfNFNATRange2 is the size of NFNATRange2. +const SizeOfNFNATRange2 = 48 diff --git a/pkg/abi/linux/netlink.go b/pkg/abi/linux/netlink.go new file mode 100644 index 0000000..2be0b75 --- /dev/null +++ b/pkg/abi/linux/netlink.go @@ -0,0 +1,159 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Netlink protocols, from uapi/linux/netlink.h. +const ( + NETLINK_ROUTE = 0 + NETLINK_UNUSED = 1 + NETLINK_USERSOCK = 2 + NETLINK_FIREWALL = 3 + NETLINK_SOCK_DIAG = 4 + NETLINK_NFLOG = 5 + NETLINK_XFRM = 6 + NETLINK_SELINUX = 7 + NETLINK_ISCSI = 8 + NETLINK_AUDIT = 9 + NETLINK_FIB_LOOKUP = 10 + NETLINK_CONNECTOR = 11 + NETLINK_NETFILTER = 12 + NETLINK_IP6_FW = 13 + NETLINK_DNRTMSG = 14 + NETLINK_KOBJECT_UEVENT = 15 + NETLINK_GENERIC = 16 + NETLINK_SCSITRANSPORT = 18 + NETLINK_ECRYPTFS = 19 + NETLINK_RDMA = 20 + NETLINK_CRYPTO = 21 +) + +// SockAddrNetlink is struct sockaddr_nl, from uapi/linux/netlink.h. +// +// +marshal +type SockAddrNetlink struct { + Family uint16 + _ uint16 + PortID uint32 + Groups uint32 +} + +// SockAddrNetlinkSize is the size of SockAddrNetlink. +const SockAddrNetlinkSize = 12 + +// NetlinkMessageHeader is struct nlmsghdr, from uapi/linux/netlink.h. +// +// +marshal +type NetlinkMessageHeader struct { + Length uint32 + Type uint16 + Flags uint16 + Seq uint32 + PortID uint32 +} + +// NetlinkMessageHeaderSize is the size of NetlinkMessageHeader. +const NetlinkMessageHeaderSize = 16 + +// Netlink message header flag values, from uapi/linux/netlink.h. +const ( + NLM_F_REQUEST = 0x1 + NLM_F_MULTI = 0x2 + NLM_F_ACK = 0x4 + NLM_F_ECHO = 0x8 + NLM_F_DUMP_INTR = 0x10 +) + +// Netlink message header flags for GET requests, from uapi/linux/netlink.h. +const ( + NLM_F_ROOT = 0x100 + NLM_F_MATCH = 0x200 + NLM_F_ATOMIC = 0x400 + NLM_F_DUMP = NLM_F_ROOT | NLM_F_MATCH +) + +// Netlink message header flags for NEW requests, from uapi/linux/netlink.h. +const ( + NLM_F_REPLACE = 0x100 + NLM_F_EXCL = 0x200 + NLM_F_CREATE = 0x400 + NLM_F_APPEND = 0x800 +) + +// Netlink message header flags for DELETE requests, from uapi/linux/netlink.h. +const ( + NLM_F_NONREC = 0x100 + NLM_F_BULK = 0x200 +) + +// Standard netlink message types, from uapi/linux/netlink.h. +const ( + NLMSG_NOOP = 0x1 + NLMSG_ERROR = 0x2 + NLMSG_DONE = 0x3 + NLMSG_OVERRUN = 0x4 + + // NLMSG_MIN_TYPE is the first value for protocol-level types. + NLMSG_MIN_TYPE = 0x10 +) + +// NLMSG_ALIGNTO is the alignment of netlink messages, from +// uapi/linux/netlink.h. +const NLMSG_ALIGNTO = 4 + +// NetlinkAttrHeader is the header of a netlink attribute, followed by payload. +// +// This is struct nlattr, from uapi/linux/netlink.h. +// +// +marshal +type NetlinkAttrHeader struct { + Length uint16 + Type uint16 +} + +// Netlink attribute flags, from uapi/linux/netlink.h. +const ( + NLA_F_NESTED uint16 = 1 << 15 + NLA_F_NET_BYTEORDER = 1 << 14 + NLA_TYPE_MASK = ^(NLA_F_NESTED | NLA_F_NET_BYTEORDER) +) + +// NetlinkAttrHeaderSize is the size of NetlinkAttrHeader. +const NetlinkAttrHeaderSize = 4 + +// NLA_ALIGNTO is the alignment of netlink attributes, from +// uapi/linux/netlink.h. +const NLA_ALIGNTO = 4 + +// Socket options, from uapi/linux/netlink.h. +const ( + NETLINK_ADD_MEMBERSHIP = 1 + NETLINK_DROP_MEMBERSHIP = 2 + NETLINK_PKTINFO = 3 + NETLINK_BROADCAST_ERROR = 4 + NETLINK_NO_ENOBUFS = 5 + NETLINK_LISTEN_ALL_NSID = 8 + NETLINK_LIST_MEMBERSHIPS = 9 + NETLINK_CAP_ACK = 10 + NETLINK_EXT_ACK = 11 + NETLINK_DUMP_STRICT_CHK = 12 +) + +// NetlinkErrorMessage is struct nlmsgerr, from uapi/linux/netlink.h. +// +// +marshal +type NetlinkErrorMessage struct { + Error int32 + Header NetlinkMessageHeader +} diff --git a/pkg/abi/linux/netlink_netfilter.go b/pkg/abi/linux/netlink_netfilter.go new file mode 100644 index 0000000..ebf178c --- /dev/null +++ b/pkg/abi/linux/netlink_netfilter.go @@ -0,0 +1,92 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Group describes Netlink Netfilter groups, from uapi/linux/netfilter/nfnetlink.h. +// Users bind to specific groups to receive processing logs from those groups. +type Group uint16 + +// Netlink Netfilter groups. +const ( + NFNLGPR_NONE Group = iota + NFNLGRP_CONNTRACK_NEW + NFNLGRP_CONNTRACK_UPDATE + NFNLGRP_CONNTRACK_DESTROY + NFNLGRP_CONNTRACK_EXP_NEW + NFNLGRP_CONNTRACK_EXP_UPDATE + NFNLGRP_CONNTRACK_EXP_DESTROY + NFNLGRP_NFTABLES + NFNLGRP_ACCT_QUOTA + NFNLGRP_NFTRACE + __NFNLGRP_MAX + NFNLGRP_MAX = __NFNLGRP_MAX - 1 +) + +// NetFilterGenMsg describes the netlink netfilter genmsg message, from uapi/linux/netfilter/nfnetlink.h. +// +// +marshal +type NetFilterGenMsg struct { + Family uint8 + Version uint8 + ResourceID uint16 +} + +// SizeOfNetfilterGenMsg is the size of the netlink netfilter genmsg message. +const SizeOfNetfilterGenMsg = 4 + +// NFNETLINK_V0 is the default version of the netlink netfilter. +const NFNETLINK_V0 = 0 + +// Netlink Netfilter subsystem IDs, from uapi/linux/netfilter/nfnetlink.h. +const ( + NFNL_SUBSYS_NONE = iota + NFNL_SUBSYS_CTNETLINK + NFNL_SUBSYS_CTNETLINK_EXP + NFNL_SUBSYS_QUEUE + NFNL_SUBSYS_ULOG + NFNL_SUBSYS_OSF + NFNL_SUBSYS_IPSET + NFNL_SUBSYS_ACCT + NFNL_SUBSYS_CTNETLINK_TIMEOUT + NFNL_SUBSYS_CTHELPER + NFNL_SUBSYS_NFTABLES + NFNL_SUBSYS_NFT_COMPAT + NFNL_SUBSYS_HOOK + NFNL_SUBSYS_COUNT +) + +// NetFilterSubsysID returns the Netfilter Subsystem ID from the netlink message header. +func (hdr *NetlinkMessageHeader) NetFilterSubsysID() uint16 { + return (hdr.Type & 0xff00) >> 8 +} + +// NetFilterMsgType returns the Netfilter Message Type from the netlink message header. +func (hdr *NetlinkMessageHeader) NetFilterMsgType() NfTableMsgType { + return NfTableMsgType(hdr.Type & 0x00ff) +} + +// Reserved control Netlink Netfilter messages, from uapi/linux/netfilter/nfnetlink.h. +const ( + NFNL_MSG_BATCH_BEGIN = NLMSG_MIN_TYPE + NFNL_MSG_BATCH_END = NLMSG_MIN_TYPE + 1 +) + +// Netlink Netfilter batch attributes. +const ( + NFNL_BATCH_UNSPEC = iota + NFNL_BATCH_GENID + __NFNL_BATCH_MAX + NFNL_BATCH_MAX = __NFNL_BATCH_MAX - 1 +) diff --git a/pkg/abi/linux/netlink_route.go b/pkg/abi/linux/netlink_route.go new file mode 100644 index 0000000..b3deba1 --- /dev/null +++ b/pkg/abi/linux/netlink_route.go @@ -0,0 +1,377 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Netlink message types for NETLINK_ROUTE sockets, from uapi/linux/rtnetlink.h. +const ( + RTM_NEWLINK = 16 + RTM_DELLINK = 17 + RTM_GETLINK = 18 + RTM_SETLINK = 19 + + RTM_NEWADDR = 20 + RTM_DELADDR = 21 + RTM_GETADDR = 22 + + RTM_NEWROUTE = 24 + RTM_DELROUTE = 25 + RTM_GETROUTE = 26 + + RTM_NEWNEIGH = 28 + RTM_DELNEIGH = 29 + RTM_GETNEIGH = 30 + + RTM_NEWRULE = 32 + RTM_DELRULE = 33 + RTM_GETRULE = 34 + + RTM_NEWQDISC = 36 + RTM_DELQDISC = 37 + RTM_GETQDISC = 38 + + RTM_NEWTCLASS = 40 + RTM_DELTCLASS = 41 + RTM_GETTCLASS = 42 + + RTM_NEWTFILTER = 44 + RTM_DELTFILTER = 45 + RTM_GETTFILTER = 46 + + RTM_NEWACTION = 48 + RTM_DELACTION = 49 + RTM_GETACTION = 50 + + RTM_NEWPREFIX = 52 + + RTM_GETMULTICAST = 58 + + RTM_GETANYCAST = 62 + + RTM_NEWNEIGHTBL = 64 + RTM_GETNEIGHTBL = 66 + RTM_SETNEIGHTBL = 67 + + RTM_NEWNDUSEROPT = 68 + + RTM_NEWADDRLABEL = 72 + RTM_DELADDRLABEL = 73 + RTM_GETADDRLABEL = 74 + + RTM_GETDCB = 78 + RTM_SETDCB = 79 + + RTM_NEWNETCONF = 80 + RTM_GETNETCONF = 82 + + RTM_NEWMDB = 84 + RTM_DELMDB = 85 + RTM_GETMDB = 86 + + RTM_NEWNSID = 88 + RTM_DELNSID = 89 + RTM_GETNSID = 90 +) + +// InterfaceInfoMessage is struct ifinfomsg, from uapi/linux/rtnetlink.h. +// +// +marshal +type InterfaceInfoMessage struct { + Family uint8 + _ uint8 + Type uint16 + Index int32 + Flags uint32 + Change uint32 +} + +// InterfaceInfoMessageSize is the size of InterfaceInfoMessage. +const InterfaceInfoMessageSize = 16 + +// Interface flags, from uapi/linux/if.h. +const ( + IFF_UP = 1 << 0 + IFF_BROADCAST = 1 << 1 + IFF_DEBUG = 1 << 2 + IFF_LOOPBACK = 1 << 3 + IFF_POINTOPOINT = 1 << 4 + IFF_NOTRAILERS = 1 << 5 + IFF_RUNNING = 1 << 6 + IFF_NOARP = 1 << 7 + IFF_PROMISC = 1 << 8 + IFF_ALLMULTI = 1 << 9 + IFF_MASTER = 1 << 10 + IFF_SLAVE = 1 << 11 + IFF_MULTICAST = 1 << 12 + IFF_PORTSEL = 1 << 13 + IFF_AUTOMEDIA = 1 << 14 + IFF_DYNAMIC = 1 << 15 + IFF_LOWER_UP = 1 << 16 + IFF_DORMANT = 1 << 17 + IFF_ECHO = 1 << 18 +) + +// Interface link attributes, from uapi/linux/if_link.h. +const ( + IFLA_UNSPEC = 0 + IFLA_ADDRESS = 1 + IFLA_BROADCAST = 2 + IFLA_IFNAME = 3 + IFLA_MTU = 4 + IFLA_LINK = 5 + IFLA_QDISC = 6 + IFLA_STATS = 7 + IFLA_COST = 8 + IFLA_PRIORITY = 9 + IFLA_MASTER = 10 + IFLA_WIRELESS = 11 + IFLA_PROTINFO = 12 + IFLA_TXQLEN = 13 + IFLA_MAP = 14 + IFLA_WEIGHT = 15 + IFLA_OPERSTATE = 16 + IFLA_LINKMODE = 17 + IFLA_LINKINFO = 18 + IFLA_NET_NS_PID = 19 + IFLA_IFALIAS = 20 + IFLA_NUM_VF = 21 + IFLA_VFINFO_LIST = 22 + IFLA_STATS64 = 23 + IFLA_VF_PORTS = 24 + IFLA_PORT_SELF = 25 + IFLA_AF_SPEC = 26 + IFLA_GROUP = 27 + IFLA_NET_NS_FD = 28 + IFLA_EXT_MASK = 29 + IFLA_PROMISCUITY = 30 + IFLA_NUM_TX_QUEUES = 31 + IFLA_NUM_RX_QUEUES = 32 + IFLA_CARRIER = 33 + IFLA_PHYS_PORT_ID = 34 + IFLA_CARRIER_CHANGES = 35 + IFLA_PHYS_SWITCH_ID = 36 + IFLA_LINK_NETNSID = 37 + IFLA_PHYS_PORT_NAME = 38 + IFLA_PROTO_DOWN = 39 + IFLA_GSO_MAX_SEGS = 40 + IFLA_GSO_MAX_SIZE = 41 +) + +// Interface link info attributes, from uapi/linux/if_link.h. +const ( + IFLA_INFO_UNSPEC = 0 + IFLA_INFO_KIND = 1 + IFLA_INFO_DATA = 2 + IFLA_INFO_XSTATS = 3 + IFLA_INFO_SLAVE_KIND = 4 + IFLA_INFO_SLAVE_DATA = 5 +) + +// Virtuall ethernet attributes, from uapi/linux/veth.h. +const ( + VETH_INFO_PEER = 1 +) + +// InterfaceAddrMessage is struct ifaddrmsg, from uapi/linux/if_addr.h. +// +// +marshal +type InterfaceAddrMessage struct { + Family uint8 + PrefixLen uint8 + Flags uint8 + Scope uint8 + Index uint32 +} + +// InterfaceAddrMessageSize is the size of InterfaceAddrMessage. +const InterfaceAddrMessageSize = 8 + +// Interface attributes, from uapi/linux/if_addr.h. +const ( + IFA_UNSPEC = 0 + IFA_ADDRESS = 1 + IFA_LOCAL = 2 + IFA_LABEL = 3 + IFA_BROADCAST = 4 + IFA_ANYCAST = 5 + IFA_CACHEINFO = 6 + IFA_MULTICAST = 7 + IFA_FLAGS = 8 +) + +// Device types, from uapi/linux/if_arp.h. +const ( + ARPHRD_NONE = 65534 + ARPHRD_ETHER = 1 + ARPHRD_LOOPBACK = 772 +) + +// RouteMessage is struct rtmsg, from uapi/linux/rtnetlink.h. +// +// +marshal +type RouteMessage struct { + Family uint8 + DstLen uint8 + SrcLen uint8 + TOS uint8 + + Table uint8 + Protocol uint8 + Scope uint8 + Type uint8 + + Flags uint32 +} + +// SizeOfRouteMessage is the size of RouteMessage. +const SizeOfRouteMessage = 12 + +// Route types, from uapi/linux/rtnetlink.h. +const ( + // RTN_UNSPEC represents an unspecified route type. + RTN_UNSPEC = 0 + + // RTN_UNICAST represents a unicast route. + RTN_UNICAST = 1 + + // RTN_LOCAL represents a route that is accepted locally. + RTN_LOCAL = 2 + + // RTN_BROADCAST represents a broadcast route (Traffic is accepted locally + // as broadcast, and sent as broadcast). + RTN_BROADCAST = 3 + + // RTN_ANYCAST represents a anycast route (Traffic is accepted locally as + // broadcast but sent as unicast). + RTN_ANYCAST = 6 + + // RTN_MULTICAST represents a multicast route. + RTN_MULTICAST = 5 + + // RTN_BLACKHOLE represents a route where all traffic is dropped. + RTN_BLACKHOLE = 6 + + // RTN_UNREACHABLE represents a route where the destination is unreachable. + RTN_UNREACHABLE = 7 + + RTN_PROHIBIT = 8 + RTN_THROW = 9 + RTN_NAT = 10 + RTN_XRESOLVE = 11 +) + +// Route protocols/origins, from uapi/linux/rtnetlink.h. +const ( + RTPROT_UNSPEC = 0 + RTPROT_REDIRECT = 1 + RTPROT_KERNEL = 2 + RTPROT_BOOT = 3 + RTPROT_STATIC = 4 + RTPROT_GATED = 8 + RTPROT_RA = 9 + RTPROT_MRT = 10 + RTPROT_ZEBRA = 11 + RTPROT_BIRD = 12 + RTPROT_DNROUTED = 13 + RTPROT_XORP = 14 + RTPROT_NTK = 15 + RTPROT_DHCP = 16 + RTPROT_MROUTED = 17 + RTPROT_BABEL = 42 + RTPROT_BGP = 186 + RTPROT_ISIS = 187 + RTPROT_OSPF = 188 + RTPROT_RIP = 189 + RTPROT_EIGRP = 192 +) + +// Route scopes, from uapi/linux/rtnetlink.h. +const ( + RT_SCOPE_UNIVERSE = 0 + RT_SCOPE_SITE = 200 + RT_SCOPE_LINK = 253 + RT_SCOPE_HOST = 254 + RT_SCOPE_NOWHERE = 255 +) + +// Route flags, from uapi/linux/rtnetlink.h. +const ( + RTM_F_NOTIFY = 0x100 + RTM_F_CLONED = 0x200 + RTM_F_EQUALIZE = 0x400 + RTM_F_PREFIX = 0x800 + RTM_F_LOOKUP_TABLE = 0x1000 + RTM_F_FIB_MATCH = 0x2000 +) + +// Route tables, from uapi/linux/rtnetlink.h. +const ( + RT_TABLE_UNSPEC = 0 + RT_TABLE_COMPAT = 252 + RT_TABLE_DEFAULT = 253 + RT_TABLE_MAIN = 254 + RT_TABLE_LOCAL = 255 +) + +// Route attributes, from uapi/linux/rtnetlink.h. +const ( + RTA_UNSPEC = 0 + RTA_DST = 1 + RTA_SRC = 2 + RTA_IIF = 3 + RTA_OIF = 4 + RTA_GATEWAY = 5 + RTA_PRIORITY = 6 + RTA_PREFSRC = 7 + RTA_METRICS = 8 + RTA_MULTIPATH = 9 + RTA_PROTOINFO = 10 + RTA_FLOW = 11 + RTA_CACHEINFO = 12 + RTA_SESSION = 13 + RTA_MP_ALGO = 14 + RTA_TABLE = 15 + RTA_MARK = 16 + RTA_MFC_STATS = 17 + RTA_VIA = 18 + RTA_NEWDST = 19 + RTA_PREF = 20 + RTA_ENCAP_TYPE = 21 + RTA_ENCAP = 22 + RTA_EXPIRES = 23 + RTA_PAD = 24 + RTA_UID = 25 + RTA_TTL_PROPAGATE = 26 + RTA_IP_PROTO = 27 + RTA_SPORT = 28 + RTA_DPORT = 29 +) + +// Route flags, from include/uapi/linux/route.h. +const ( + RTF_GATEWAY = 0x2 + RTF_UP = 0x1 +) + +// RtAttr is the header of optional addition route information, as a netlink +// attribute. From include/uapi/linux/rtnetlink.h. +// +// +marshal +type RtAttr struct { + Len uint16 + Type uint16 +} + +// SizeOfRtAttr is the size of RtAttr. +const SizeOfRtAttr = 4 diff --git a/pkg/abi/linux/nf_tables.go b/pkg/abi/linux/nf_tables.go new file mode 100644 index 0000000..dcd1d9e --- /dev/null +++ b/pkg/abi/linux/nf_tables.go @@ -0,0 +1,464 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// This file contains constants required to support nf_tables. + +const NFT_MAX_HOOKS = NF_INET_NUMHOOKS + 1 + +// Name length constants for nf_table structures. These correspond to values in +// include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_NAME_MAXLEN = 256 + NFT_TABLE_MAXNAMELEN = NFT_NAME_MAXLEN + NFT_CHAIN_MAXNAMELEN = NFT_NAME_MAXLEN + NFT_SET_MAXNAMELEN = NFT_NAME_MAXLEN + NFT_OBJ_MAXNAMELEN = NFT_NAME_MAXLEN + NFT_USERDATA_MAXLEN = 256 + NFT_OSF_MAXGENRELEN = 16 +) + +// 16-byte Registers that can be used to maintain state for rules. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_REG_VERDICT = iota + NFT_REG_1 + NFT_REG_2 + NFT_REG_3 + NFT_REG_4 + __NFT_REG_MAX +) + +// 4-byte Registers that can be used to maintain state for rules. +// Note that these overlap with the 16-byte registers in memory. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_REG32_00 = 8 + iota + NFT_REG32_01 + NFT_REG32_02 + NFT_REG32_03 + NFT_REG32_04 + NFT_REG32_05 + NFT_REG32_06 + NFT_REG32_07 + NFT_REG32_08 + NFT_REG32_09 + NFT_REG32_10 + NFT_REG32_11 + NFT_REG32_12 + NFT_REG32_13 + NFT_REG32_14 + NFT_REG32_15 +) + +// Other register constants, corresponding to values in +// include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_REG_MAX = __NFT_REG_MAX - 1 // Maximum register value + NFT_REG_SIZE = 16 // Size of NFT_REG + NFT_REG32_SIZE = 4 // Size of NFT_REG32 + NFT_REG32_COUNT = NFT_REG32_15 - NFT_REG32_00 + 1 // Count of 4-byte registers +) + +// Internal nf table verdicts. These are used for ruleset evaluation and +// are not returned to userspace. +// +// These also share their numeric name space with the netfilter verdicts. When +// used these values are converted to uint32 (purposefully overflowing the int). +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + // Continue evaluation of the current rule. + NFT_CONTINUE int32 = -1 + + // Terminate evaluation of the current rule. + NFT_BREAK int32 = -2 + + // Push the current chain on the jump stack and jump to a chain. + NFT_JUMP int32 = -3 + + // Jump to a chain without pushing the current chain on the jump stack. + NFT_GOTO int32 = -4 + + // Return to the topmost chain on the jump stack. + NFT_RETURN int32 = -5 +) + +// NfTableMsgType values map to operations within the nftables api. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +type NfTableMsgType uint16 + +// Netlink Netfilter table message types. +const ( + NFT_MSG_NEWTABLE NfTableMsgType = iota + NFT_MSG_GETTABLE + NFT_MSG_DELTABLE + NFT_MSG_NEWCHAIN + NFT_MSG_GETCHAIN + NFT_MSG_DELCHAIN + NFT_MSG_NEWRULE + NFT_MSG_GETRULE + NFT_MSG_DELRULE + NFT_MSG_NEWSET + NFT_MSG_GETSET + NFT_MSG_DELSET + NFT_MSG_NEWSETELEM + NFT_MSG_GETSETELEM + NFT_MSG_DELSETELEM + NFT_MSG_NEWGEN + NFT_MSG_GETGEN + NFT_MSG_TRACE + NFT_MSG_NEWOBJ + NFT_MSG_GETOBJ + NFT_MSG_DELOBJ + NFT_MSG_GETOBJ_RESET + NFT_MSG_NEWFLOWTABLE + NFT_MSG_GETFLOWTABLE + NFT_MSG_DELFLOWTABLE + NFT_MSG_GETRULE_RESET + NFT_MSG_DESTROYTABLE + NFT_MSG_DESTROYCHAIN + NFT_MSG_DESTROYRULE + NFT_MSG_DESTROYSET + NFT_MSG_DESTROYSETELEM + NFT_MSG_DESTROYOBJ + NFT_MSG_DESTROYFLOWTABLE + NFT_MSG_GETSETELEM_RESET + NFT_MSG_MAX +) + +var nfTableMsgTypeStrings = [...]string{ + NFT_MSG_NEWTABLE: "NFT_MSG_NEWTABLE", + NFT_MSG_GETTABLE: "NFT_MSG_GETTABLE", + NFT_MSG_DELTABLE: "NFT_MSG_DELTABLE", + NFT_MSG_NEWCHAIN: "NFT_MSG_NEWCHAIN", + NFT_MSG_GETCHAIN: "NFT_MSG_GETCHAIN", + NFT_MSG_DELCHAIN: "NFT_MSG_DELCHAIN", + NFT_MSG_NEWRULE: "NFT_MSG_NEWRULE", + NFT_MSG_GETRULE: "NFT_MSG_GETRULE", + NFT_MSG_DELRULE: "NFT_MSG_DELRULE", + NFT_MSG_NEWSET: "NFT_MSG_NEWSET", + NFT_MSG_GETSET: "NFT_MSG_GETSET", + NFT_MSG_DELSET: "NFT_MSG_DELSET", + NFT_MSG_NEWSETELEM: "NFT_MSG_NEWSETELEM", + NFT_MSG_GETSETELEM: "NFT_MSG_GETSETELEM", + NFT_MSG_DELSETELEM: "NFT_MSG_DELSETELEM", + NFT_MSG_NEWGEN: "NFT_MSG_NEWGEN", + NFT_MSG_GETGEN: "NFT_MSG_GETGEN", + NFT_MSG_TRACE: "NFT_MSG_TRACE", + NFT_MSG_NEWOBJ: "NFT_MSG_NEWOBJ", + NFT_MSG_GETOBJ: "NFT_MSG_GETOBJ", + NFT_MSG_DELOBJ: "NFT_MSG_DELOBJ", + NFT_MSG_GETOBJ_RESET: "NFT_MSG_GETOBJ_RESET", + NFT_MSG_NEWFLOWTABLE: "NFT_MSG_NEWFLOWTABLE", + NFT_MSG_GETFLOWTABLE: "NFT_MSG_GETFLOWTABLE", + NFT_MSG_DELFLOWTABLE: "NFT_MSG_DELFLOWTABLE", + NFT_MSG_GETRULE_RESET: "NFT_MSG_GETRULE_RESET", + NFT_MSG_DESTROYTABLE: "NFT_MSG_DESTROYTABLE", + NFT_MSG_DESTROYCHAIN: "NFT_MSG_DESTROYCHAIN", + NFT_MSG_DESTROYRULE: "NFT_MSG_DESTROYRULE", + NFT_MSG_DESTROYSET: "NFT_MSG_DESTROYSET", + NFT_MSG_DESTROYSETELEM: "NFT_MSG_DESTROYSETELEM", + NFT_MSG_DESTROYOBJ: "NFT_MSG_DESTROYOBJ", + NFT_MSG_DESTROYFLOWTABLE: "NFT_MSG_DESTROYFLOWTABLE", + NFT_MSG_GETSETELEM_RESET: "NFT_MSG_GETSETELEM_RESET", + NFT_MSG_MAX: "NFT_MSG_MAX", +} + +// String returns the string representation of the NfTableMsgType. +func (msg NfTableMsgType) String() string { + if int(msg) < len(nfTableMsgTypeStrings) { + return nfTableMsgTypeStrings[msg] + } + return "UNKNOWN" +} + +// NfTableListAttributes represents the netfilter attributes for lists of data. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_LIST_UNSPEC uint16 = iota + NFTA_LIST_ELEM + __NFTA_LIST_MAX + NFTA_LIST_MAX = __NFTA_LIST_MAX - 1 +) + +// NfTableHookAttributes represents the netfilter hook attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_HOOK_UNSPEC uint16 = iota + NFTA_HOOK_HOOKNUM + NFTA_HOOK_PRIORITY + NFTA_HOOK_DEV + NFTA_HOOK_DEVS + __NFTA_HOOK_MAX + NFTA_HOOK_MAX = __NFTA_HOOK_MAX - 1 +) + +// NfTableFlags represents table flags that can be set for a table, namely dormant. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_TABLE_F_DORMANT uint32 = 0x1 + NFT_TABLE_F_OWNER = 0x2 + NFT_TABLE_F_PERSIST = 0x4 + NFT_TABLE_F_MASK = NFT_TABLE_F_DORMANT | NFT_TABLE_F_OWNER | NFT_TABLE_F_PERSIST +) + +// NfTableAttributes represents the netfilter table attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_TABLE_UNSPEC uint16 = iota + NFTA_TABLE_NAME + NFTA_TABLE_FLAGS + NFTA_TABLE_USE + NFTA_TABLE_HANDLE + NFTA_TABLE_PAD + NFTA_TABLE_USERDATA + NFTA_TABLE_OWNER + __NFTA_TABLE_MAX +) + +// NFTA_TABLE_MAX is the maximum netfilter table attribute. +const NFTA_TABLE_MAX = __NFTA_TABLE_MAX - 1 + +// NfTableChainFlags represents chain flags that can be set for a chain. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_CHAIN_BASE uint32 = (1 << 0) + NFT_CHAIN_HW_OFFLOAD = (1 << 1) + NFT_CHAIN_BINDING = (1 << 2) + NFT_CHAIN_FLAGS = (NFT_CHAIN_BASE | NFT_CHAIN_HW_OFFLOAD | NFT_CHAIN_BINDING) +) + +// NfTableChainAttributes represents the netfilter chain attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_CHAIN_UNSPEC uint16 = iota + NFTA_CHAIN_TABLE + NFTA_CHAIN_HANDLE + NFTA_CHAIN_NAME + NFTA_CHAIN_HOOK + NFTA_CHAIN_POLICY + NFTA_CHAIN_USE + NFTA_CHAIN_TYPE + NFTA_CHAIN_COUNTERS + NFTA_CHAIN_PAD + NFTA_CHAIN_FLAGS + NFTA_CHAIN_ID + NFTA_CHAIN_USERDATA + __NFTA_CHAIN_MAX + NFTA_CHAIN_MAX = __NFTA_CHAIN_MAX - 1 +) + +// NfTableRuleAttributes represents the netfilter rule attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_RULE_UNSPEC uint16 = iota + NFTA_RULE_TABLE + NFTA_RULE_CHAIN + NFTA_RULE_HANDLE + NFTA_RULE_EXPRESSIONS + NFTA_RULE_COMPAT + NFTA_RULE_POSITION + NFTA_RULE_USERDATA + NFTA_RULE_PAD + NFTA_RULE_ID + NFTA_RULE_POSITION_ID + NFTA_RULE_CHAIN_ID + __NFTA_RULE_MAX + NFTA_RULE_MAX = __NFTA_RULE_MAX - 1 +) + +// NfTableDataTypes represents the netfilter data types. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_DATA_VALUE = iota + NFT_DATA_VERDICT = 0xffffff00 +) + +// NfTableDataReservedMask represents the netfilter data reserved mask for internally used types. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_DATA_RESERVED_MASK = 0xffffff00 +) + +// NfTableDataAttributes represents the netfilter data attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_DATA_UNSPEC uint16 = iota + NFTA_DATA_VALUE + NFTA_DATA_VERDICT + __NFTA_DATA_MAX + NFTA_DATA_MAX = __NFTA_DATA_MAX - 1 +) + +// NFT_DATA_VALUE_MAXLEN is the maximum length of a netfilter data value. +const NFT_DATA_VALUE_MAXLEN = 64 + +// NfTableVerdictAttributes represents the netfilter verdict attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_VERDICT_UNSPEC uint16 = iota + NFTA_VERDICT_CODE + NFTA_VERDICT_CHAIN + NFTA_VERDICT_CHAIN_ID + __NFTA_VERDICT_MAX + NFTA_VERDICT_MAX = __NFTA_VERDICT_MAX - 1 +) + +// NfTableExprAttributes represents the netfilter expression attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_EXPR_UNSPEC uint16 = iota + NFTA_EXPR_NAME + NFTA_EXPR_DATA + __NFTA_EXPR_MAX + NFTA_EXPR_MAX = __NFTA_EXPR_MAX - 1 +) + +// NfTableImmediateAttributes represents the netfilter immediate attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_IMMEDIATE_UNSPEC uint16 = iota + NFTA_IMMEDIATE_DREG + NFTA_IMMEDIATE_DATA + __NFTA_IMMEDIATE_MAX + NFTA_IMMEDIATE_MAX = __NFTA_IMMEDIATE_MAX - 1 +) + +// Nf table relational operators. +// Used by the nft comparison operation to compare values in registers. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_CMP_EQ = iota // equal + NFT_CMP_NEQ // not equal + NFT_CMP_LT // less than + NFT_CMP_LTE // less than or equal to + NFT_CMP_GT // greater than + NFT_CMP_GTE // greater than or equal to +) + +// Nf table range operators. +// Used by the nft range operation to compare values in registers. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_RANGE_EQ = iota + NFT_RANGE_NEQ +) + +// Nf table payload expression offset bases. +// Used by the nft payload operations to access appropriate data in the packet. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_PAYLOAD_LL_HEADER = iota // link layer header + NFT_PAYLOAD_NETWORK_HEADER // network header + NFT_PAYLOAD_TRANSPORT_HEADER // transport header + NFT_PAYLOAD_INNER_HEADER // inner header / payload + NFT_PAYLOAD_TUN_HEADER // tunneling protocol header +) + +// Nf table payload expression checksum types. +// Used by the nft payload set operation to mark the type of checksum to use. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_PAYLOAD_CSUM_NONE = iota // no checksumming + NFT_PAYLOAD_CSUM_INET // internet checksum (RFC 791) + NFT_PAYLOAD_CSUM_SCTP // CRC-32c, for use in SCTP header (RFC 3309) +) + +// Nf table payload expression checksum flags. +// Used by the nft payload set operation to mark the flags for checksumming. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_PAYLOAD_L4CSUM_PSEUDOHDR = (1 << 0) // use pseudoheader for L4 checksum +) + +// Nf table bitwise operators. +// Used by the nft bitwise operation to perform bitwise math over register data. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_BITWISE_BOOL = iota // mask-and-xor operation for NOT, AND, OR, & XOR + NFT_BITWISE_LSHIFT // left-shift operation + NFT_BITWISE_RSHIFT // right-shift operation +) + +// Nf table route expression keys. +// Used by the nft route operation to determine the routing data to retrieve. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + // Traffic Class Identifier (Realm) associated with route + NFT_RT_CLASSID = iota + + // Routing nexthop for IPv4 (next IPv4 address to jump to) + NFT_RT_NEXTHOP4 + + // Routing nexthop for IPv6 (next IPv6 address to jump to) + NFT_RT_NEXTHOP6 + + // Maximum Segment Size for TCP connections (largest size for a single packet) + NFT_RT_TCPMSS + + // Bool for whether packet route involves a IPsec transform st xfrm is applied + NFT_RT_XFRM +) + +// Nf table byteorder operators. +// Used by the nft byteorder operation to convert data in a register to a +// specific byte order. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_BYTEORDER_NTOH = iota // network to host operator + NFT_BYTEORDER_HTON // host to network operator +) + +// Nf tables meta expression keys. +// Used by the nft meta operation to retrieve meta data from the packet. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_META_LEN = iota // Packet length + NFT_META_PROTOCOL // Packet ethertype protocol, invalid in OUTPUT + NFT_META_PRIORITY // Packet priority + NFT_META_MARK // Packet mark + NFT_META_IIF // Packet input interface index + NFT_META_OIF // Packet output interface index + NFT_META_IIFNAME // Packet input interface name + NFT_META_OIFNAME // Packet output interface name + NFT_META_IIFTYPE // Packet input interface type + NFT_META_OIFTYPE // Packet output interface type + NFT_META_SKUID // Originating socket UID + NFT_META_SKGID // Originating socket GID + NFT_META_NFTRACE // Packet nftrace bit + NFT_META_RTCLASSID // Realm value of packet's route + NFT_META_SECMARK // Packet secmark + NFT_META_NFPROTO // Netfilter protocol + NFT_META_L4PROTO // Layer 4 protocol number + NFT_META_BRI_IIFNAME // Packet input bridge interface name + NFT_META_BRI_OIFNAME // Packet output bridge interface name + NFT_META_PKTTYPE // Packet type, special handling for loopback + NFT_META_CPU // CPU id through smp_processor_id() + NFT_META_IIFGROUP // Packet input interface group + NFT_META_OIFGROUP // Packet output interface group + NFT_META_CGROUP // Socket control group + NFT_META_PRANDOM // A 32bit pseudo-random number + NFT_META_SECPATH // Boolean, secpath_exists + NFT_META_IIFKIND // Packet input interface kind name + NFT_META_OIFKIND // Packet output interface kind name + NFT_META_BRI_IIFPVID // Packet input bridge port pvid + NFT_META_BRI_IIFVPROTO // Packet input bridge vlan proto + NFT_META_TIME_NS // Time since epoch (in nanoseconds) + NFT_META_TIME_DAY // Day of week (from 0 = Sunday to 6 = Saturday) + NFT_META_TIME_HOUR // Hour of day (in sec), secs since start of day + NFT_META_SDIF // Slave device interface index + NFT_META_SDIFNAME // Slave device interface name + NFT_META_BRI_BROUTE // Packet br_netfilter_broute bit +) diff --git a/pkg/abi/linux/poll.go b/pkg/abi/linux/poll.go new file mode 100644 index 0000000..3443a57 --- /dev/null +++ b/pkg/abi/linux/poll.go @@ -0,0 +1,44 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// PollFD is struct pollfd, used by poll(2)/ppoll(2), from uapi/asm-generic/poll.h. +// +// +marshal slice:PollFDSlice +type PollFD struct { + FD int32 + Events int16 + REvents int16 +} + +// Poll event flags, used by poll(2)/ppoll(2) and/or +// epoll_ctl(2)/epoll_wait(2), from uapi/asm-generic/poll.h. +const ( + POLLIN = 0x0001 + POLLPRI = 0x0002 + POLLOUT = 0x0004 + POLLERR = 0x0008 + POLLHUP = 0x0010 + POLLNVAL = 0x0020 + POLLRDNORM = 0x0040 + POLLRDBAND = 0x0080 + POLLWRNORM = 0x0100 + POLLWRBAND = 0x0200 + POLLMSG = 0x0400 + POLLREMOVE = 0x1000 + POLLRDHUP = 0x2000 + POLLFREE = 0x4000 + POLL_BUSY_LOOP = 0x8000 +) diff --git a/pkg/abi/linux/prctl.go b/pkg/abi/linux/prctl.go new file mode 100644 index 0000000..c1928ae --- /dev/null +++ b/pkg/abi/linux/prctl.go @@ -0,0 +1,179 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// PR_* flags, from for prctl(2). +const ( + // PR_SET_PDEATHSIG sets the process' death signal. + PR_SET_PDEATHSIG = 1 + + // PR_GET_PDEATHSIG gets the process' death signal. + PR_GET_PDEATHSIG = 2 + + // PR_GET_DUMPABLE gets the process' dumpable flag. + PR_GET_DUMPABLE = 3 + + // PR_SET_DUMPABLE sets the process' dumpable flag. + PR_SET_DUMPABLE = 4 + + // PR_GET_KEEPCAPS gets the value of the keep capabilities flag. + PR_GET_KEEPCAPS = 7 + + // PR_SET_KEEPCAPS sets the value of the keep capabilities flag. + PR_SET_KEEPCAPS = 8 + + // PR_GET_TIMING gets the process' timing method. + PR_GET_TIMING = 13 + + // PR_SET_TIMING sets the process' timing method. + PR_SET_TIMING = 14 + + // PR_SET_NAME sets the process' name. + PR_SET_NAME = 15 + + // PR_GET_NAME gets the process' name. + PR_GET_NAME = 16 + + // PR_GET_SECCOMP gets a process' seccomp mode. + PR_GET_SECCOMP = 21 + + // PR_SET_SECCOMP sets a process' seccomp mode. + PR_SET_SECCOMP = 22 + + // PR_CAPBSET_READ gets the capability bounding set. + PR_CAPBSET_READ = 23 + + // PR_CAPBSET_DROP sets the capability bounding set. + PR_CAPBSET_DROP = 24 + + // PR_GET_TSC gets the value of the flag determining whether the + // timestamp counter can be read. + PR_GET_TSC = 25 + + // PR_SET_TSC sets the value of the flag determining whether the + // timestamp counter can be read. + PR_SET_TSC = 26 + + // PR_SET_TIMERSLACK sets the process' time slack. + PR_SET_TIMERSLACK = 29 + + // PR_GET_TIMERSLACK gets the process' time slack. + PR_GET_TIMERSLACK = 30 + + // PR_TASK_PERF_EVENTS_DISABLE disables all performance counters + // attached to the calling process. + PR_TASK_PERF_EVENTS_DISABLE = 31 + + // PR_TASK_PERF_EVENTS_ENABLE enables all performance counters attached + // to the calling process. + PR_TASK_PERF_EVENTS_ENABLE = 32 + + // PR_MCE_KILL sets the machine check memory corruption kill policy for + // the calling thread. + PR_MCE_KILL = 33 + + // PR_MCE_KILL_GET gets the machine check memory corruption kill policy + // for the calling thread. + PR_MCE_KILL_GET = 34 + + // PR_SET_MM modifies certain kernel memory map descriptor fields of + // the calling process. See prctl(2) for more information. + PR_SET_MM = 35 + + PR_SET_MM_START_CODE = 1 + PR_SET_MM_END_CODE = 2 + PR_SET_MM_START_DATA = 3 + PR_SET_MM_END_DATA = 4 + PR_SET_MM_START_STACK = 5 + PR_SET_MM_START_BRK = 6 + PR_SET_MM_BRK = 7 + PR_SET_MM_ARG_START = 8 + PR_SET_MM_ARG_END = 9 + PR_SET_MM_ENV_START = 10 + PR_SET_MM_ENV_END = 11 + PR_SET_MM_AUXV = 12 + // PR_SET_MM_EXE_FILE supersedes the /proc/pid/exe symbolic link with a + // new one pointing to a new executable file identified by the file + // descriptor provided in arg3 argument. See prctl(2) for more + // information. + PR_SET_MM_EXE_FILE = 13 + PR_SET_MM_MAP = 14 + PR_SET_MM_MAP_SIZE = 15 + + // PR_SET_CHILD_SUBREAPER sets the "child subreaper" attribute of the + // calling process. + PR_SET_CHILD_SUBREAPER = 36 + + // PR_GET_CHILD_SUBREAPER gets the "child subreaper" attribute of the + // calling process. + PR_GET_CHILD_SUBREAPER = 37 + + // PR_SET_NO_NEW_PRIVS sets the calling thread's no_new_privs bit. + PR_SET_NO_NEW_PRIVS = 38 + + // PR_GET_NO_NEW_PRIVS gets the calling thread's no_new_privs bit. + PR_GET_NO_NEW_PRIVS = 39 + + // PR_GET_TID_ADDRESS retrieves the clear_child_tid address. + PR_GET_TID_ADDRESS = 40 + + // PR_SET_THP_DISABLE sets the state of the "THP disable" flag for the + // calling thread. + PR_SET_THP_DISABLE = 41 + + // PR_GET_THP_DISABLE gets the state of the "THP disable" flag for the + // calling thread. + PR_GET_THP_DISABLE = 42 + + // PR_MPX_ENABLE_MANAGEMENT enables kernel management of Memory + // Protection eXtensions (MPX) bounds tables. + PR_MPX_ENABLE_MANAGEMENT = 43 + + // PR_MPX_DISABLE_MANAGEMENT disables kernel management of Memory + // Protection eXtensions (MPX) bounds tables. + PR_MPX_DISABLE_MANAGEMENT = 44 + + // The following constants are used to control thread scheduling on cores. + PR_SCHED_CORE_SCOPE_THREAD = 0 + PR_SCHED_CORE_SCOPE_THREAD_GROUP = 1 + + // PR_SET_VMA sets VMA attributes. + PR_SET_VMA = 0x53564d41 + PR_SET_VMA_ANON_NAME = 0 + // From kernel/sys.c: + ANON_VMA_NAME_MAX_LEN = 80 + + // PR_SET_PTRACER allows a specific process (or any, if PR_SET_PTRACER_ANY is + // specified) to ptrace the current task. + PR_SET_PTRACER = 0x59616d61 + PR_SET_PTRACER_ANY = -1 +) + +// From +// Flags are used in syscall arch_prctl(2). +const ( + ARCH_SET_GS = 0x1001 + ARCH_SET_FS = 0x1002 + ARCH_GET_FS = 0x1003 + ARCH_GET_GS = 0x1004 + ARCH_SET_CPUID = 0x1012 +) + +// Flags for prctl(PR_SET_DUMPABLE), defined in include/linux/sched/coredump.h. +const ( + SUID_DUMP_DISABLE = 0 + SUID_DUMP_USER = 1 + SUID_DUMP_ROOT = 2 +) diff --git a/pkg/abi/linux/ptrace.go b/pkg/abi/linux/ptrace.go new file mode 100644 index 0000000..db1c6a0 --- /dev/null +++ b/pkg/abi/linux/ptrace.go @@ -0,0 +1,95 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// ptrace commands from include/uapi/linux/ptrace.h. +const ( + PTRACE_TRACEME = 0 + PTRACE_PEEKTEXT = 1 + PTRACE_PEEKDATA = 2 + PTRACE_PEEKUSR = 3 + PTRACE_POKETEXT = 4 + PTRACE_POKEDATA = 5 + PTRACE_POKEUSR = 6 + PTRACE_CONT = 7 + PTRACE_KILL = 8 + PTRACE_SINGLESTEP = 9 + PTRACE_ATTACH = 16 + PTRACE_DETACH = 17 + PTRACE_SYSCALL = 24 + PTRACE_SETOPTIONS = 0x4200 + PTRACE_GETEVENTMSG = 0x4201 + PTRACE_GETSIGINFO = 0x4202 + PTRACE_SETSIGINFO = 0x4203 + PTRACE_GETREGSET = 0x4204 + PTRACE_SETREGSET = 0x4205 + PTRACE_SEIZE = 0x4206 + PTRACE_INTERRUPT = 0x4207 + PTRACE_LISTEN = 0x4208 + PTRACE_PEEKSIGINFO = 0x4209 + PTRACE_GETSIGMASK = 0x420a + PTRACE_SETSIGMASK = 0x420b + PTRACE_SECCOMP_GET_FILTER = 0x420c + PTRACE_SECCOMP_GET_METADATA = 0x420d +) + +// ptrace commands from arch/x86/include/uapi/asm/ptrace-abi.h. +const ( + PTRACE_GETREGS = 12 + PTRACE_SETREGS = 13 + PTRACE_GETFPREGS = 14 + PTRACE_SETFPREGS = 15 + PTRACE_GETFPXREGS = 18 + PTRACE_SETFPXREGS = 19 + PTRACE_OLDSETOPTIONS = 21 + PTRACE_GET_THREAD_AREA = 25 + PTRACE_SET_THREAD_AREA = 26 + PTRACE_ARCH_PRCTL = 30 + PTRACE_SYSEMU = 31 + PTRACE_SYSEMU_SINGLESTEP = 32 + PTRACE_SINGLEBLOCK = 33 +) + +// ptrace event codes from include/uapi/linux/ptrace.h. +const ( + PTRACE_EVENT_FORK = 1 + PTRACE_EVENT_VFORK = 2 + PTRACE_EVENT_CLONE = 3 + PTRACE_EVENT_EXEC = 4 + PTRACE_EVENT_VFORK_DONE = 5 + PTRACE_EVENT_EXIT = 6 + PTRACE_EVENT_SECCOMP = 7 + PTRACE_EVENT_STOP = 128 +) + +// PTRACE_SETOPTIONS options from include/uapi/linux/ptrace.h. +const ( + PTRACE_O_TRACESYSGOOD = 1 + PTRACE_O_TRACEFORK = 1 << PTRACE_EVENT_FORK + PTRACE_O_TRACEVFORK = 1 << PTRACE_EVENT_VFORK + PTRACE_O_TRACECLONE = 1 << PTRACE_EVENT_CLONE + PTRACE_O_TRACEEXEC = 1 << PTRACE_EVENT_EXEC + PTRACE_O_TRACEVFORKDONE = 1 << PTRACE_EVENT_VFORK_DONE + PTRACE_O_TRACEEXIT = 1 << PTRACE_EVENT_EXIT + PTRACE_O_TRACESECCOMP = 1 << PTRACE_EVENT_SECCOMP + PTRACE_O_EXITKILL = 1 << 20 + PTRACE_O_SUSPEND_SECCOMP = 1 << 21 +) + +// YAMA ptrace_scope levels from security/yama/yama_lsm.c. +const ( + YAMA_SCOPE_DISABLED = 0 + YAMA_SCOPE_RELATIONAL = 1 +) diff --git a/pkg/abi/linux/ptrace_amd64.go b/pkg/abi/linux/ptrace_amd64.go new file mode 100644 index 0000000..e970b5b --- /dev/null +++ b/pkg/abi/linux/ptrace_amd64.go @@ -0,0 +1,69 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package linux + +// PtraceRegs is the set of CPU registers exposed by ptrace. Source: +// syscall.PtraceRegs. +// +// +marshal +// +stateify savable +type PtraceRegs struct { + R15 uint64 + R14 uint64 + R13 uint64 + R12 uint64 + Rbp uint64 + Rbx uint64 + R11 uint64 + R10 uint64 + R9 uint64 + R8 uint64 + Rax uint64 + Rcx uint64 + Rdx uint64 + Rsi uint64 + Rdi uint64 + Orig_rax uint64 + Rip uint64 + Cs uint64 + Eflags uint64 + Rsp uint64 + Ss uint64 + Fs_base uint64 + Gs_base uint64 + Ds uint64 + Es uint64 + Fs uint64 + Gs uint64 +} + +// InstructionPointer returns the address of the next instruction to +// be executed. +func (p *PtraceRegs) InstructionPointer() uint64 { + return p.Rip +} + +// StackPointer returns the address of the Stack pointer. +func (p *PtraceRegs) StackPointer() uint64 { + return p.Rsp +} + +// SetStackPointer sets the stack pointer to the specified value. +func (p *PtraceRegs) SetStackPointer(sp uint64) { + p.Rsp = sp +} diff --git a/pkg/abi/linux/ptrace_arm64.go b/pkg/abi/linux/ptrace_arm64.go new file mode 100644 index 0000000..87c197d --- /dev/null +++ b/pkg/abi/linux/ptrace_arm64.go @@ -0,0 +1,77 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package linux + +const ( + // PSR bits + PSR_MODE_EL0t = 0x00000000 + PSR_MODE_EL1t = 0x00000004 + PSR_MODE_EL1h = 0x00000005 + PSR_MODE_EL2t = 0x00000008 + PSR_MODE_EL2h = 0x00000009 + PSR_MODE_EL3t = 0x0000000c + PSR_MODE_EL3h = 0x0000000d + PSR_MODE_MASK = 0x0000000f + + // AArch32 CPSR bits + PSR_MODE32_BIT = 0x00000010 + + // AArch64 SPSR bits + PSR_F_BIT = 0x00000040 + PSR_I_BIT = 0x00000080 + PSR_A_BIT = 0x00000100 + PSR_D_BIT = 0x00000200 + PSR_BTYPE_MASK = 0x00000c00 + PSR_SSBS_BIT = 0x00001000 + PSR_PAN_BIT = 0x00400000 + PSR_UAO_BIT = 0x00800000 + PSR_DIT_BIT = 0x01000000 + PSR_TCO_BIT = 0x02000000 + PSR_V_BIT = 0x10000000 + PSR_C_BIT = 0x20000000 + PSR_Z_BIT = 0x40000000 + PSR_N_BIT = 0x80000000 +) + +// PtraceRegs is the set of CPU registers exposed by ptrace. Source: +// syscall.PtraceRegs. +// +// +marshal +// +stateify savable +type PtraceRegs struct { + Regs [31]uint64 + Sp uint64 + Pc uint64 + Pstate uint64 +} + +// InstructionPointer returns the address of the next instruction to be +// executed. +func (p *PtraceRegs) InstructionPointer() uint64 { + return p.Pc +} + +// StackPointer returns the address of the Stack pointer. +func (p *PtraceRegs) StackPointer() uint64 { + return p.Sp +} + +// SetStackPointer sets the stack pointer to the specified value. +func (p *PtraceRegs) SetStackPointer(sp uint64) { + p.Sp = sp +} diff --git a/pkg/abi/linux/rseq.go b/pkg/abi/linux/rseq.go new file mode 100644 index 0000000..76253ba --- /dev/null +++ b/pkg/abi/linux/rseq.go @@ -0,0 +1,130 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Flags passed to rseq(2). +// +// Defined in include/uapi/linux/rseq.h. +const ( + // RSEQ_FLAG_UNREGISTER unregisters the current thread. + RSEQ_FLAG_UNREGISTER = 1 << 0 +) + +// Critical section flags used in RSeqCriticalSection.Flags and RSeq.Flags. +// +// Defined in include/uapi/linux/rseq.h. +const ( + // RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT inhibits restart on preemption. + RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT = 1 << 0 + + // RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL inhibits restart on signal + // delivery. + RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL = 1 << 1 + + // RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE inhibits restart on CPU + // migration. + RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE = 1 << 2 +) + +// RSeqCriticalSection describes a restartable sequences critical section. It +// is equivalent to struct rseq_cs, defined in include/uapi/linux/rseq.h. +// +// In userspace, this structure is always aligned to 32 bytes. +// +// +marshal +type RSeqCriticalSection struct { + // Version is the version of this structure. Version 0 is defined here. + Version uint32 + + // Flags are the critical section flags, defined above. + Flags uint32 + + // Start is the start address of the critical section. + Start uint64 + + // PostCommitOffset is the offset from Start of the first instruction + // outside of the critical section. + PostCommitOffset uint64 + + // Abort is the abort address. It must be outside the critical section, + // and the 4 bytes prior must match the abort signature. + Abort uint64 +} + +const ( + // SizeOfRSeqCriticalSection is the size of RSeqCriticalSection. + SizeOfRSeqCriticalSection = 32 + + // SizeOfRSeqSignature is the size of the signature immediately + // preceding RSeqCriticalSection.Abort. + SizeOfRSeqSignature = 4 +) + +// Special values for RSeq.CPUID, defined in include/uapi/linux/rseq.h. +const ( + // RSEQ_CPU_ID_UNINITIALIZED indicates that this thread has not + // performed rseq initialization. + RSEQ_CPU_ID_UNINITIALIZED = ^uint32(0) // -1 + + // RSEQ_CPU_ID_REGISTRATION_FAILED indicates that rseq initialization + // failed. + RSEQ_CPU_ID_REGISTRATION_FAILED = ^uint32(1) // -2 +) + +// RSeq is the thread-local restartable sequences config/status. It +// is equivalent to struct rseq, defined in include/uapi/linux/rseq.h. +// +// In userspace, this structure is always aligned to 32 bytes. +type RSeq struct { + // CPUIDStart contains the current CPU ID if rseq is initialized. + // + // This field should only be read by the thread which registered this + // structure, and must be read atomically. + CPUIDStart uint32 + + // CPUID contains the current CPU ID or one of the CPU ID special + // values defined above. + // + // This field should only be read by the thread which registered this + // structure, and must be read atomically. + CPUID uint32 + + // RSeqCriticalSection is a pointer to the current RSeqCriticalSection + // block, or NULL. It is reset to NULL by the kernel on restart or + // non-restarting preempt/signal. + // + // This field should only be written by the thread which registered + // this structure, and must be written atomically. + RSeqCriticalSection uint64 + + // Flags are the critical section flags that apply to all critical + // sections on this thread, defined above. + Flags uint32 +} + +const ( + // SizeOfRSeq is the size of RSeq. + // + // Note that RSeq is naively 24 bytes. However, it has 32-byte + // alignment, which in C increases sizeof to 32. That is the size that + // the Linux kernel uses. + SizeOfRSeq = 32 + + // AlignOfRSeq is the standard alignment of RSeq. + AlignOfRSeq = 32 + + // OffsetOfRSeqCriticalSection is the offset of RSeqCriticalSection in RSeq. + OffsetOfRSeqCriticalSection = 8 +) diff --git a/pkg/abi/linux/rusage.go b/pkg/abi/linux/rusage.go new file mode 100644 index 0000000..e29d0ac --- /dev/null +++ b/pkg/abi/linux/rusage.go @@ -0,0 +1,48 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Flags that may be used with wait4(2) and getrusage(2). +const ( + // wait4(2) uses this to aggregate RUSAGE_SELF and RUSAGE_CHILDREN. + RUSAGE_BOTH = -0x2 + + // getrusage(2) flags. + RUSAGE_CHILDREN = -0x1 + RUSAGE_SELF = 0x0 + RUSAGE_THREAD = 0x1 +) + +// Rusage represents the Linux struct rusage. +// +// +marshal +type Rusage struct { + UTime Timeval + STime Timeval + MaxRSS int64 + IXRSS int64 + IDRSS int64 + ISRSS int64 + MinFlt int64 + MajFlt int64 + NSwap int64 + InBlock int64 + OuBlock int64 + MsgSnd int64 + MsgRcv int64 + NSignals int64 + NVCSw int64 + NIvCSw int64 +} diff --git a/pkg/abi/linux/sched.go b/pkg/abi/linux/sched.go new file mode 100644 index 0000000..2a67921 --- /dev/null +++ b/pkg/abi/linux/sched.go @@ -0,0 +1,37 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Scheduling policies, exposed by sched_getscheduler(2)/sched_setscheduler(2). +const ( + SCHED_NORMAL = 0 + SCHED_FIFO = 1 + SCHED_RR = 2 + SCHED_BATCH = 3 + SCHED_IDLE = 5 + SCHED_DEADLINE = 6 + SCHED_MICROQ = 16 + + // SCHED_RESET_ON_FORK is a flag that indicates that the process is + // reverted back to SCHED_NORMAL on fork. + SCHED_RESET_ON_FORK = 0x40000000 +) + +// Scheduling priority group selectors. +const ( + PRIO_PGRP = 0x1 + PRIO_PROCESS = 0x0 + PRIO_USER = 0x2 +) diff --git a/pkg/abi/linux/seccomp.go b/pkg/abi/linux/seccomp.go new file mode 100644 index 0000000..81c307f --- /dev/null +++ b/pkg/abi/linux/seccomp.go @@ -0,0 +1,173 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import "fmt" + +// Seccomp constants taken from . +const ( + SECCOMP_MODE_NONE = 0 + SECCOMP_MODE_FILTER = 2 + + SECCOMP_RET_ACTION_FULL = 0xffff0000 + SECCOMP_RET_ACTION = 0x7fff0000 + SECCOMP_RET_DATA = 0x0000ffff + + SECCOMP_SET_MODE_FILTER = 1 + SECCOMP_GET_ACTION_AVAIL = 2 + SECCOMP_GET_NOTIF_SIZES = 3 + + SECCOMP_FILTER_FLAG_TSYNC = 1 + SECCOMP_FILTER_FLAG_NEW_LISTENER = 1 << 3 + + SECCOMP_USER_NOTIF_FLAG_CONTINUE = 1 + + SECCOMP_IOCTL_NOTIF_RECV = 0xc0502100 + SECCOMP_IOCTL_NOTIF_SEND = 0xc0182101 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 + + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP = 1 +) + +// BPFAction is an action for a BPF filter. +type BPFAction uint32 + +// BPFAction definitions. +const ( + SECCOMP_RET_KILL_PROCESS BPFAction = 0x80000000 + SECCOMP_RET_KILL_THREAD BPFAction = 0x00000000 + SECCOMP_RET_TRAP BPFAction = 0x00030000 + SECCOMP_RET_ERRNO BPFAction = 0x00050000 + SECCOMP_RET_TRACE BPFAction = 0x7ff00000 + SECCOMP_RET_USER_NOTIF BPFAction = 0x7fc00000 + SECCOMP_RET_ALLOW BPFAction = 0x7fff0000 +) + +func (a BPFAction) String() string { + switch a & SECCOMP_RET_ACTION_FULL { + case SECCOMP_RET_KILL_PROCESS: + return "kill process" + case SECCOMP_RET_KILL_THREAD: + return "kill thread" + case SECCOMP_RET_TRAP: + data := a.Data() + if data == 0 { + return "trap" + } + return fmt.Sprintf("trap (data=%#x)", data) + case SECCOMP_RET_ERRNO: + return fmt.Sprintf("return errno=%#x", a.Data()) + case SECCOMP_RET_TRACE: + data := a.Data() + if data == 0 { + return "trace" + } + return fmt.Sprintf("trace (data=%#x)", data) + case SECCOMP_RET_ALLOW: + return "allow" + case SECCOMP_RET_USER_NOTIF: + return "unotify" + } + return fmt.Sprintf("invalid action: %#x", uint32(a)) +} + +// Data returns the SECCOMP_RET_DATA portion of the action. +func (a BPFAction) Data() uint16 { + return uint16(a & SECCOMP_RET_DATA) +} + +// WithReturnCode sets the lower 16 bits of the SECCOMP_RET_ERRNO or +// SECCOMP_RET_TRACE actions to the provided return code, overwriting the previous +// action, and returns a new BPFAction. If not SECCOMP_RET_ERRNO or +// SECCOMP_RET_TRACE then this panics. +func (a BPFAction) WithReturnCode(code uint16) BPFAction { + // mask out the previous return value + baseAction := a & SECCOMP_RET_ACTION_FULL + if baseAction == SECCOMP_RET_ERRNO || baseAction == SECCOMP_RET_TRACE { + return BPFAction(uint32(baseAction) | uint32(code)) + } + panic("WithReturnCode only valid for SECCOMP_RET_ERRNO and SECCOMP_RET_TRACE") +} + +// SockFprog is sock_fprog taken from . +type SockFprog struct { + Len uint16 + pad [6]byte + Filter *BPFInstruction +} + +// SeccompData is equivalent to struct seccomp_data, which contains the data +// passed to seccomp-bpf filters. +// +// +marshal +type SeccompData struct { + // Nr is the system call number. + Nr int32 + + // Arch is an AUDIT_ARCH_* value indicating the system call convention. + Arch uint32 + + // InstructionPointer is the value of the instruction pointer at the time + // of the system call. + InstructionPointer uint64 + + // Args contains the first 6 system call arguments. + Args [6]uint64 +} + +// SeccompNotifResp is equivalent to struct seccomp_notif_resp. +// +// +marshal +type SeccompNotifResp struct { + ID uint64 + Val int64 + Error int32 + Flags uint32 +} + +// SeccompNotifSizes is equivalent to struct seccomp_notif_sizes. +// +// +marshal +type SeccompNotifSizes struct { + Notif uint16 + Notif_resp uint16 + Data uint16 +} + +// SeccompNotif is equivalent to struct seccomp_notif. +// +// +marshal +type SeccompNotif struct { + ID uint64 + Pid int32 + Flags uint32 + Data SeccompData +} + +// String returns a human-friendly representation of this `SeccompData`. +func (sd SeccompData) String() string { + return fmt.Sprintf( + "sysno=%d arch=%#x rip=%#x args=[%#x %#x %#x %#x %#x %#x]", + sd.Nr, + sd.Arch, + sd.InstructionPointer, + sd.Args[0], + sd.Args[1], + sd.Args[2], + sd.Args[3], + sd.Args[4], + sd.Args[5], + ) +} diff --git a/pkg/abi/linux/sem.go b/pkg/abi/linux/sem.go new file mode 100644 index 0000000..a9c708d --- /dev/null +++ b/pkg/abi/linux/sem.go @@ -0,0 +1,82 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// semctl Command Definitions. Source: include/uapi/linux/sem.h +const ( + GETPID = 11 + GETVAL = 12 + GETALL = 13 + GETNCNT = 14 + GETZCNT = 15 + SETVAL = 16 + SETALL = 17 +) + +// ipcs ctl cmds. Source: include/uapi/linux/sem.h +const ( + SEM_STAT = 18 + SEM_INFO = 19 + SEM_STAT_ANY = 20 +) + +// Information about system-wide semaphore limits and parameters. +// +// Source: include/uapi/linux/sem.h +const ( + SEMMNI = 32000 + SEMMSL = 32000 + SEMMNS = SEMMNI * SEMMSL + SEMOPM = 500 + SEMVMX = 32767 + SEMAEM = SEMVMX + + SEMUME = SEMOPM + SEMMNU = SEMMNS + SEMMAP = SEMMNS + SEMUSZ = 20 +) + +// Semaphore flags. +const ( + SEM_UNDO = 0x1000 +) + +// Sembuf is equivalent to struct sembuf. +// +// +marshal slice:SembufSlice +type Sembuf struct { + SemNum uint16 + SemOp int16 + SemFlg int16 +} + +// SemInfo is equivalent to struct seminfo. +// +// Source: include/uapi/linux/sem.h +// +// +marshal +type SemInfo struct { + SemMap uint32 + SemMni uint32 + SemMns uint32 + SemMnu uint32 + SemMsl uint32 + SemOpm uint32 + SemUme uint32 + SemUsz uint32 + SemVmx uint32 + SemAem uint32 +} diff --git a/pkg/abi/linux/sem_amd64.go b/pkg/abi/linux/sem_amd64.go new file mode 100644 index 0000000..cabd2d4 --- /dev/null +++ b/pkg/abi/linux/sem_amd64.go @@ -0,0 +1,34 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package linux + +// SemidDS is equivalent to struct semid64_ds. +// +// Source: arch/x86/include/uapi/asm/sembuf.h +// +// +marshal +type SemidDS struct { + SemPerm IPCPerm + SemOTime TimeT + unused1 uint64 + SemCTime TimeT + unused2 uint64 + SemNSems uint64 + unused3 uint64 + unused4 uint64 +} diff --git a/pkg/abi/linux/sem_arm64.go b/pkg/abi/linux/sem_arm64.go new file mode 100644 index 0000000..a0c467d --- /dev/null +++ b/pkg/abi/linux/sem_arm64.go @@ -0,0 +1,32 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package linux + +// SemidDS is equivalent to struct semid64_ds. +// +// Source: include/uapi/asm-generic/sembuf.h +// +// +marshal +type SemidDS struct { + SemPerm IPCPerm + SemOTime TimeT + SemCTime TimeT + SemNSems uint64 + unused3 uint64 + unused4 uint64 +} diff --git a/pkg/abi/linux/shm.go b/pkg/abi/linux/shm.go new file mode 100644 index 0000000..77945bd --- /dev/null +++ b/pkg/abi/linux/shm.go @@ -0,0 +1,92 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import "math" + +// shmat(2) flags. Source: include/uapi/linux/shm.h +const ( + SHM_RDONLY = 0o10000 // Read-only access. + SHM_RND = 0o20000 // Round attach address to SHMLBA boundary. + SHM_REMAP = 0o40000 // Take-over region on attach. + SHM_EXEC = 0o100000 // Execution access. +) + +// IPCPerm.Mode upper byte flags. Source: include/linux/shm.h +const ( + SHM_DEST = 0o1000 // Segment will be destroyed on last detach. + SHM_LOCKED = 0o2000 // Segment will not be swapped. + SHM_HUGETLB = 0o4000 // Segment will use huge TLB pages. + SHM_NORESERVE = 0o10000 // Don't check for reservations. +) + +// Additional Linux-only flags for shmctl(2). Source: include/uapi/linux/shm.h +const ( + SHM_LOCK = 11 + SHM_UNLOCK = 12 + SHM_STAT = 13 + SHM_INFO = 14 +) + +// SHM defaults as specified by linux. Source: include/uapi/linux/shm.h +const ( + SHMMIN = 1 + SHMMNI = 4096 + SHMMAX = math.MaxUint64 - 1<<24 + SHMALL = math.MaxUint64 - 1<<24 + SHMSEG = 4096 +) + +// ShmidDS is equivalent to struct shmid64_ds. Source: +// include/uapi/asm-generic/shmbuf.h +// +// +marshal +type ShmidDS struct { + ShmPerm IPCPerm + ShmSegsz uint64 + ShmAtime TimeT + ShmDtime TimeT + ShmCtime TimeT + ShmCpid int32 + ShmLpid int32 + ShmNattach uint64 + + Unused4 uint64 + Unused5 uint64 +} + +// ShmParams is equivalent to struct shminfo. Source: include/uapi/linux/shm.h +// +// +marshal +type ShmParams struct { + ShmMax uint64 + ShmMin uint64 + ShmMni uint64 + ShmSeg uint64 + ShmAll uint64 +} + +// ShmInfo is equivalent to struct shm_info. Source: include/uapi/linux/shm.h +// +// +marshal +type ShmInfo struct { + UsedIDs int32 // Number of currently existing segments. + _ [4]byte + ShmTot uint64 // Total number of shared memory pages. + ShmRss uint64 // Number of resident shared memory pages. + ShmSwp uint64 // Number of swapped shared memory pages. + SwapAttempts uint64 // Unused since Linux 2.4. + SwapSuccesses uint64 // Unused since Linux 2.4. +} diff --git a/pkg/abi/linux/signal.go b/pkg/abi/linux/signal.go new file mode 100644 index 0000000..1c2795c --- /dev/null +++ b/pkg/abi/linux/signal.go @@ -0,0 +1,552 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "github.com/sagernet/gvisor/pkg/bits" + "github.com/sagernet/gvisor/pkg/hostarch" +) + +const ( + // SignalMaximum is the highest valid signal number. + SignalMaximum = 64 + + // FirstStdSignal is the lowest standard signal number. + FirstStdSignal = 1 + + // LastStdSignal is the highest standard signal number. + LastStdSignal = 31 + + // FirstRTSignal is the lowest real-time signal number. + // + // 32 (SIGCANCEL) and 33 (SIGSETXID) are used internally by glibc. + FirstRTSignal = 32 + + // LastRTSignal is the highest real-time signal number. + LastRTSignal = 64 + + // NumStdSignals is the number of standard signals. + NumStdSignals = LastStdSignal - FirstStdSignal + 1 + + // NumRTSignals is the number of realtime signals. + NumRTSignals = LastRTSignal - FirstRTSignal + 1 +) + +// Signal is a signal number. +type Signal int + +// IsValid returns true if s is a valid standard or realtime signal. (0 is not +// considered valid; interfaces special-casing signal number 0 should check for +// 0 first before asserting validity.) +func (s Signal) IsValid() bool { + return s > 0 && s <= SignalMaximum +} + +// IsStandard returns true if s is a standard signal. +// +// Preconditions: s.IsValid(). +func (s Signal) IsStandard() bool { + return s <= LastStdSignal +} + +// IsRealtime returns true if s is a realtime signal. +// +// Preconditions: s.IsValid(). +func (s Signal) IsRealtime() bool { + return s >= FirstRTSignal +} + +// Index returns the index for signal s into arrays of both standard and +// realtime signals (e.g. signal masks). +// +// Preconditions: s.IsValid(). +func (s Signal) Index() int { + return int(s - 1) +} + +// Signals. +const ( + SIGABRT = Signal(6) + SIGALRM = Signal(14) + SIGBUS = Signal(7) + SIGCHLD = Signal(17) + SIGCLD = Signal(17) + SIGCONT = Signal(18) + SIGFPE = Signal(8) + SIGHUP = Signal(1) + SIGILL = Signal(4) + SIGINT = Signal(2) + SIGIO = Signal(29) + SIGIOT = Signal(6) + SIGKILL = Signal(9) + SIGPIPE = Signal(13) + SIGPOLL = Signal(29) + SIGPROF = Signal(27) + SIGPWR = Signal(30) + SIGQUIT = Signal(3) + SIGSEGV = Signal(11) + SIGSTKFLT = Signal(16) + SIGSTOP = Signal(19) + SIGSYS = Signal(31) + SIGTERM = Signal(15) + SIGTRAP = Signal(5) + SIGTSTP = Signal(20) + SIGTTIN = Signal(21) + SIGTTOU = Signal(22) + SIGUNUSED = Signal(31) + SIGURG = Signal(23) + SIGUSR1 = Signal(10) + SIGUSR2 = Signal(12) + SIGVTALRM = Signal(26) + SIGWINCH = Signal(28) + SIGXCPU = Signal(24) + SIGXFSZ = Signal(25) +) + +// SignalSet is a signal mask with a bit corresponding to each signal. +// +// +marshal +type SignalSet uint64 + +// SignalSetSize is the size in bytes of a SignalSet. +const SignalSetSize = 8 + +// MakeSignalSet returns SignalSet with the bit corresponding to each of the +// given signals set. +func MakeSignalSet(sigs ...Signal) SignalSet { + indices := make([]int, len(sigs)) + for i, sig := range sigs { + indices[i] = sig.Index() + } + return SignalSet(bits.Mask64(indices...)) +} + +// SignalSetOf returns a SignalSet with a single signal set. +func SignalSetOf(sig Signal) SignalSet { + return SignalSet(bits.MaskOf64(sig.Index())) +} + +// ForEachSignal invokes f for each signal set in the given mask. +func ForEachSignal(mask SignalSet, f func(sig Signal)) { + bits.ForEachSetBit64(uint64(mask), func(i int) { + f(Signal(i + 1)) + }) +} + +// 'how' values for rt_sigprocmask(2). +const ( + // SIG_BLOCK blocks the signals in the set. + SIG_BLOCK = 0 + + // SIG_UNBLOCK blocks the signals in the set. + SIG_UNBLOCK = 1 + + // SIG_SETMASK sets the signal mask to set. + SIG_SETMASK = 2 +) + +// Signal actions for rt_sigaction(2), from uapi/asm-generic/signal-defs.h. +const ( + // SIG_DFL performs the default action. + SIG_DFL = 0 + + // SIG_IGN ignores the signal. + SIG_IGN = 1 +) + +// Signal action flags for rt_sigaction(2), from uapi/asm-generic/signal.h. +const ( + SA_NOCLDSTOP = 0x00000001 + SA_NOCLDWAIT = 0x00000002 + SA_SIGINFO = 0x00000004 + SA_RESTORER = 0x04000000 + SA_ONSTACK = 0x08000000 + SA_RESTART = 0x10000000 + SA_NODEFER = 0x40000000 + SA_RESETHAND = 0x80000000 + SA_NOMASK = SA_NODEFER + SA_ONESHOT = SA_RESETHAND +) + +// Signal stack flags for signalstack(2), from include/uapi/linux/signal.h. +const ( + SS_ONSTACK = 1 + SS_DISABLE = 2 +) + +// SIGPOLL si_codes. +const ( + // SI_POLL is defined as __SI_POLL in Linux 2.6. + SI_POLL = 2 << 16 + + // POLL_IN indicates that data input available. + POLL_IN = SI_POLL | 1 + + // POLL_OUT indicates that output buffers available. + POLL_OUT = SI_POLL | 2 + + // POLL_MSG indicates that an input message available. + POLL_MSG = SI_POLL | 3 + + // POLL_ERR indicates that there was an i/o error. + POLL_ERR = SI_POLL | 4 + + // POLL_PRI indicates that a high priority input available. + POLL_PRI = SI_POLL | 5 + + // POLL_HUP indicates that a device disconnected. + POLL_HUP = SI_POLL | 6 +) + +// Possible values for si_code. +const ( + // SI_USER is sent by kill, sigsend, raise. + SI_USER = 0 + + // SI_KERNEL is sent by the kernel from somewhere. + SI_KERNEL = 0x80 + + // SI_QUEUE is sent by sigqueue. + SI_QUEUE = -1 + + // SI_TIMER is sent by timer expiration. + SI_TIMER = -2 + + // SI_MESGQ is sent by real time mesq state change. + SI_MESGQ = -3 + + // SI_ASYNCIO is sent by AIO completion. + SI_ASYNCIO = -4 + + // SI_SIGIO is sent by queued SIGIO. + SI_SIGIO = -5 + + // SI_TKILL is sent by tkill system call. + SI_TKILL = -6 + + // SI_DETHREAD is sent by execve() killing subsidiary threads. + SI_DETHREAD = -7 + + // SI_ASYNCNL is sent by glibc async name lookup completion. + SI_ASYNCNL = -60 +) + +// CLD_* codes are only meaningful for SIGCHLD. +const ( + // CLD_EXITED indicates that a task exited. + CLD_EXITED = 1 + + // CLD_KILLED indicates that a task was killed by a signal. + CLD_KILLED = 2 + + // CLD_DUMPED indicates that a task was killed by a signal and then dumped + // core. + CLD_DUMPED = 3 + + // CLD_TRAPPED indicates that a task was stopped by ptrace. + CLD_TRAPPED = 4 + + // CLD_STOPPED indicates that a thread group completed a group stop. + CLD_STOPPED = 5 + + // CLD_CONTINUED indicates that a group-stopped thread group was continued. + CLD_CONTINUED = 6 +) + +// SYS_* codes are only meaningful for SIGSYS. +const ( + // SYS_SECCOMP indicates that a signal originates from seccomp. + SYS_SECCOMP = 1 +) + +// Possible values for Sigevent.Notify, aka struct sigevent::sigev_notify. +const ( + SIGEV_SIGNAL = 0 + SIGEV_NONE = 1 + SIGEV_THREAD = 2 + SIGEV_THREAD_ID = 4 +) + +// SIGTRAP si_codes +const ( + TRAP_BRKPT = 1 + TRAP_TRACE = 2 + TRAP_BRANCH = 3 + TRAP_HWBKPT = 4 +) + +// Sigevent represents struct sigevent. +// +// +marshal +type Sigevent struct { + Value uint64 // union sigval {int, void*} + Signo int32 + Notify int32 + + // struct sigevent here contains 48-byte union _sigev_un. However, only + // member _tid is significant to the kernel. + Tid int32 + UnRemainder [44]byte +} + +// SigAction represents struct sigaction. +// +// +marshal +// +stateify savable +type SigAction struct { + Handler uint64 + Flags uint64 + Restorer uint64 + Mask SignalSet +} + +// SignalStack represents information about a user stack, and is equivalent to +// stack_t. +// +// +marshal +// +stateify savable +type SignalStack struct { + Addr uint64 + Flags uint32 + _ uint32 + Size uint64 +} + +// Contains checks if the stack pointer is within this stack. +func (s *SignalStack) Contains(sp hostarch.Addr) bool { + return hostarch.Addr(s.Addr) < sp && sp <= hostarch.Addr(s.Addr+s.Size) +} + +// Top returns the stack's top address. +func (s *SignalStack) Top() hostarch.Addr { + return hostarch.Addr(s.Addr + s.Size) +} + +// IsEnabled returns true iff this signal stack is marked as enabled. +func (s *SignalStack) IsEnabled() bool { + return s.Flags&SS_DISABLE == 0 +} + +// SignalInfo represents information about a signal being delivered, and is +// equivalent to struct siginfo in linux kernel(linux/include/uapi/asm-generic/siginfo.h). +// +// +marshal +// +stateify savable +type SignalInfo struct { + Signo int32 // Signal number + Errno int32 // Errno value + Code int32 // Signal code + _ uint32 + + // struct siginfo::_sifields is a union. In SignalInfo, fields in the union + // are accessed through methods. + // + // For reference, here is the definition of _sifields: (_sigfault._trapno, + // which does not exist on x86, omitted for clarity) + // + // union { + // int _pad[SI_PAD_SIZE]; + // + // /* kill() */ + // struct { + // __kernel_pid_t _pid; /* sender's pid */ + // __ARCH_SI_UID_T _uid; /* sender's uid */ + // } _kill; + // + // /* POSIX.1b timers */ + // struct { + // __kernel_timer_t _tid; /* timer id */ + // int _overrun; /* overrun count */ + // char _pad[sizeof( __ARCH_SI_UID_T) - sizeof(int)]; + // sigval_t _sigval; /* same as below */ + // int _sys_private; /* not to be passed to user */ + // } _timer; + // + // /* POSIX.1b signals */ + // struct { + // __kernel_pid_t _pid; /* sender's pid */ + // __ARCH_SI_UID_T _uid; /* sender's uid */ + // sigval_t _sigval; + // } _rt; + // + // /* SIGCHLD */ + // struct { + // __kernel_pid_t _pid; /* which child */ + // __ARCH_SI_UID_T _uid; /* sender's uid */ + // int _status; /* exit code */ + // __ARCH_SI_CLOCK_T _utime; + // __ARCH_SI_CLOCK_T _stime; + // } _sigchld; + // + // /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ + // struct { + // void *_addr; /* faulting insn/memory ref. */ + // short _addr_lsb; /* LSB of the reported address */ + // } _sigfault; + // + // /* SIGPOLL */ + // struct { + // __ARCH_SI_BAND_T _band; /* POLL_IN, POLL_OUT, POLL_MSG */ + // int _fd; + // } _sigpoll; + // + // /* SIGSYS */ + // struct { + // void *_call_addr; /* calling user insn */ + // int _syscall; /* triggering system call number */ + // unsigned int _arch; /* AUDIT_ARCH_* of syscall */ + // } _sigsys; + // } _sifields; + // + // _sifields is padded so that the size of siginfo is SI_MAX_SIZE = 128 + // bytes. + Fields [128 - 16]byte +} + +// FixSignalCodeForUser fixes up si_code. +// +// The si_code we get from Linux may contain the kernel-specific code in the +// top 16 bits if it's positive (e.g., from ptrace). Linux's +// copy_siginfo_to_user does: +// err |= __put_user((short)from->si_code, &to->si_code); +// to mask out those bits and we need to do the same. +func (s *SignalInfo) FixSignalCodeForUser() { + if s.Code > 0 { + s.Code &= 0x0000ffff + } +} + +// PID returns the si_pid field. +func (s *SignalInfo) PID() int32 { + return int32(hostarch.ByteOrder.Uint32(s.Fields[0:4])) +} + +// SetPID mutates the si_pid field. +func (s *SignalInfo) SetPID(val int32) { + hostarch.ByteOrder.PutUint32(s.Fields[0:4], uint32(val)) +} + +// UID returns the si_uid field. +func (s *SignalInfo) UID() int32 { + return int32(hostarch.ByteOrder.Uint32(s.Fields[4:8])) +} + +// SetUID mutates the si_uid field. +func (s *SignalInfo) SetUID(val int32) { + hostarch.ByteOrder.PutUint32(s.Fields[4:8], uint32(val)) +} + +// Sigval returns the sigval field, which is aliased to both si_int and si_ptr. +func (s *SignalInfo) Sigval() uint64 { + return hostarch.ByteOrder.Uint64(s.Fields[8:16]) +} + +// SetSigval mutates the sigval field. +func (s *SignalInfo) SetSigval(val uint64) { + hostarch.ByteOrder.PutUint64(s.Fields[8:16], val) +} + +// TimerID returns the si_timerid field. +func (s *SignalInfo) TimerID() TimerID { + return TimerID(hostarch.ByteOrder.Uint32(s.Fields[0:4])) +} + +// SetTimerID sets the si_timerid field. +func (s *SignalInfo) SetTimerID(val TimerID) { + hostarch.ByteOrder.PutUint32(s.Fields[0:4], uint32(val)) +} + +// Overrun returns the si_overrun field. +func (s *SignalInfo) Overrun() int32 { + return int32(hostarch.ByteOrder.Uint32(s.Fields[4:8])) +} + +// SetOverrun sets the si_overrun field. +func (s *SignalInfo) SetOverrun(val int32) { + hostarch.ByteOrder.PutUint32(s.Fields[4:8], uint32(val)) +} + +// Addr returns the si_addr field. +func (s *SignalInfo) Addr() uint64 { + return hostarch.ByteOrder.Uint64(s.Fields[0:8]) +} + +// SetAddr sets the si_addr field. +func (s *SignalInfo) SetAddr(val uint64) { + hostarch.ByteOrder.PutUint64(s.Fields[0:8], val) +} + +// Status returns the si_status field. +func (s *SignalInfo) Status() int32 { + return int32(hostarch.ByteOrder.Uint32(s.Fields[8:12])) +} + +// SetStatus mutates the si_status field. +func (s *SignalInfo) SetStatus(val int32) { + hostarch.ByteOrder.PutUint32(s.Fields[8:12], uint32(val)) +} + +// CallAddr returns the si_call_addr field. +func (s *SignalInfo) CallAddr() uint64 { + return hostarch.ByteOrder.Uint64(s.Fields[0:8]) +} + +// SetCallAddr mutates the si_call_addr field. +func (s *SignalInfo) SetCallAddr(val uint64) { + hostarch.ByteOrder.PutUint64(s.Fields[0:8], val) +} + +// Syscall returns the si_syscall field. +func (s *SignalInfo) Syscall() int32 { + return int32(hostarch.ByteOrder.Uint32(s.Fields[8:12])) +} + +// SetSyscall mutates the si_syscall field. +func (s *SignalInfo) SetSyscall(val int32) { + hostarch.ByteOrder.PutUint32(s.Fields[8:12], uint32(val)) +} + +// Arch returns the si_arch field. +func (s *SignalInfo) Arch() uint32 { + return hostarch.ByteOrder.Uint32(s.Fields[12:16]) +} + +// SetArch mutates the si_arch field. +func (s *SignalInfo) SetArch(val uint32) { + hostarch.ByteOrder.PutUint32(s.Fields[12:16], val) +} + +// Band returns the si_band field. +func (s *SignalInfo) Band() int64 { + return int64(hostarch.ByteOrder.Uint64(s.Fields[0:8])) +} + +// SetBand mutates the si_band field. +func (s *SignalInfo) SetBand(val int64) { + // Note: this assumes the platform uses `long` as `__ARCH_SI_BAND_T`. + // On some platforms, which gVisor doesn't support, `__ARCH_SI_BAND_T` is + // `int`. See siginfo.h. + hostarch.ByteOrder.PutUint64(s.Fields[0:8], uint64(val)) +} + +// FD returns the si_fd field. +func (s *SignalInfo) FD() uint32 { + return hostarch.ByteOrder.Uint32(s.Fields[8:12]) +} + +// SetFD mutates the si_fd field. +func (s *SignalInfo) SetFD(val uint32) { + hostarch.ByteOrder.PutUint32(s.Fields[8:12], val) +} diff --git a/pkg/abi/linux/signalfd.go b/pkg/abi/linux/signalfd.go new file mode 100644 index 0000000..8608972 --- /dev/null +++ b/pkg/abi/linux/signalfd.go @@ -0,0 +1,47 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +const ( + // SFD_NONBLOCK is a signalfd(2) flag. + SFD_NONBLOCK = 0o0004000 + + // SFD_CLOEXEC is a signalfd(2) flag. + SFD_CLOEXEC = 0o2000000 +) + +// SignalfdSiginfo is the siginfo encoding for signalfds. +// +// +marshal +type SignalfdSiginfo struct { + Signo uint32 + Errno int32 + Code int32 + PID uint32 + UID uint32 + FD int32 + TID uint32 + Band uint32 + Overrun uint32 + TrapNo uint32 + Status int32 + Int int32 + Ptr uint64 + UTime uint64 + STime uint64 + Addr uint64 + AddrLSB uint16 + _ [48]uint8 `marshal:"unaligned"` +} diff --git a/pkg/abi/linux/socket.go b/pkg/abi/linux/socket.go new file mode 100644 index 0000000..a131995 --- /dev/null +++ b/pkg/abi/linux/socket.go @@ -0,0 +1,740 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Address families, from linux/socket.h. +const ( + AF_UNSPEC = 0 + AF_UNIX = 1 + AF_INET = 2 + AF_AX25 = 3 + AF_IPX = 4 + AF_APPLETALK = 5 + AF_NETROM = 6 + AF_BRIDGE = 7 + AF_ATMPVC = 8 + AF_X25 = 9 + AF_INET6 = 10 + AF_ROSE = 11 + AF_DECnet = 12 + AF_NETBEUI = 13 + AF_SECURITY = 14 + AF_KEY = 15 + AF_NETLINK = 16 + AF_PACKET = 17 + AF_ASH = 18 + AF_ECONET = 19 + AF_ATMSVC = 20 + AF_RDS = 21 + AF_SNA = 22 + AF_IRDA = 23 + AF_PPPOX = 24 + AF_WANPIPE = 25 + AF_LLC = 26 + AF_IB = 27 + AF_MPLS = 28 + AF_CAN = 29 + AF_TIPC = 30 + AF_BLUETOOTH = 31 + AF_IUCV = 32 + AF_RXRPC = 33 + AF_ISDN = 34 + AF_PHONET = 35 + AF_IEEE802154 = 36 + AF_CAIF = 37 + AF_ALG = 38 + AF_NFC = 39 + AF_VSOCK = 40 +) + +// sendmsg(2)/recvmsg(2) flags, from linux/socket.h. +const ( + MSG_OOB = 0x1 + MSG_PEEK = 0x2 + MSG_DONTROUTE = 0x4 + MSG_TRYHARD = 0x4 + MSG_CTRUNC = 0x8 + MSG_PROBE = 0x10 + MSG_TRUNC = 0x20 + MSG_DONTWAIT = 0x40 + MSG_EOR = 0x80 + MSG_WAITALL = 0x100 + MSG_FIN = 0x200 + MSG_EOF = MSG_FIN + MSG_SYN = 0x400 + MSG_CONFIRM = 0x800 + MSG_RST = 0x1000 + MSG_ERRQUEUE = 0x2000 + MSG_NOSIGNAL = 0x4000 + MSG_MORE = 0x8000 + MSG_WAITFORONE = 0x10000 + MSG_SENDPAGE_NOTLAST = 0x20000 + MSG_ZEROCOPY = 0x4000000 + MSG_FASTOPEN = 0x20000000 + MSG_CMSG_CLOEXEC = 0x40000000 +) + +// Set/get socket option levels, from socket.h. +const ( + SOL_IP = 0 + SOL_SOCKET = 1 + SOL_TCP = 6 + SOL_UDP = 17 + SOL_IPV6 = 41 + SOL_ICMPV6 = 58 + SOL_RAW = 255 + SOL_PACKET = 263 + SOL_NETLINK = 270 +) + +// A SockType is a type (as opposed to family) of sockets. These are enumerated +// below as SOCK_* constants. +type SockType int + +// Socket types, from linux/net.h. +const ( + SOCK_STREAM SockType = 1 + SOCK_DGRAM SockType = 2 + SOCK_RAW SockType = 3 + SOCK_RDM SockType = 4 + SOCK_SEQPACKET SockType = 5 + SOCK_DCCP SockType = 6 + SOCK_PACKET SockType = 10 +) + +// SOCK_TYPE_MASK covers all of the above socket types. The remaining bits are +// flags. From linux/net.h. +const SOCK_TYPE_MASK = 0xf + +// socket(2)/socketpair(2)/accept4(2) flags, from linux/net.h. +const ( + SOCK_CLOEXEC = O_CLOEXEC + SOCK_NONBLOCK = O_NONBLOCK +) + +// shutdown(2) how commands, from . +const ( + SHUT_RD = 0 + SHUT_WR = 1 + SHUT_RDWR = 2 +) + +// Packet types from +const ( + PACKET_HOST = 0 // To us + PACKET_BROADCAST = 1 // To all + PACKET_MULTICAST = 2 // To group + PACKET_OTHERHOST = 3 // To someone else + PACKET_OUTGOING = 4 // Outgoing of any type +) + +// Packet socket options from +const ( + PACKET_ADD_MEMBERSHIP = 1 + PACKET_RX_RING = 5 + PACKET_STATISTICS = 6 + PACKET_AUXDATA = 8 + PACKET_VERSION = 10 + PACKET_HDRLEN = 11 + PACKET_RESERVE = 12 +) + +// Statuses for a frame in a packet_mmap ring buffer from . +const ( + TP_STATUS_KERNEL = 0 + TP_STATUS_USER = 0x1 + TP_STATUS_COPY = 0x2 + TP_STATUS_LOSING = 0x4 + TP_STATUS_CSUM_NOT_READY = 0x8 + TP_STATUS_VLAN_VALID = 0x10 + TP_STATUS_BLK_TMO = 0x20 + TP_STATUS_VLAN_TPID_VALID = 0x40 + TP_STATUS_CSUM_VALID = 0x80 + TP_STATUS_GSO_TCP = 0x100 +) + +// TpacketReq is the request for a packet_mmap ring buffer from +// . +// +// +marshal +type TpacketReq struct { + TpBlockSize uint32 + TpBlockNr uint32 + TpFrameSize uint32 + TpFrameNr uint32 +} + +// TpacketHdr is the header for a frame in a packet_mmap ring buffer from +// . +// +// +marshal +type TpacketHdr struct { + TpStatus uint64 + TpLen uint32 + TpSnaplen uint32 + TpMac uint16 + TpNet uint16 + TpSec uint32 + TpUsec uint32 + _ [4]uint8 +} + +// Tpacket2Hdr is the header for a frame in a packet_mmap ring buffer from +// . +// +// +marshal +type Tpacket2Hdr struct { + TpStatus uint32 + TpLen uint32 + TpSnaplen uint32 + TpMac uint16 + TpNet uint16 + TpSec uint32 + TpNSec uint32 + TpVlanTci uint16 + TpVlanTpid uint16 + _ [4]uint8 +} + +// TpacketStats is the statistics for a packet_mmap ring buffer from +// . +// +// +marshal +type TpacketStats struct { + Packets uint32 + Dropped uint32 +} + +// TpacketAlignment is the alignment of a frame in a packet_mmap ring buffer +// from . +const ( + TPACKET_ALIGNMENT = 16 +) + +// TPACKET_V1 is the version of a packet_mmap ring buffer from +// that is implemented in gVisor. +const ( + // TPACKET_V1 is the default version of PACKET_MMAP. + TPACKET_V1 = iota + // TPACKET_V2 is the version of PACKET_MMAP for tpacket2_hdr. + TPACKET_V2 +) + +var ( + // TPACKET_HDRLEN is the length of a TpacketHdr from . + TPACKET_HDRLEN = TPacketAlign(uint32((*TpacketHdr)(nil).SizeBytes()) + uint32((*SockAddrLink)(nil).SizeBytes())) + // TPACKET2_HDRLEN is the length of a Tpacket2Hdr from . + TPACKET2_HDRLEN = TPacketAlign(uint32((*Tpacket2Hdr)(nil).SizeBytes()) + uint32((*SockAddrLink)(nil).SizeBytes())) +) + +// TPacketAlign aligns a value to the alignment of a TPacket. +func TPacketAlign(x uint32) uint32 { + return (x + TPACKET_ALIGNMENT - 1) &^ (TPACKET_ALIGNMENT - 1) +} + +// Socket options from socket.h. +const ( + SO_DEBUG = 1 + SO_REUSEADDR = 2 + SO_TYPE = 3 + SO_ERROR = 4 + SO_DONTROUTE = 5 + SO_BROADCAST = 6 + SO_SNDBUF = 7 + SO_RCVBUF = 8 + SO_KEEPALIVE = 9 + SO_OOBINLINE = 10 + SO_NO_CHECK = 11 + SO_PRIORITY = 12 + SO_LINGER = 13 + SO_BSDCOMPAT = 14 + SO_REUSEPORT = 15 + SO_PASSCRED = 16 + SO_PEERCRED = 17 + SO_RCVLOWAT = 18 + SO_SNDLOWAT = 19 + SO_RCVTIMEO = 20 + SO_SNDTIMEO = 21 + SO_BINDTODEVICE = 25 + SO_ATTACH_FILTER = 26 + SO_DETACH_FILTER = 27 + SO_GET_FILTER = SO_ATTACH_FILTER + SO_PEERNAME = 28 + SO_TIMESTAMP = 29 + SO_ACCEPTCONN = 30 + SO_PEERSEC = 31 + SO_SNDBUFFORCE = 32 + SO_RCVBUFFORCE = 33 + SO_PASSSEC = 34 + SO_TIMESTAMPNS = 35 + SO_MARK = 36 + SO_TIMESTAMPING = 37 + SO_PROTOCOL = 38 + SO_DOMAIN = 39 + SO_RXQ_OVFL = 40 + SO_WIFI_STATUS = 41 + SO_PEEK_OFF = 42 + SO_NOFCS = 43 + SO_LOCK_FILTER = 44 + SO_SELECT_ERR_QUEUE = 45 + SO_BUSY_POLL = 46 + SO_MAX_PACING_RATE = 47 + SO_BPF_EXTENSIONS = 48 + SO_INCOMING_CPU = 49 + SO_ATTACH_BPF = 50 + SO_ATTACH_REUSEPORT_CBPF = 51 + SO_ATTACH_REUSEPORT_EBPF = 52 + SO_CNX_ADVICE = 53 + SO_MEMINFO = 55 + SO_INCOMING_NAPI_ID = 56 + SO_COOKIE = 57 + SO_PEERGROUPS = 59 + SO_ZEROCOPY = 60 + SO_TXTIME = 61 + SO_BINDTOIFINDEX = 62 + SO_TIMESTAMP_OLD = 29 + SO_TIMESTAMPNS_OLD = 35 + SO_TIMESTAMPING_OLD = 37 + SO_TIMESTAMP_NEW = 63 + SO_TIMESTAMPNS_NEW = 64 + SO_TIMESTAMPING_NEW = 65 + SO_RCVTIMEO_NEW = 66 + SO_SNDTIMEO_NEW = 67 + SO_DETACH_REUSEPORT_BPF = 68 + SO_PREFER_BUSY_POLL = 69 + SO_BUSY_POLL_BUDGET = 70 + SO_NETNS_COOKIE = 71 + SO_BUF_LOCK = 72 + SO_RESERVE_MEM = 73 + SO_TXREHASH = 74 + SO_RCVMARK = 75 + SO_PASSPIDFD = 76 + SO_PEERPIDFD = 77 + SO_DEVMEM_LINEAR = 78 + SO_DEVMEM_DMABUF = 79 + SO_DEVMEM_DONTNEED = 80 + SO_RCVPRIORITY = 82 +) + +// enum socket_state, from uapi/linux/net.h. +const ( + SS_FREE = 0 // Not allocated. + SS_UNCONNECTED = 1 // Unconnected to any socket. + SS_CONNECTING = 2 // In process of connecting. + SS_CONNECTED = 3 // Connected to socket. + SS_DISCONNECTING = 4 // In process of disconnecting. +) + +// TCP protocol states, from include/net/tcp_states.h. +const ( + TCP_ESTABLISHED uint32 = iota + 1 + TCP_SYN_SENT + TCP_SYN_RECV + TCP_FIN_WAIT1 + TCP_FIN_WAIT2 + TCP_TIME_WAIT + TCP_CLOSE + TCP_CLOSE_WAIT + TCP_LAST_ACK + TCP_LISTEN + TCP_CLOSING + TCP_NEW_SYN_RECV +) + +// SockAddrMax is the maximum size of a struct sockaddr, from +// uapi/linux/socket.h. +const SockAddrMax = 128 + +// InetAddr is struct in_addr, from uapi/linux/in.h. +// +// +marshal +type InetAddr [4]byte + +// SizeOfInetAddr is the size of InetAddr. +var SizeOfInetAddr = uint32((*InetAddr)(nil).SizeBytes()) + +// SockAddrInet is struct sockaddr_in, from uapi/linux/in.h. +// +// +marshal +type SockAddrInet struct { + Family uint16 + Port uint16 + Addr InetAddr + _ [8]uint8 // pad to sizeof(struct sockaddr). +} + +// Inet6MulticastRequest is struct ipv6_mreq, from uapi/linux/in6.h. +// +// +marshal +type Inet6MulticastRequest struct { + MulticastAddr Inet6Addr + InterfaceIndex int32 +} + +// InetMulticastRequest is struct ip_mreq, from uapi/linux/in.h. +// +// +marshal +type InetMulticastRequest struct { + MulticastAddr InetAddr + InterfaceAddr InetAddr +} + +// InetMulticastRequestWithNIC is struct ip_mreqn, from uapi/linux/in.h. +// +// +marshal +type InetMulticastRequestWithNIC struct { + InetMulticastRequest + InterfaceIndex int32 +} + +// Inet6Addr is struct in6_addr, from uapi/linux/in6.h. +// +// +marshal +type Inet6Addr [16]byte + +// SockAddrInet6 is struct sockaddr_in6, from uapi/linux/in6.h. +// +// +marshal +type SockAddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte + Scope_id uint32 +} + +// SockAddrLink is a struct sockaddr_ll, from uapi/linux/if_packet.h. +// +// +marshal +type SockAddrLink struct { + Family uint16 + Protocol uint16 + InterfaceIndex int32 + ARPHardwareType uint16 + PacketType byte + HardwareAddrLen byte + HardwareAddr [8]byte +} + +// UnixPathMax is the maximum length of the path in an AF_UNIX socket. +// +// From uapi/linux/un.h. +const UnixPathMax = 108 + +// SockAddrUnix is struct sockaddr_un, from uapi/linux/un.h. +// +// +marshal +type SockAddrUnix struct { + Family uint16 + Path [UnixPathMax]int8 +} + +// SockAddr represents a union of valid socket address types. This is logically +// equivalent to struct sockaddr. SockAddr ensures that a well-defined set of +// types can be used as socket addresses. +type SockAddr interface { + marshal.Marshallable + + // implementsSockAddr exists purely to allow a type to indicate that they + // implement this interface. This method is a no-op and shouldn't be called. + implementsSockAddr() +} + +func (s *SockAddrInet) implementsSockAddr() {} +func (s *SockAddrInet6) implementsSockAddr() {} +func (s *SockAddrLink) implementsSockAddr() {} +func (s *SockAddrUnix) implementsSockAddr() {} +func (s *SockAddrNetlink) implementsSockAddr() {} + +// Linger is struct linger, from include/linux/socket.h. +// +// +marshal +type Linger struct { + OnOff int32 + Linger int32 +} + +// SizeOfLinger is the binary size of a Linger struct. +const SizeOfLinger = 8 + +// TCPInfo is a collection of TCP statistics. +// +// From uapi/linux/tcp.h. Newer versions of Linux continue to add new fields to +// the end of this struct or within existing unused space, so its size grows +// over time. The current iteration is based on linux v4.17. New versions are +// always backwards compatible. +// +// +marshal +type TCPInfo struct { + // State is the state of the connection. + State uint8 + + // CaState is the congestion control state. + CaState uint8 + + // Retransmits is the number of retransmissions triggered by RTO. + Retransmits uint8 + + // Probes is the number of unanswered zero window probes. + Probes uint8 + + // BackOff indicates exponential backoff. + Backoff uint8 + + // Options indicates the options enabled for the connection. + Options uint8 + + // WindowScale is the combination of snd_wscale (first 4 bits) and + // rcv_wscale (second 4 bits) + WindowScale uint8 + + // DeliveryRateAppLimited is a boolean and only the first bit is + // meaningful. + DeliveryRateAppLimited uint8 + + // RTO is the retransmission timeout. + RTO uint32 + + // ATO is the acknowledgement timeout interval. + ATO uint32 + + // SndMss is the send maximum segment size. + SndMss uint32 + + // RcvMss is the receive maximum segment size. + RcvMss uint32 + + // Unacked is the number of packets sent but not acknowledged. + Unacked uint32 + + // Sacked is the number of packets which are selectively acknowledged. + Sacked uint32 + + // Lost is the number of packets marked as lost. + Lost uint32 + + // Retrans is the number of retransmitted packets. + Retrans uint32 + + // Fackets is not used and is always zero. + Fackets uint32 + + // Times. + LastDataSent uint32 + LastAckSent uint32 + LastDataRecv uint32 + LastAckRecv uint32 + + // Metrics. + PMTU uint32 + RcvSsthresh uint32 + RTT uint32 + RTTVar uint32 + SndSsthresh uint32 + SndCwnd uint32 + Advmss uint32 + Reordering uint32 + + // RcvRTT is the receiver round trip time. + RcvRTT uint32 + + // RcvSpace is the current buffer space available for receiving data. + RcvSpace uint32 + + // TotalRetrans is the total number of retransmits seen since the start + // of the connection. + TotalRetrans uint32 + + // PacingRate is the pacing rate in bytes per second. + PacingRate uint64 + + // MaxPacingRate is the maximum pacing rate. + MaxPacingRate uint64 + + // BytesAcked is RFC4898 tcpEStatsAppHCThruOctetsAcked. + BytesAcked uint64 + + // BytesReceived is RFC4898 tcpEStatsAppHCThruOctetsReceived. + BytesReceived uint64 + + // SegsOut is RFC4898 tcpEStatsPerfSegsOut. + SegsOut uint32 + + // SegsIn is RFC4898 tcpEStatsPerfSegsIn. + SegsIn uint32 + + // NotSentBytes is the amount of bytes in the write queue that are not + // yet sent. + NotSentBytes uint32 + + // MinRTT is the minimum round trip time seen in the connection. + MinRTT uint32 + + // DataSegsIn is RFC4898 tcpEStatsDataSegsIn. + DataSegsIn uint32 + + // DataSegsOut is RFC4898 tcpEStatsDataSegsOut. + DataSegsOut uint32 + + // DeliveryRate is the most recent delivery rate in bytes per second. + DeliveryRate uint64 + + // BusyTime is the time in microseconds busy sending data. + BusyTime uint64 + + // RwndLimited is the time in microseconds limited by receive window. + RwndLimited uint64 + + // SndBufLimited is the time in microseconds limited by send buffer. + SndBufLimited uint64 + + // Delivered is the total data packets delivered including retransmits. + Delivered uint32 + + // DeliveredCE is the total ECE marked data packets delivered including + // retransmits. + DeliveredCE uint32 + + // BytesSent is RFC4898 tcpEStatsPerfHCDataOctetsOut. + BytesSent uint64 + + // BytesRetrans is RFC4898 tcpEStatsPerfOctetsRetrans. + BytesRetrans uint64 + + // DSACKDups is RFC4898 tcpEStatsStackDSACKDups. + DSACKDups uint32 + + // ReordSeen is the number of reordering events seen since the start of + // the connection. + ReordSeen uint32 +} + +// SizeOfTCPInfo is the binary size of a TCPInfo struct. +var SizeOfTCPInfo = (*TCPInfo)(nil).SizeBytes() + +// Control message types, from linux/socket.h. +const ( + SCM_CREDENTIALS = 0x2 + SCM_RIGHTS = 0x1 +) + +// A ControlMessageHeader is the header for a socket control message. +// +// ControlMessageHeader represents struct cmsghdr from linux/socket.h. +// +// +marshal +type ControlMessageHeader struct { + Length uint64 + Level int32 + Type int32 +} + +// SizeOfControlMessageHeader is the binary size of a ControlMessageHeader +// struct. +var SizeOfControlMessageHeader = (*ControlMessageHeader)(nil).SizeBytes() + +// A ControlMessageCredentials is an SCM_CREDENTIALS socket control message. +// +// ControlMessageCredentials represents struct ucred from linux/socket.h. +// +// +marshal +type ControlMessageCredentials struct { + PID int32 + UID uint32 + GID uint32 +} + +// A ControlMessageIPPacketInfo is IP_PKTINFO socket control message. +// +// ControlMessageIPPacketInfo represents struct in_pktinfo from linux/in.h. +// +// +marshal +// +stateify savable +type ControlMessageIPPacketInfo struct { + NIC int32 + LocalAddr InetAddr + DestinationAddr InetAddr +} + +// ControlMessageIPv6PacketInfo represents struct in6_pktinfo from linux/ipv6.h. +// +// +marshal +// +stateify savable +type ControlMessageIPv6PacketInfo struct { + Addr Inet6Addr + NIC uint32 +} + +// SizeOfControlMessageCredentials is the binary size of a +// ControlMessageCredentials struct. +var SizeOfControlMessageCredentials = (*ControlMessageCredentials)(nil).SizeBytes() + +// SizeOfControlMessageRight is the size of a single element in +// ControlMessageRights. +const SizeOfControlMessageRight = 4 + +// SizeOfControlMessageInq is the size of a TCP_INQ control message. +const SizeOfControlMessageInq = 4 + +// SizeOfControlMessageTOS is the size of an IP_TOS control message. +const SizeOfControlMessageTOS = 1 + +// SizeOfControlMessageTTL is the size of an IP_TTL control message. +const SizeOfControlMessageTTL = 4 + +// SizeOfControlMessageTClass is the size of an IPV6_TCLASS control message. +const SizeOfControlMessageTClass = 4 + +// SizeOfControlMessageHopLimit is the size of an IPV6_HOPLIMIT control message. +const SizeOfControlMessageHopLimit = 4 + +// SizeOfControlMessageIPPacketInfo is the size of an IP_PKTINFO control +// message. +const SizeOfControlMessageIPPacketInfo = 12 + +// SizeOfControlMessageIPv6PacketInfo is the size of a +// ControlMessageIPv6PacketInfo. +const SizeOfControlMessageIPv6PacketInfo = 20 + +// SCM_MAX_FD is the maximum number of FDs accepted in a single sendmsg call. +// From net/scm.h. +const SCM_MAX_FD = 253 + +// SO_ACCEPTCON is defined as __SO_ACCEPTCON in +// include/uapi/linux/net.h, which represents a listening socket +// state. Note that this is distinct from SO_ACCEPTCONN, which is a +// socket option for querying whether a socket is in a listening +// state. +const SO_ACCEPTCON = 1 << 16 + +// ICMP6Filter represents struct icmp6_filter from linux/icmpv6.h. +// +// +marshal +// +stateify savable +type ICMP6Filter struct { + Filter [8]uint32 +} + +// Size of corresponding structs. +var ( + ICMP6FilterSize = (*ICMP6Filter)(nil).SizeBytes() + SockAddrInetSize = (*SockAddrInet)(nil).SizeBytes() + SockAddrInet6Size = (*SockAddrInet6)(nil).SizeBytes() + SockAddrLinkSize = (*SockAddrLink)(nil).SizeBytes() +) diff --git a/pkg/abi/linux/splice.go b/pkg/abi/linux/splice.go new file mode 100644 index 0000000..650eb87 --- /dev/null +++ b/pkg/abi/linux/splice.go @@ -0,0 +1,23 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Constants for splice(2), sendfile(2) and tee(2). +const ( + SPLICE_F_MOVE = 1 << iota + SPLICE_F_NONBLOCK + SPLICE_F_MORE + SPLICE_F_GIFT +) diff --git a/pkg/abi/linux/tcp.go b/pkg/abi/linux/tcp.go new file mode 100644 index 0000000..d11f7c2 --- /dev/null +++ b/pkg/abi/linux/tcp.go @@ -0,0 +1,71 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Socket options from uapi/linux/tcp.h. +const ( + TCP_NODELAY = 1 + TCP_MAXSEG = 2 + TCP_CORK = 3 + TCP_KEEPIDLE = 4 + TCP_KEEPINTVL = 5 + TCP_KEEPCNT = 6 + TCP_SYNCNT = 7 + TCP_LINGER2 = 8 + TCP_DEFER_ACCEPT = 9 + TCP_WINDOW_CLAMP = 10 + TCP_INFO = 11 + TCP_QUICKACK = 12 + TCP_CONGESTION = 13 + TCP_MD5SIG = 14 + TCP_THIN_LINEAR_TIMEOUTS = 16 + TCP_THIN_DUPACK = 17 + TCP_USER_TIMEOUT = 18 + TCP_REPAIR = 19 + TCP_REPAIR_QUEUE = 20 + TCP_QUEUE_SEQ = 21 + TCP_REPAIR_OPTIONS = 22 + TCP_FASTOPEN = 23 + TCP_TIMESTAMP = 24 + TCP_NOTSENT_LOWAT = 25 + TCP_CC_INFO = 26 + TCP_SAVE_SYN = 27 + TCP_SAVED_SYN = 28 + TCP_REPAIR_WINDOW = 29 + TCP_FASTOPEN_CONNECT = 30 + TCP_ULP = 31 + TCP_MD5SIG_EXT = 32 + TCP_FASTOPEN_KEY = 33 + TCP_FASTOPEN_NO_COOKIE = 34 + TCP_ZEROCOPY_RECEIVE = 35 + TCP_INQ = 36 + TCP_TX_DELAY = 37 +) + +// Socket constants from include/net/tcp.h. +const ( + MAX_TCP_KEEPIDLE = 32767 + MAX_TCP_KEEPINTVL = 32767 + MAX_TCP_KEEPCNT = 127 +) + +// Congestion control states from include/uapi/linux/tcp.h. +const ( + TCP_CA_Open = 0 + TCP_CA_Disorder = 1 + TCP_CA_CWR = 2 + TCP_CA_Recovery = 3 + TCP_CA_Loss = 4 +) diff --git a/pkg/abi/linux/time.go b/pkg/abi/linux/time.go new file mode 100644 index 0000000..09407b7 --- /dev/null +++ b/pkg/abi/linux/time.go @@ -0,0 +1,288 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "math" + "time" +) + +const ( + // ClockTick is the length of time represented by a single clock tick, as + // used by times(2) and /proc/[pid]/stat. + ClockTick = time.Second / CLOCKS_PER_SEC + + // CLOCKS_PER_SEC is the number of ClockTicks per second. + // + // Linux defines this to be 100 on most architectures, irrespective of + // CONFIG_HZ. Userspace obtains the value through sysconf(_SC_CLK_TCK), + // which uses the AT_CLKTCK entry in the auxiliary vector if one is + // provided, and assumes 100 otherwise (glibc: + // sysdeps/posix/sysconf.c:__sysconf() => + // sysdeps/unix/sysv/linux/getclktck.c, elf/dl-support.c:_dl_aux_init()). + // + // Not to be confused with POSIX CLOCKS_PER_SEC, as used by clock(3); "XSI + // requires that [POSIX] CLOCKS_PER_SEC equals 1000000 independent of the + // actual resolution" - clock(3). + CLOCKS_PER_SEC = 100 +) + +// CPU clock types for use with clock_gettime(2) et al. +// +// The 29 most significant bits of a 32 bit clock ID are either a PID or a FD. +// +// Bits 1 and 0 give the type: PROF=0, VIRT=1, SCHED=2, or FD=3. +// +// Bit 2 indicates whether a cpu clock refers to a thread or a process. +const ( + CPUCLOCK_PROF = 0 + CPUCLOCK_VIRT = 1 + CPUCLOCK_SCHED = 2 + CPUCLOCK_MAX = 3 + CLOCKFD = CPUCLOCK_MAX + + CPUCLOCK_CLOCK_MASK = 3 + CPUCLOCK_PERTHREAD_MASK = 4 +) + +// Clock identifiers for use with clock_gettime(2), clock_getres(2), +// clock_nanosleep(2). +const ( + CLOCK_REALTIME = 0 + CLOCK_MONOTONIC = 1 + CLOCK_PROCESS_CPUTIME_ID = 2 + CLOCK_THREAD_CPUTIME_ID = 3 + CLOCK_MONOTONIC_RAW = 4 + CLOCK_REALTIME_COARSE = 5 + CLOCK_MONOTONIC_COARSE = 6 + CLOCK_BOOTTIME = 7 + CLOCK_REALTIME_ALARM = 8 + CLOCK_BOOTTIME_ALARM = 9 +) + +// Flags for clock_nanosleep(2). +const ( + TIMER_ABSTIME = 1 +) + +// Flags for timerfd syscalls (timerfd_create(2), timerfd_settime(2)). +const ( + // TFD_CLOEXEC is a timerfd_create flag. + TFD_CLOEXEC = O_CLOEXEC + + // TFD_NONBLOCK is a timerfd_create flag. + TFD_NONBLOCK = O_NONBLOCK + + // TFD_TIMER_ABSTIME is a timerfd_settime flag. + TFD_TIMER_ABSTIME = 1 +) + +// The safe number of seconds you can represent by int64. +const maxSecInDuration = math.MaxInt64 / int64(time.Second) + +// TimeT represents time_t in . It represents time in seconds. +// +// +marshal +type TimeT int64 + +// NsecToTimeT translates nanoseconds to TimeT (seconds). +func NsecToTimeT(nsec int64) TimeT { + return TimeT(nsec / 1e9) +} + +// Timespec represents struct timespec in . +// +// +marshal slice:TimespecSlice +type Timespec struct { + Sec int64 + Nsec int64 +} + +// Unix returns the second and nanosecond. +func (ts Timespec) Unix() (sec int64, nsec int64) { + return int64(ts.Sec), int64(ts.Nsec) +} + +// ToTime returns the Go time.Time representation. +func (ts Timespec) ToTime() time.Time { + return time.Unix(ts.Sec, ts.Nsec) +} + +// ToNsec returns the nanosecond representation. +func (ts Timespec) ToNsec() int64 { + return int64(ts.Sec)*1e9 + int64(ts.Nsec) +} + +// ToNsecCapped returns the safe nanosecond representation. +func (ts Timespec) ToNsecCapped() int64 { + if ts.Sec > maxSecInDuration { + return math.MaxInt64 + } + return ts.ToNsec() +} + +// ToDuration returns the safe nanosecond representation as time.Duration. +func (ts Timespec) ToDuration() time.Duration { + return time.Duration(ts.ToNsecCapped()) +} + +// Valid returns whether the timespec contains valid values. +func (ts Timespec) Valid() bool { + return !(ts.Sec < 0 || ts.Nsec < 0 || ts.Nsec >= int64(time.Second)) +} + +// NsecToTimespec translates nanoseconds to Timespec. +func NsecToTimespec(nsec int64) (ts Timespec) { + ts.Sec = nsec / 1e9 + ts.Nsec = nsec % 1e9 + return +} + +// DurationToTimespec translates time.Duration to Timespec. +func DurationToTimespec(dur time.Duration) Timespec { + return NsecToTimespec(dur.Nanoseconds()) +} + +// SizeOfTimeval is the size of a Timeval struct in bytes. +const SizeOfTimeval = 16 + +// Timeval represents struct timeval in . +// +// +marshal slice:TimevalSlice +type Timeval struct { + Sec int64 + Usec int64 +} + +// ToNsecCapped returns the safe nanosecond representation. +func (tv Timeval) ToNsecCapped() int64 { + if tv.Sec > maxSecInDuration { + return math.MaxInt64 + } + return int64(tv.Sec)*1e9 + int64(tv.Usec)*1e3 +} + +// ToDuration returns the safe nanosecond representation as a time.Duration. +func (tv Timeval) ToDuration() time.Duration { + return time.Duration(tv.ToNsecCapped()) +} + +// ToTime returns the Go time.Time representation. +func (tv Timeval) ToTime() time.Time { + return time.Unix(tv.Sec, tv.Usec*1e3) +} + +// NsecToTimeval translates nanosecond to Timeval. +func NsecToTimeval(nsec int64) (tv Timeval) { + nsec += 999 // round up to microsecond + tv.Sec = nsec / 1e9 + tv.Usec = nsec % 1e9 / 1e3 + return +} + +// DurationToTimeval translates time.Duration to Timeval. +func DurationToTimeval(dur time.Duration) Timeval { + return NsecToTimeval(dur.Nanoseconds()) +} + +// Itimerspec represents struct itimerspec in . +// +// +marshal +type Itimerspec struct { + Interval Timespec + Value Timespec +} + +// ItimerVal mimics the following struct in +// +// struct itimerval { +// struct timeval it_interval; /* next value */ +// struct timeval it_value; /* current value */ +// }; +// +// +marshal +type ItimerVal struct { + Interval Timeval + Value Timeval +} + +// ClockT represents type clock_t. +// +// +marshal +type ClockT int64 + +// ClockTFromDuration converts time.Duration to clock_t. +func ClockTFromDuration(d time.Duration) ClockT { + return ClockT(d / ClockTick) +} + +// Tms represents struct tms, used by times(2). +// +// +marshal +type Tms struct { + UTime ClockT + STime ClockT + CUTime ClockT + CSTime ClockT +} + +// TimerID represents type timer_t, which identifies a POSIX per-process +// interval timer. +// +// +marshal +type TimerID int32 + +// StatxTimestamp represents struct statx_timestamp. +// +// +marshal +type StatxTimestamp struct { + Sec int64 + Nsec uint32 + _ int32 +} + +// ToNsec returns the nanosecond representation. +func (sxts StatxTimestamp) ToNsec() int64 { + return int64(sxts.Sec)*1e9 + int64(sxts.Nsec) +} + +// ToNsecCapped returns the safe nanosecond representation. +func (sxts StatxTimestamp) ToNsecCapped() int64 { + if sxts.Sec > maxSecInDuration { + return math.MaxInt64 + } + return sxts.ToNsec() +} + +// NsecToStatxTimestamp translates nanoseconds to StatxTimestamp. +func NsecToStatxTimestamp(nsec int64) (ts StatxTimestamp) { + return StatxTimestamp{ + Sec: nsec / 1e9, + Nsec: uint32(nsec % 1e9), + } +} + +// ToTime returns the Go time.Time representation. +func (sxts StatxTimestamp) ToTime() time.Time { + return time.Unix(sxts.Sec, int64(sxts.Nsec)) +} + +// Utime represents struct utimbuf used by utimes(2). +// +// +marshal +type Utime struct { + Actime int64 + Modtime int64 +} diff --git a/pkg/abi/linux/timer.go b/pkg/abi/linux/timer.go new file mode 100644 index 0000000..e32d09e --- /dev/null +++ b/pkg/abi/linux/timer.go @@ -0,0 +1,23 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// itimer types for getitimer(2) and setitimer(2), from +// include/uapi/linux/time.h. +const ( + ITIMER_REAL = 0 + ITIMER_VIRTUAL = 1 + ITIMER_PROF = 2 +) diff --git a/pkg/abi/linux/tty.go b/pkg/abi/linux/tty.go new file mode 100644 index 0000000..048d878 --- /dev/null +++ b/pkg/abi/linux/tty.go @@ -0,0 +1,349 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +const ( + // NumControlCharacters is the number of control characters in Termios. + NumControlCharacters = 19 + // disabledChar is used to indicate that a control character is + // disabled. + disabledChar = 0 +) + +// Winsize is struct winsize, defined in uapi/asm-generic/termios.h. +// +// +marshal +type Winsize struct { + Row uint16 + Col uint16 + Xpixel uint16 + Ypixel uint16 +} + +// Termios is struct termios, defined in uapi/asm-generic/termbits.h. +// +// +marshal +type Termios struct { + InputFlags uint32 + OutputFlags uint32 + ControlFlags uint32 + LocalFlags uint32 + LineDiscipline uint8 + ControlCharacters [NumControlCharacters]uint8 +} + +// KernelTermios is struct ktermios/struct termios2, defined in +// uapi/asm-generic/termbits.h. +// +// +stateify savable +type KernelTermios struct { + InputFlags uint32 + OutputFlags uint32 + ControlFlags uint32 + LocalFlags uint32 + LineDiscipline uint8 + ControlCharacters [NumControlCharacters]uint8 + InputSpeed uint32 + OutputSpeed uint32 +} + +// IEnabled returns whether flag is enabled in termios input flags. +func (t *KernelTermios) IEnabled(flag uint32) bool { + return t.InputFlags&flag == flag +} + +// OEnabled returns whether flag is enabled in termios output flags. +func (t *KernelTermios) OEnabled(flag uint32) bool { + return t.OutputFlags&flag == flag +} + +// CEnabled returns whether flag is enabled in termios control flags. +func (t *KernelTermios) CEnabled(flag uint32) bool { + return t.ControlFlags&flag == flag +} + +// LEnabled returns whether flag is enabled in termios local flags. +func (t *KernelTermios) LEnabled(flag uint32) bool { + return t.LocalFlags&flag == flag +} + +// ToTermios copies fields that are shared with Termios into a new Termios +// struct. +func (t *KernelTermios) ToTermios() Termios { + return Termios{ + InputFlags: t.InputFlags, + OutputFlags: t.OutputFlags, + ControlFlags: t.ControlFlags, + LocalFlags: t.LocalFlags, + LineDiscipline: t.LineDiscipline, + ControlCharacters: t.ControlCharacters, + } +} + +// FromTermios copies fields that are shared with Termios into this +// KernelTermios struct. +func (t *KernelTermios) FromTermios(term Termios) { + t.InputFlags = term.InputFlags + t.OutputFlags = term.OutputFlags + t.ControlFlags = term.ControlFlags + t.LocalFlags = term.LocalFlags + t.LineDiscipline = term.LineDiscipline + t.ControlCharacters = term.ControlCharacters +} + +// IsTerminating returns whether c is a line terminating character. +func (t *KernelTermios) IsTerminating(cBytes []byte) bool { + // All terminating characters are 1 byte. + if len(cBytes) != 1 { + return false + } + c := cBytes[0] + + // Is this the user-set EOF character? + if t.IsEOF(c) { + return true + } + + switch c { + case disabledChar: + return false + case '\n', t.ControlCharacters[VEOL]: + return true + case t.ControlCharacters[VEOL2]: + return t.LEnabled(IEXTEN) + } + return false +} + +// IsEOF returns whether c is the EOF character. +func (t *KernelTermios) IsEOF(c byte) bool { + return c == t.ControlCharacters[VEOF] && t.ControlCharacters[VEOF] != disabledChar +} + +// Input flags. +const ( + IGNBRK = 0o000001 + BRKINT = 0o000002 + IGNPAR = 0o000004 + PARMRK = 0o000010 + INPCK = 0o000020 + ISTRIP = 0o000040 + INLCR = 0o000100 + IGNCR = 0o000200 + ICRNL = 0o000400 + IUCLC = 0o001000 + IXON = 0o002000 + IXANY = 0o004000 + IXOFF = 0o010000 + IMAXBEL = 0o020000 + IUTF8 = 0o040000 +) + +// Output flags. +const ( + OPOST = 0o000001 + OLCUC = 0o000002 + ONLCR = 0o000004 + OCRNL = 0o000010 + ONOCR = 0o000020 + ONLRET = 0o000040 + OFILL = 0o000100 + OFDEL = 0o000200 + NLDLY = 0o000400 + NL0 = 0o000000 + NL1 = 0o000400 + CRDLY = 0o003000 + CR0 = 0o000000 + CR1 = 0o001000 + CR2 = 0o002000 + CR3 = 0o003000 + TABDLY = 0o014000 + TAB0 = 0o000000 + TAB1 = 0o004000 + TAB2 = 0o010000 + TAB3 = 0o014000 + XTABS = 0o014000 + BSDLY = 0o020000 + BS0 = 0o000000 + BS1 = 0o020000 + VTDLY = 0o040000 + VT0 = 0o000000 + VT1 = 0o040000 + FFDLY = 0o100000 + FF0 = 0o000000 + FF1 = 0o100000 +) + +// Control flags. +const ( + CBAUD = 0o010017 + B0 = 0o000000 + B50 = 0o000001 + B75 = 0o000002 + B110 = 0o000003 + B134 = 0o000004 + B150 = 0o000005 + B200 = 0o000006 + B300 = 0o000007 + B600 = 0o000010 + B1200 = 0o000011 + B1800 = 0o000012 + B2400 = 0o000013 + B4800 = 0o000014 + B9600 = 0o000015 + B19200 = 0o000016 + B38400 = 0o000017 + EXTA = B19200 + EXTB = B38400 + CSIZE = 0o000060 + CS5 = 0o000000 + CS6 = 0o000020 + CS7 = 0o000040 + CS8 = 0o000060 + CSTOPB = 0o000100 + CREAD = 0o000200 + PARENB = 0o000400 + PARODD = 0o001000 + HUPCL = 0o002000 + CLOCAL = 0o004000 + CBAUDEX = 0o010000 + BOTHER = 0o010000 + B57600 = 0o010001 + B115200 = 0o010002 + B230400 = 0o010003 + B460800 = 0o010004 + B500000 = 0o010005 + B576000 = 0o010006 + B921600 = 0o010007 + B1000000 = 0o010010 + B1152000 = 0o010011 + B1500000 = 0o010012 + B2000000 = 0o010013 + B2500000 = 0o010014 + B3000000 = 0o010015 + B3500000 = 0o010016 + B4000000 = 0o010017 + CIBAUD = 0o02003600000 + CMSPAR = 0o10000000000 + CRTSCTS = 0o20000000000 + + // IBSHIFT is the shift from CBAUD to CIBAUD. + IBSHIFT = 16 +) + +// Local flags. +const ( + ISIG = 0o000001 + ICANON = 0o000002 + XCASE = 0o000004 + ECHO = 0o000010 + ECHOE = 0o000020 + ECHOK = 0o000040 + ECHONL = 0o000100 + NOFLSH = 0o000200 + TOSTOP = 0o000400 + ECHOCTL = 0o001000 + ECHOPRT = 0o002000 + ECHOKE = 0o004000 + FLUSHO = 0o010000 + PENDIN = 0o040000 + IEXTEN = 0o100000 + EXTPROC = 0o200000 +) + +// Control Character indices. +const ( + VINTR = 0 + VQUIT = 1 + VERASE = 2 + VKILL = 3 + VEOF = 4 + VTIME = 5 + VMIN = 6 + VSWTC = 7 + VSTART = 8 + VSTOP = 9 + VSUSP = 10 + VEOL = 11 + VREPRINT = 12 + VDISCARD = 13 + VWERASE = 14 + VLNEXT = 15 + VEOL2 = 16 +) + +// ControlCharacter returns the termios-style control character for the passed +// character. +// +// e.g., for Ctrl-C, i.e., ^C, call ControlCharacter('C'). +// +// Standard control characters are ASCII bytes 0 through 31. +func ControlCharacter(c byte) uint8 { + // A is 1, B is 2, etc. + return uint8(c - 'A' + 1) +} + +// DefaultControlCharacters is the default set of Termios control characters. +var DefaultControlCharacters = [NumControlCharacters]uint8{ + ControlCharacter('C'), // VINTR = ^C + ControlCharacter('\\'), // VQUIT = ^\ + '\x7f', // VERASE = DEL + ControlCharacter('U'), // VKILL = ^U + ControlCharacter('D'), // VEOF = ^D + 0, // VTIME + 1, // VMIN + 0, // VSWTC + ControlCharacter('Q'), // VSTART = ^Q + ControlCharacter('S'), // VSTOP = ^S + ControlCharacter('Z'), // VSUSP = ^Z + 0, // VEOL + ControlCharacter('R'), // VREPRINT = ^R + ControlCharacter('O'), // VDISCARD = ^O + ControlCharacter('W'), // VWERASE = ^W + ControlCharacter('V'), // VLNEXT = ^V + 0, // VEOL2 +} + +// MasterTermios is the terminal configuration of the master end of a Unix98 +// pseudoterminal. +var MasterTermios = KernelTermios{ + ControlFlags: B38400 | CS8 | CREAD, + ControlCharacters: DefaultControlCharacters, + InputSpeed: 38400, + OutputSpeed: 38400, +} + +// DefaultReplicaTermios is the default terminal configuration of the replica +// end of a Unix98 pseudoterminal. +var DefaultReplicaTermios = KernelTermios{ + InputFlags: ICRNL | IXON, + OutputFlags: OPOST | ONLCR, + ControlFlags: B38400 | CS8 | CREAD, + LocalFlags: ISIG | ICANON | ECHO | ECHOE | ECHOK | ECHOCTL | ECHOKE | IEXTEN, + ControlCharacters: DefaultControlCharacters, + InputSpeed: 38400, + OutputSpeed: 38400, +} + +// WindowSize corresponds to struct winsize defined in +// include/uapi/asm-generic/termios.h. +// +// +stateify savable +// +marshal +type WindowSize struct { + Rows uint16 + Cols uint16 + _ [4]byte // Padding for 2 unused shorts. +} diff --git a/pkg/abi/linux/uio.go b/pkg/abi/linux/uio.go new file mode 100644 index 0000000..1fd1e98 --- /dev/null +++ b/pkg/abi/linux/uio.go @@ -0,0 +1,18 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// UIO_MAXIOV is the maximum number of struct iovecs in a struct iovec array. +const UIO_MAXIOV = 1024 diff --git a/pkg/abi/linux/utsname.go b/pkg/abi/linux/utsname.go new file mode 100644 index 0000000..cb7c954 --- /dev/null +++ b/pkg/abi/linux/utsname.go @@ -0,0 +1,51 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "bytes" + "fmt" +) + +const ( + // UTSLen is the maximum length of strings contained in fields of + // UtsName. + UTSLen = 64 +) + +// UtsName represents struct utsname, the struct returned by uname(2). +// +// +marshal +type UtsName struct { + Sysname [UTSLen + 1]byte + Nodename [UTSLen + 1]byte + Release [UTSLen + 1]byte + Version [UTSLen + 1]byte + Machine [UTSLen + 1]byte + Domainname [UTSLen + 1]byte +} + +// utsNameString converts a UtsName entry to a string without NULs. +func utsNameString(s [UTSLen + 1]byte) string { + // The NUL bytes will remain even in a cast to string. We must + // explicitly strip them. + return string(bytes.TrimRight(s[:], "\x00")) +} + +func (u UtsName) String() string { + return fmt.Sprintf("{Sysname: %s, Nodename: %s, Release: %s, Version: %s, Machine: %s, Domainname: %s}", + utsNameString(u.Sysname), utsNameString(u.Nodename), utsNameString(u.Release), + utsNameString(u.Version), utsNameString(u.Machine), utsNameString(u.Domainname)) +} diff --git a/pkg/abi/linux/vfio.go b/pkg/abi/linux/vfio.go new file mode 100644 index 0000000..ed821a6 --- /dev/null +++ b/pkg/abi/linux/vfio.go @@ -0,0 +1,215 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The package implements VFIOuserspace driver interface. + +package linux + +// For IOCTLs requests from include/uapi/linux/vfio.h. +const ( + VFIO_TYPE = ';' + VFIO_BASE = 100 + + // VFIO extensions. + VFIO_TYPE1_IOMMU = 1 + VFIO_SPAPR_TCE_IOMMU = 2 + VFIO_TYPE1v2_IOMMU = 3 +) + +// VFIO device info flags. +const ( + // Device supports reset. + VFIO_DEVICE_FLAGS_RESET = 1 << iota + // VFIO-pci device. + VFIO_DEVICE_FLAGS_PCI + // VFIO-platform device. + VFIO_DEVICE_FLAGS_PLATFORM + // VFIO-amba device. + VFIO_DEVICE_FLAGS_AMBA + // VFIO-ccw device. + VFIO_DEVICE_FLAGS_CCW + // VFIO-ap device. + VFIO_DEVICE_FLAGS_AP + // VFIO-fsl-mc device. + VFIO_DEVICE_FLAGS_FSL_MC + // Info supports caps. + VFIO_DEVICE_FLAGS_CAPS + // VFIO-cdx device. + VFIO_DEVICE_FLAGS_CDX +) + +// VFIO region info flags. +const ( + // Region supports read. + VFIO_REGION_INFO_FLAG_READ = 1 << iota + // Region supports write. + VFIO_REGION_INFO_FLAG_WRITE + // Region supports mmap. + VFIO_REGION_INFO_FLAG_MMAP + // Info supports caps. + VFIO_REGION_INFO_FLAG_CAPS +) + +// VFIOIrqInfo flags. +const ( + VFIO_IRQ_INFO_EVENTFD = 1 << iota + VFIO_IRQ_INFO_MASKABLE + VFIO_IRQ_INFO_AUTOMASKED + VFIO_IRQ_INFO_NORESIZE +) + +// VFIOIrqSet flags. +const ( + VFIO_IRQ_SET_DATA_NONE = 1 << iota + VFIO_IRQ_SET_DATA_BOOL + VFIO_IRQ_SET_DATA_EVENTFD + VFIO_IRQ_SET_ACTION_MASK + VFIO_IRQ_SET_ACTION_UNMASK + VFIO_IRQ_SET_ACTION_TRIGGER + + VFIO_IRQ_SET_DATA_TYPE_MASK = VFIO_IRQ_SET_DATA_NONE | + VFIO_IRQ_SET_DATA_BOOL | + VFIO_IRQ_SET_DATA_EVENTFD + VFIO_IRQ_SET_ACTION_TYPE_MASK = VFIO_IRQ_SET_ACTION_MASK | + VFIO_IRQ_SET_ACTION_UNMASK | + VFIO_IRQ_SET_ACTION_TRIGGER +) + +// VFIOIrqSet index. +const ( + VFIO_PCI_INTX_IRQ_INDEX = iota + VFIO_PCI_MSI_IRQ_INDEX + VFIO_PCI_MSIX_IRQ_INDEX + VFIO_PCI_ERR_IRQ_INDEX + VFIO_PCI_REQ_IRQ_INDEX + VFIO_PCI_NUM_IRQS +) + +// VFIOIommuType1DmaMap flags. +const ( + // Readable from device. + VFIO_DMA_MAP_FLAG_READ = 1 << iota + // Writable from device. + VFIO_DMA_MAP_FLAG_WRITE + // Update the device's virtual address. + VFIO_DMA_MAP_FLAG_VADDR +) + +const ( + VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP = 1 +) + +// IOCTLs for VFIO file descriptor from include/uapi/linux/vfio.h. +var ( + VFIO_CHECK_EXTENSION = IO(VFIO_TYPE, VFIO_BASE+1) + VFIO_SET_IOMMU = IO(VFIO_TYPE, VFIO_BASE+2) + VFIO_GROUP_SET_CONTAINER = IO(VFIO_TYPE, VFIO_BASE+4) + VFIO_GROUP_UNSET_CONTAINER = IO(VFIO_TYPE, VFIO_BASE+5) + VFIO_GROUP_GET_DEVICE_FD = IO(VFIO_TYPE, VFIO_BASE+6) + VFIO_DEVICE_GET_INFO = IO(VFIO_TYPE, VFIO_BASE+7) + VFIO_DEVICE_GET_REGION_INFO = IO(VFIO_TYPE, VFIO_BASE+8) + VFIO_DEVICE_GET_IRQ_INFO = IO(VFIO_TYPE, VFIO_BASE+9) + VFIO_DEVICE_SET_IRQS = IO(VFIO_TYPE, VFIO_BASE+10) + VFIO_DEVICE_RESET = IO(VFIO_TYPE, VFIO_BASE+11) + VFIO_IOMMU_MAP_DMA = IO(VFIO_TYPE, VFIO_BASE+13) + VFIO_IOMMU_UNMAP_DMA = IO(VFIO_TYPE, VFIO_BASE+14) +) + +// VFIODeviceInfo is analogous to vfio_device_info +// from include/uapi/linux/vfio.h. +// +// +marshal +type VFIODeviceInfo struct { + Argsz uint32 + Flags uint32 + // The total amount of regions. + NumRegions uint32 + // The maximum number of IRQ. + NumIrqs uint32 + // Offset within info struct of first cap. + CapOffset uint32 + pad uint32 +} + +// VFIORegionInfo is analogous to vfio_region_info +// from include/uapi/linux/vfio.h. +// +// +marshal +type VFIORegionInfo struct { + Argsz uint32 + Flags uint32 + Index uint32 + // Offset within info struct of first cap. + capOffset uint32 + // Region size in bytes. + Size uint64 + // Region offset from start of device fd. + Offset uint64 +} + +// VFIOIrqInfo is analogous to vfio_irq_info +// from include/uapi/linux/vfio.h. +// +// +marshal +type VFIOIrqInfo struct { + Argsz uint32 + Flags uint32 + Index uint32 + Count uint32 +} + +// VFIOIrqSet is analogous to vfio_irq_set +// from include/uapi/linux/vfio.h. +// The last field `data` from vfio_irq_set is omitted which is an +// flexible array member. It will be handled separately. +// +// +marshal +type VFIOIrqSet struct { + Argsz uint32 + Flags uint32 + Index uint32 + Start uint32 + Count uint32 +} + +// VFIOIommuType1DmaMap is analogous to vfio_iommu_type1_dma_map +// from include/uapi/linux/vfio.h. +// +// +marshal +type VFIOIommuType1DmaMap struct { + Argsz uint32 + Flags uint32 + // Process virtual address. + Vaddr uint64 + // IO virtual address. + IOVa uint64 + // Size of mapping in bytes. + Size uint64 +} + +// VFIOIommuType1DmaUnmap is analogous to vfio_iommu_type1_dma_unmap +// from include/uapi/linux/vfio.h. +// +// +marshal +type VFIOIommuType1DmaUnmap struct { + Argsz uint32 + Flags uint32 + // IO virtual address. + IOVa uint64 + // Size of mapping in bytes. + Size uint64 + // The `data` field from vfio_iommu_type1_dma_unmap is omitted. The + // field is a flexible array member, and is needed only if the flag + // VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP is enabled. +} diff --git a/pkg/abi/linux/vfio_unsafe.go b/pkg/abi/linux/vfio_unsafe.go new file mode 100644 index 0000000..8448991 --- /dev/null +++ b/pkg/abi/linux/vfio_unsafe.go @@ -0,0 +1,22 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import "unsafe" + +// Size returns the number of bytes for a VFIOIrqSet object. +func (vfioIrqSet VFIOIrqSet) Size() uint64 { + return uint64(unsafe.Sizeof(vfioIrqSet)) +} diff --git a/pkg/abi/linux/wait.go b/pkg/abi/linux/wait.go new file mode 100644 index 0000000..7107291 --- /dev/null +++ b/pkg/abi/linux/wait.go @@ -0,0 +1,161 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "fmt" +) + +// Options for waitpid(2), wait4(2), and/or waitid(2), from +// include/uapi/linux/wait.h. +const ( + WNOHANG = 0x00000001 + WUNTRACED = 0x00000002 + WSTOPPED = WUNTRACED + WEXITED = 0x00000004 + WCONTINUED = 0x00000008 + WNOWAIT = 0x01000000 + WNOTHREAD = 0x20000000 + WALL = 0x40000000 + WCLONE = 0x80000000 +) + +// ID types for waitid(2), from include/uapi/linux/wait.h. +const ( + P_ALL = 0x0 + P_PID = 0x1 + P_PGID = 0x2 +) + +// WaitStatus represents a thread status, as returned by the wait* family of +// syscalls. +type WaitStatus uint32 + +// WaitStatusExit returns a WaitStatus representing the given exit status. +func WaitStatusExit(status int32) WaitStatus { + return WaitStatus(uint32(status) << 8) +} + +// WaitStatusTerminationSignal returns a WaitStatus representing termination by +// the given signal. +func WaitStatusTerminationSignal(sig Signal) WaitStatus { + return WaitStatus(uint32(sig)) +} + +// WaitStatusStopped returns a WaitStatus representing stoppage by the given +// signal or ptrace trap code. +func WaitStatusStopped(code uint32) WaitStatus { + return WaitStatus(code<<8 | 0x7f) +} + +// WaitStatusContinued returns a WaitStatus representing continuation by +// SIGCONT. +func WaitStatusContinued() WaitStatus { + return WaitStatus(0xffff) +} + +// WithCoreDump returns a copy of ws that indicates that a core dump was +// generated. +// +// Preconditions: ws.Signaled(). +func (ws WaitStatus) WithCoreDump() WaitStatus { + return ws | 0x80 +} + +// Exited returns true if ws represents an exit status, consistent with +// WIFEXITED. +func (ws WaitStatus) Exited() bool { + return ws&0x7f == 0 +} + +// Signaled returns true if ws represents a termination by signal, consistent +// with WIFSIGNALED. +func (ws WaitStatus) Signaled() bool { + // ws&0x7f != 0 (exited) and ws&0x7f != 0x7f (stopped or continued) + return ((ws&0x7f)+1)>>1 != 0 +} + +// CoreDumped returns true if ws indicates that a core dump was produced, +// consistent with WCOREDUMP. +// +// Preconditions: ws.Signaled(). +func (ws WaitStatus) CoreDumped() bool { + return ws&0x80 != 0 +} + +// Stopped returns true if ws represents a stoppage, consistent with +// WIFSTOPPED. +func (ws WaitStatus) Stopped() bool { + return ws&0xff == 0x7f +} + +// Continued returns true if ws represents a continuation by SIGCONT, +// consistent with WIFCONTINUED. +func (ws WaitStatus) Continued() bool { + return ws == 0xffff +} + +// ExitStatus returns the lower 8 bits of the exit status represented by ws, +// consistent with WEXITSTATUS. +// +// Preconditions: ws.Exited(). +func (ws WaitStatus) ExitStatus() uint32 { + return uint32((ws & 0xff00) >> 8) +} + +// TerminationSignal returns the termination signal represented by ws, +// consistent with WTERMSIG. +// +// Preconditions: ws.Signaled(). +func (ws WaitStatus) TerminationSignal() Signal { + return Signal(ws & 0x7f) +} + +// StopSignal returns the stop signal represented by ws, consistent with +// WSTOPSIG. +// +// Preconditions: ws.Stopped(). +func (ws WaitStatus) StopSignal() Signal { + return Signal((ws & 0xff00) >> 8) +} + +// PtraceEvent returns the PTRACE_EVENT_* field in ws. +// +// Preconditions: ws.Stopped(). +func (ws WaitStatus) PtraceEvent() uint32 { + return uint32(ws >> 16) +} + +// String implements fmt.Stringer.String. +func (ws WaitStatus) String() string { + switch { + case ws.Exited(): + return fmt.Sprintf("exit status %d", ws.ExitStatus()) + case ws.Signaled(): + if ws.CoreDumped() { + return fmt.Sprintf("killed by signal %d (core dumped)", ws.TerminationSignal()) + } + return fmt.Sprintf("killed by signal %d", ws.TerminationSignal()) + case ws.Stopped(): + if ev := ws.PtraceEvent(); ev != 0 { + return fmt.Sprintf("stopped by signal %d (PTRACE_EVENT %d)", ws.StopSignal(), ev) + } + return fmt.Sprintf("stopped by signal %d", ws.StopSignal()) + case ws.Continued(): + return "continued" + default: + return fmt.Sprintf("unknown status %#x", uint32(ws)) + } +} diff --git a/pkg/abi/linux/xattr.go b/pkg/abi/linux/xattr.go new file mode 100644 index 0000000..6d6606e --- /dev/null +++ b/pkg/abi/linux/xattr.go @@ -0,0 +1,39 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +// Constants for extended attributes. +const ( + XATTR_NAME_MAX = 255 + XATTR_SIZE_MAX = 65536 + XATTR_LIST_MAX = 65536 + + XATTR_CREATE = 1 + XATTR_REPLACE = 2 + + XATTR_SECURITY_PREFIX = "security." + XATTR_SECURITY_PREFIX_LEN = len(XATTR_SECURITY_PREFIX) + + XATTR_SECURITY_CAPABILITY = XATTR_SECURITY_PREFIX + "capability" + + XATTR_SYSTEM_PREFIX = "system." + XATTR_SYSTEM_PREFIX_LEN = len(XATTR_SYSTEM_PREFIX) + + XATTR_TRUSTED_PREFIX = "trusted." + XATTR_TRUSTED_PREFIX_LEN = len(XATTR_TRUSTED_PREFIX) + + XATTR_USER_PREFIX = "user." + XATTR_USER_PREFIX_LEN = len(XATTR_USER_PREFIX) +) diff --git a/pkg/abi/nvgpu/classes.go b/pkg/abi/nvgpu/classes.go new file mode 100644 index 0000000..b186874 --- /dev/null +++ b/pkg/abi/nvgpu/classes.go @@ -0,0 +1,590 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +import ( + "fmt" +) + +// ClassID is a client class ID, in the sense of +// src/nvidia/src/kernel/rmapi/resource_desc.h:RS_RESOURCE_DESC::externalClassID. +// +// +marshal +type ClassID uint32 + +// String implements fmt.Stringer.String. +func (id ClassID) String() string { + // Include leading zeroes for easier searchability, both here and in + // g_allclasses.h. + return fmt.Sprintf("0x%08x", uint32(id)) +} + +// IsRootClient returns true if the given class ID is a root client class. +func (id ClassID) IsRootClient() bool { + switch id { + case NV01_ROOT, NV01_ROOT_NON_PRIV, NV01_ROOT_CLIENT: + return true + default: + return false + } +} + +// Class IDs, from src/nvidia/generated/g_allclasses.h. +const ( + NV01_ROOT = 0x00000000 + NV01_ROOT_NON_PRIV = 0x00000001 + NV01_CONTEXT_DMA = 0x00000002 + NV01_EVENT = 0x00000005 + NV01_MEMORY_SYSTEM = 0x0000003e + NV01_MEMORY_LOCAL_PRIVILEGED = 0x0000003f + NV01_MEMORY_LOCAL_USER = 0x00000040 + NV01_ROOT_CLIENT = 0x00000041 + NV_MEMORY_EXTENDED_USER = 0x00000042 + NV01_MEMORY_VIRTUAL = 0x00000070 + NV01_MEMORY_SYSTEM_OS_DESCRIPTOR = 0x00000071 + NV01_EVENT_OS_EVENT = 0x00000079 + NV01_DEVICE_0 = 0x00000080 + NV_SEMAPHORE_SURFACE = 0x000000da + RM_USER_SHARED_DATA = 0x000000de + NV_MEMORY_FABRIC = 0x000000f8 + NV_MEMORY_MULTICAST_FABRIC = 0x000000fd + NV_MEMORY_MAPPER = 0x000000fe + NV20_SUBDEVICE_0 = 0x00002080 + NV2081_BINAPI = 0x00002081 + NV50_P2P = 0x0000503b + NV50_THIRD_PARTY_P2P = 0x0000503c + NV50_MEMORY_VIRTUAL = 0x000050a0 + GT200_DEBUGGER = 0x000083de + FERMI_TWOD_A = 0x0000902d + FERMI_CONTEXT_SHARE_A = 0x00009067 + GF100_DISP_SW = 0x00009072 + GF100_ZBC_CLEAR = 0x00009096 + GF100_PROFILER = 0x000090cc + GF100_SUBDEVICE_MASTER = 0x000090e6 + FERMI_VASPACE_A = 0x000090f1 + KEPLER_CHANNEL_GROUP_A = 0x0000a06c + NVENC_SW_SESSION = 0x0000a0bc + KEPLER_INLINE_TO_MEMORY_B = 0x0000a140 + VOLTA_USERMODE_A = 0x0000c361 + TURING_USERMODE_A = 0x0000c461 + TURING_CHANNEL_GPFIFO_A = 0x0000c46f + NVB8B0_VIDEO_DECODER = 0x0000b8b0 // Hopper + NVC4B0_VIDEO_DECODER = 0x0000c4b0 // Turing + NVC6B0_VIDEO_DECODER = 0x0000c6b0 // Ampere + NVC7B0_VIDEO_DECODER = 0x0000c7b0 // Ampere + NVC9B0_VIDEO_DECODER = 0x0000c9b0 // Ada + NVCDB0_VIDEO_DECODER = 0x0000cdb0 // Blackwell + NVC4B7_VIDEO_ENCODER = 0x0000c4b7 + NVC7B7_VIDEO_ENCODER = 0x0000c7b7 + NVC9B7_VIDEO_ENCODER = 0x0000c9b7 + AMPERE_CHANNEL_GPFIFO_A = 0x0000c56f + TURING_A = 0x0000c597 + TURING_DMA_COPY_A = 0x0000c5b5 + TURING_COMPUTE_A = 0x0000c5c0 + HOPPER_USERMODE_A = 0x0000c661 + BLACKWELL_USERMODE_A = 0x0000c761 + AMPERE_A = 0x0000c697 + AMPERE_DMA_COPY_A = 0x0000c6b5 + AMPERE_COMPUTE_A = 0x0000c6c0 + AMPERE_DMA_COPY_B = 0x0000c7b5 + AMPERE_COMPUTE_B = 0x0000c7c0 + HOPPER_CHANNEL_GPFIFO_A = 0x0000c86f + HOPPER_DMA_COPY_A = 0x0000c8b5 + BLACKWELL_CHANNEL_GPFIFO_A = 0x0000c96f + BLACKWELL_DMA_COPY_A = 0x0000c9b5 + BLACKWELL_CHANNEL_GPFIFO_B = 0x0000ca6f + BLACKWELL_DMA_COPY_B = 0x0000cab5 + ADA_A = 0x0000c997 + ADA_COMPUTE_A = 0x0000c9c0 + NV_CONFIDENTIAL_COMPUTE = 0x0000cb33 + HOPPER_A = 0x0000cb97 + HOPPER_SEC2_WORK_LAUNCH_A = 0x0000cba2 + HOPPER_COMPUTE_A = 0x0000cbc0 + BLACKWELL_INLINE_TO_MEMORY_A = 0x0000cd40 + BLACKWELL_A = 0x0000cd97 + BLACKWELL_COMPUTE_A = 0x0000cdc0 + BLACKWELL_B = 0x0000ce97 + BLACKWELL_COMPUTE_B = 0x0000cec0 +) + +// From src/common/sdk/nvidia/inc/class/cl0000.h: +const ( + NV01_NULL_OBJECT = 0x0 +) + +// NV2081_ALLOC_PARAMETERS is the alloc params type for NV2081_BINAPI, from +// src/common/sdk/nvidia/inc/class/cl2081.h. +// +// +marshal +type NV2081_ALLOC_PARAMETERS struct { + Reserved uint32 +} + +// NV0005_ALLOC_PARAMETERS is the alloc params type for NV01_EVENT* classes +// from src/common/sdk/nvidia/inc/class/cl0005.h. +// +// +marshal +type NV0005_ALLOC_PARAMETERS struct { + HParentClient Handle + HSrcResource Handle + HClass ClassID + NotifyIndex uint32 + Data P64 // actually FD for NV01_EVENT_OS_EVENT, see src/nvidia/src/kernel/rmapi/event.c:eventConstruct_IMPL() => src/nvidia/arch/nvalloc/unix/src/os.c:osUserHandleToKernelPtr() +} + +// From src/common/sdk/nvidia/inc/class/cl208f.h +const ( + NV20_SUBDEVICE_DIAG = 0x0000208f +) + +// From src/common/sdk/nvidia/inc/class/cl0070.h: +const ( + NV_MEMORY_VIRTUAL_SYSMEM_DYNAMIC_HVASPACE = 0xffffffff +) + +// NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS is the alloc params type for +// NV01_MEMORY_VIRTUAL, from src/common/sdk/nvidia/inc/class/cl0070.h. +// +// +marshal +type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS struct { + Offset uint64 + Limit uint64 + HVASpace Handle + Pad0 [4]byte +} + +// From src/common/sdk/nvidia/inc/class/cl0073.h +const ( + NV04_DISPLAY_COMMON = 0x00000073 +) + +// NV0080_ALLOC_PARAMETERS is the alloc params type for NV01_DEVICE_0, from +// src/common/sdk/nvidia/inc/class/cl0080.h. +// +// +marshal +type NV0080_ALLOC_PARAMETERS struct { + DeviceID uint32 + HClientShare Handle + HTargetClient Handle + HTargetDevice Handle + Flags uint32 + Pad0 [4]byte + VASpaceSize uint64 + VAStartInternal uint64 + VALimitInternal uint64 + VAMode uint32 + Pad1 [4]byte +} + +// NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS is the alloc params type for +// NV_SEMAPHORE_SURFACE, from src/common/sdk/nvidia/inc/class/cl00da.h. +// +// +marshal +type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS struct { + HSemaphoreMem Handle + HMaxSubmittedMem Handle + flags uint64 +} + +// NV2080_ALLOC_PARAMETERS is the alloc params type for NV20_SUBDEVICE_0, from +// src/common/sdk/nvidia/inc/class/cl2080.h. +// +// +marshal +type NV2080_ALLOC_PARAMETERS struct { + SubDeviceID uint32 +} + +// NV_CONTEXT_DMA_ALLOCATION_PARAMS is the alloc params type for various NV01_CONTEXT_DMA +// allocation classes, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_CONTEXT_DMA_ALLOCATION_PARAMS struct { + HSubDevice Handle + Flags uint32 + HMemory Handle + _ uint32 + Offset uint64 + Limit uint64 +} + +// NV_MEMORY_ALLOCATION_PARAMS is the alloc params type for various NV*_MEMORY* +// allocation classes, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_MEMORY_ALLOCATION_PARAMS struct { + Owner uint32 + Type uint32 + Flags uint32 + Width uint32 + Height uint32 + Pitch int32 + Attr uint32 + Attr2 uint32 + Format uint32 + ComprCovg uint32 + ZcullCovg uint32 + _ uint32 + RangeLo uint64 + RangeHi uint64 + Size uint64 + Alignment uint64 + Offset uint64 + Limit uint64 + Address P64 + CtagOffset uint32 + HVASpace Handle + InternalFlags uint32 + Tag uint32 +} + +// NV_MEMORY_ALLOCATION_PARAMS_V545 is the updated version of +// NV_MEMORY_ALLOCATION_PARAMS since 545.23.06. +// +// +marshal +type NV_MEMORY_ALLOCATION_PARAMS_V545 struct { + NV_MEMORY_ALLOCATION_PARAMS + NumaNode int32 + _ uint32 +} + +// NV503B_BAR1_P2P_DMA_INFO from src/common/sdk/nvidia/inc/class/cl503b.h. +// +// +marshal +type NV503B_BAR1_P2P_DMA_INFO struct { + DmaAddress uint64 + DmaSize uint64 +} + +// NV503B_ALLOC_PARAMETERS is the alloc params type for NV50_P2P, from +// src/common/sdk/nvidia/inc/class/cl503b.h. +// +// +marshal +type NV503B_ALLOC_PARAMETERS struct { + HSubDevice Handle + HPeerSubDevice Handle + SubDevicePeerIDMask uint32 + PeerSubDevicePeerIDMask uint32 + MailboxBar1Addr uint64 + MailboxTotalSize uint32 + Flags uint32 + SubDeviceEgmPeerIDMask uint32 + PeerSubDeviceEgmPeerIDMask uint32 + L2pBar1P2PDmaInfo NV503B_BAR1_P2P_DMA_INFO + P2lBar1P2PDmaInfo NV503B_BAR1_P2P_DMA_INFO +} + +// NV503C_ALLOC_PARAMETERS is the alloc params type for NV50_THIRD_PARTY_P2P, +// from src/common/sdk/nvidia/inc/class/cl503c.h. +// +// +marshal +type NV503C_ALLOC_PARAMETERS struct { + Flags uint32 +} + +// NV83DE_ALLOC_PARAMETERS is the alloc params type for GT200_DEBUGGER, +// from src/common/sdk/nvidia/inc/class/cl83de.h. +// +// +marshal +type NV83DE_ALLOC_PARAMETERS struct { + HDebuggerClient_Obsolete Handle + HAppClient Handle + HClass3DObject Handle +} + +// NV_CTXSHARE_ALLOCATION_PARAMETERS is the alloc params type for +// FERMI_CONTEXT_SHARE_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_CTXSHARE_ALLOCATION_PARAMETERS struct { + HVASpace Handle + Flags uint32 + SubctxID uint32 +} + +// NV_VASPACE_ALLOCATION_PARAMETERS is the alloc params type for +// FERMI_VASPACE_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_VASPACE_ALLOCATION_PARAMETERS struct { + Index uint32 + Flags uint32 + VASize uint64 + VAStartInternal uint64 + VALimitInternal uint64 + BigPageSize uint32 + Pad0 [4]byte + VABase uint64 +} + +// NV_VASPACE_ALLOCATION_PARAMETERS_V580 is the updated version of +// NV_VASPACE_ALLOCATION_PARAMETERS since 580.65.06. +// +// +marshal +type NV_VASPACE_ALLOCATION_PARAMETERS_V580 struct { + NV_VASPACE_ALLOCATION_PARAMETERS + Pasid uint32 + Pad1 [4]byte +} + +// NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS is the alloc params type for +// KEPLER_CHANNEL_GROUP_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS struct { + HObjectError Handle + HObjectECCError Handle + HVASpace Handle + EngineType uint32 + BIsCallingContextVgpuPlugin uint8 + Pad0 [3]byte +} + +// NV_MEMORY_DESC_PARAMS is from +// src/common/sdk/nvidia/inc/alloc/alloc_channel.h. +// +// +marshal +type NV_MEMORY_DESC_PARAMS struct { + Base uint64 + Size uint64 + AddressSpace uint32 + CacheAttrib uint32 +} + +// NV_BSP_ALLOCATION_PARAMETERS is the alloc params type for +// NV*VIDEO_DECODER, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_BSP_ALLOCATION_PARAMETERS struct { + Size uint32 + ProhibitMultipleInstances uint32 + EngineInstance uint32 +} + +// NV_MSENC_ALLOCATION_PARAMETERS is the alloc params type for +// NV*_VIDEO_ENCODER, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_MSENC_ALLOCATION_PARAMETERS struct { + Size uint32 + ProhibitMultipleInstances uint32 + EngineInstance uint32 +} + +// NV_CHANNEL_ALLOC_PARAMS is the alloc params type for TURING_CHANNEL_GPFIFO_A +// and AMPERE_CHANNEL_GPFIFO_A, from +// src/common/sdk/nvidia/inc/alloc/alloc_channel.h. +// +// +marshal +type NV_CHANNEL_ALLOC_PARAMS struct { + HObjectError Handle + HObjectBuffer Handle + GPFIFOOffset uint64 + GPFIFOEntries uint32 + Flags uint32 + HContextShare Handle + HVASpace Handle + HUserdMemory [NV_MAX_SUBDEVICES]Handle + UserdOffset [NV_MAX_SUBDEVICES]uint64 + EngineType uint32 + CID uint32 + SubDeviceID uint32 + HObjectECCError Handle + InstanceMem NV_MEMORY_DESC_PARAMS + UserdMem NV_MEMORY_DESC_PARAMS + RamfcMem NV_MEMORY_DESC_PARAMS + MthdbufMem NV_MEMORY_DESC_PARAMS + HPhysChannelGroup Handle + InternalFlags uint32 + ErrorNotifierMem NV_MEMORY_DESC_PARAMS + ECCErrorNotifierMem NV_MEMORY_DESC_PARAMS + ProcessID uint32 + SubProcessID uint32 + EncryptIv [CC_CHAN_ALLOC_IV_SIZE_DWORD]uint32 + DecryptIv [CC_CHAN_ALLOC_IV_SIZE_DWORD]uint32 + HmacNonce [CC_CHAN_ALLOC_NONCE_SIZE_DWORD]uint32 +} + +// NV_CHANNEL_ALLOC_PARAMS_V570 is the updated version of +// NV_CHANNEL_ALLOC_PARAMS since 570.86.15. +// +// +marshal +type NV_CHANNEL_ALLOC_PARAMS_V570 struct { + NV_CHANNEL_ALLOC_PARAMS + TPCConfigID uint32 + _ uint32 +} + +// NVB0B5_ALLOCATION_PARAMETERS is the alloc param type for TURING_DMA_COPY_A, +// AMPERE_DMA_COPY_A, and AMPERE_DMA_COPY_B from +// src/common/sdk/nvidia/inc/class/clb0b5sw.h. +// +// +marshal +type NVB0B5_ALLOCATION_PARAMETERS struct { + Version uint32 + EngineType uint32 +} + +// NV_GR_ALLOCATION_PARAMETERS is the alloc param type for TURING_COMPUTE_A, +// AMPERE_COMPUTE_A, and ADA_COMPUTE_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_GR_ALLOCATION_PARAMETERS struct { + Version uint32 + Flags uint32 + Size uint32 + Caps uint32 +} + +// NV_HOPPER_USERMODE_A_PARAMS is the alloc param type for HOPPER_USERMODE_A, +// from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_HOPPER_USERMODE_A_PARAMS struct { + Bar1Mapping uint8 + Priv uint8 +} + +// NV9072_ALLOCATION_PARAMETERS is the alloc param type for GF100_DISP_SW, +// from src/common/sdk/nvidia/inc/class/cl9072.h. +// +// +marshal +type NV9072_ALLOCATION_PARAMETERS struct { + LogicalHeadID uint32 + DisplayMask uint32 + Caps uint32 +} + +// NV00DE_ALLOC_PARAMETERS is the alloc param type for RM_USER_SHARED_DATA, +// from src/common/sdk/nvidia/inc/class/cl00de.h. +// +// +marshal +type NV00DE_ALLOC_PARAMETERS struct { + Reserved uint32 +} + +// NV00DE_ALLOC_PARAMETERS_V545 is the updated version of +// NV00DE_ALLOC_PARAMETERS since 545.23.06. +// +// +marshal +type NV00DE_ALLOC_PARAMETERS_V545 struct { + PolledDataMask uint64 +} + +// +marshal +type nv00f8Map struct { + offset uint64 + hVidMem Handle + flags uint32 +} + +// NV00F8_ALLOCATION_PARAMETERS is the alloc param type for NV_MEMORY_FABRIC, +// from src/common/sdk/nvidia/inc/class/cl00f8.h. +// +// +marshal +type NV00F8_ALLOCATION_PARAMETERS struct { + Alignment uint64 + AllocSize uint64 + PageSize uint64 + AllocFlags uint32 + _ uint32 + Map nv00f8Map +} + +// From src/common/sdk/nvidia/inc/class/cl00e0.h +const ( + NV_MEM_EXPORT_UUID_LEN = 16 +) + +// NV_EXPORT_MEM_PACKET is from +// src/common/sdk/nvidia/inc/class/cl00e0.h +// +// +marshal +type NV_EXPORT_MEM_PACKET struct { + UUID [NV_MEM_EXPORT_UUID_LEN]uint8 + Opaque [16]uint8 +} + +// NV00FD_ALLOCATION_PARAMETERS is the alloc param type for NV_MEMORY_MULTICAST_FABRIC +// from src/common/sdk/nvidia/inc/class/cl00fd.h +// +// +marshal +type NV00FD_ALLOCATION_PARAMETERS struct { + Alignment uint64 + AllocSize uint64 + PageSize uint32 + AllocFlags uint32 + NumGPUs uint32 + _ uint32 + POsEvent P64 +} + +// NV00FD_ALLOCATION_PARAMETERS_V545 is the updated version of +// NV00FD_ALLOCATION_PARAMETERS since 545.23.06. +// +// +marshal +type NV00FD_ALLOCATION_PARAMETERS_V545 struct { + ExpPacket NV_EXPORT_MEM_PACKET + Index uint16 + _ [6]byte + NV00FD_ALLOCATION_PARAMETERS +} + +// NV_MEMORY_MAPPER_ALLOCATION_PARAMS is the alloc param type for +// NV_MEMORY_MAPPER, from src/common/sdk/nvidia/inc/class/cl00fe.h. +// +// +marshal +type NV_MEMORY_MAPPER_ALLOCATION_PARAMS struct { + unused uint8 +} + +// NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 is the updated version of +// NV_MEMORY_MAPPER_ALLOCATION_PARAMS since 550.40.07. +// +// +marshal +type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 struct { + HSemaphoreSurface Handle + MaxQueueSize uint32 +} + +// NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 is the updated version of +// NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 since 555.42.02. +// +// +marshal +type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 struct { + NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 + HNotificationMemory Handle + _ uint32 + NotificationOffset uint64 +} + +// NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS is the alloc param type for +// NV_CONFIDENTIAL_COMPUTE, from src/common/sdk/nvidia/inc/class/clcb33.h. +// +// +marshal +type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS struct { + Handle Handle +} + +// NVA0BC_ALLOC_PARAMETERS is the alloc param type for +// NVENC_SW_SESSION, from src/common/sdk/nvidia/inc/class/cla0bc.h +// +// +marshal +type NVA0BC_ALLOC_PARAMETERS struct { + CodecType uint32 + HResolution uint32 + VResolution uint32 + Version uint32 + HMem Handle +} diff --git a/pkg/abi/nvgpu/ctrl.go b/pkg/abi/nvgpu/ctrl.go new file mode 100644 index 0000000..4d3db96 --- /dev/null +++ b/pkg/abi/nvgpu/ctrl.go @@ -0,0 +1,719 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// From src/nvidia/interface/deprecated/rmapi_deprecated.h: +const ( + RM_GSS_LEGACY_MASK = 0x00008000 +) + +// From src/nvidia/inc/kernel/rmapi/param_copy.h: +const ( + // RMAPI_PARAM_COPY_MAX_PARAMS_SIZE is the size limit imposed while copying + // "embedded pointers" in rmapi parameter structs. + // See src/nvidia/src/kernel/rmapi/param_copy.c:rmapiParamsAcquire(). + RMAPI_PARAM_COPY_MAX_PARAMS_SIZE = 1 * 1024 * 1024 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlxxxx.h: + +// NVXXXX_CTRL_XXX_INFO is typedef-ed as the following in the driver: +// - NV0080_CTRL_GR_INFO +// - NV2080_CTRL_FB_INFO +// - NV2080_CTRL_GR_INFO +// - NV2080_CTRL_BIOS_INFO +// - NV0041_CTRL_SURFACE_INFO +// +// +marshal +type NVXXXX_CTRL_XXX_INFO struct { + Index uint32 + Data uint32 +} + +// CtrlXxxInfoSize is sizeof(NVXXXX_CTRL_XXX_INFO). +var CtrlXxxInfoSize = uint32((*NVXXXX_CTRL_XXX_INFO)(nil).SizeBytes()) + +// HasCtrlInfoList is a type constraint for parameter structs containing a list +// of NVXXXX_CTRL_XXX_INFO and are simple otherwise. +type HasCtrlInfoList interface { + ListSize() uint32 + SetCtrlInfoList(ptr P64) + CtrlInfoList() P64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000client.h: +const ( + NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE = 0xd01 + NV0000_CTRL_CMD_CLIENT_SET_INHERITED_SHARE_POLICY = 0xd04 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gpu.h: +const ( + NV0000_CTRL_CMD_GPU_GET_ATTACHED_IDS = 0x201 + NV0000_CTRL_CMD_GPU_GET_ID_INFO = 0x202 + NV0000_CTRL_CMD_GPU_GET_DEVICE_IDS = 0x204 + NV0000_CTRL_CMD_GPU_GET_ID_INFO_V2 = 0x205 + NV0000_CTRL_CMD_GPU_GET_PROBED_IDS = 0x214 + NV0000_CTRL_CMD_GPU_ATTACH_IDS = 0x215 + NV0000_CTRL_CMD_GPU_DETACH_IDS = 0x216 + NV0000_CTRL_CMD_GPU_GET_PCI_INFO = 0x21b + NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID = 0x275 + NV0000_CTRL_CMD_GPU_QUERY_DRAIN_STATE = 0x279 + NV0000_CTRL_CMD_GPU_GET_MEMOP_ENABLE = 0x27b + NV0000_CTRL_CMD_GPU_GET_ACTIVE_DEVICE_IDS = 0x288 + NV0000_CTRL_CMD_GPU_ASYNC_ATTACH_ID = 0x289 + NV0000_CTRL_CMD_GPU_WAIT_ATTACH_ID = 0x290 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gsync.h: +const ( + NV0000_CTRL_CMD_GSYNC_GET_ATTACHED_IDS = 0x301 +) + +// NV0000_CTRL_GPU_GET_ID_INFO_PARAMS is the param type for NV0000_CTRL_CMD_GPU_GET_ID_INFO, +// from src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gpu.h. +// +// +marshal +type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS struct { + GpuID uint32 + GpuFlags uint32 + DeviceInstance uint32 + SubDeviceInstance uint32 + SzName P64 + SliStatus uint32 + BoardID uint32 + GpuInstance uint32 + NumaID int32 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000syncgpuboost.h: +const ( + NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_INFO = 0xa04 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000system.h: +const ( + NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION = 0x101 + NV0000_CTRL_CMD_SYSTEM_GET_CPU_INFO = 0x102 + NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS = 0x127 + NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_V2 = 0x12b + NV0000_CTRL_CMD_SYSTEM_GET_FABRIC_STATUS = 0x136 + NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_MATRIX = 0x13a + NV0000_CTRL_CMD_SYSTEM_GET_FEATURES = 0x1f0 + NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS = 32 + NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE = 9 +) + +// NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS is the param type for NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS, +// from src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000system.h. +// +// +marshal +type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS struct { + GpuIDs [NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS]uint32 + GpuCount uint32 + P2PCaps uint32 + P2POptimalReadCEs uint32 + P2POptimalWriteCEs uint32 + P2PCapsStatus [NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE]uint8 + _ [7]byte + BusPeerIDs P64 +} + +// NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 is the updated version of +// NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS since 550.40.07. +// +// +marshal +type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 struct { + NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS + BusEgmPeerIDs P64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000unix.h: +const ( + NV0000_CTRL_CMD_OS_UNIX_EXPORT_OBJECT_TO_FD = 0x3d05 + NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_FROM_FD = 0x3d06 + NV0000_CTRL_CMD_OS_UNIX_GET_EXPORT_OBJECT_INFO = 0x3d08 + NV0000_CTRL_CMD_OS_UNIX_EXPORT_OBJECTS_TO_FD = 0x3d0b + NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECTS_FROM_FD = 0x3d0c + NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE = 64 + NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS = 512 + NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS = 128 +) + +// +marshal +type NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS struct { + FD int32 + DeviceInstance uint32 + MaxObjects uint16 + Metadata [NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE]uint8 + Pad [2]byte +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) GetFrontendFD() int32 { + return p.FD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) SetFrontendFD(fd int32) { + p.FD = fd +} + +// +marshal +type NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545 struct { + FD int32 + DeviceInstance uint32 + GpuInstanceID uint32 + MaxObjects uint16 + Metadata [NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE]uint8 + Pad [2]byte +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) GetFrontendFD() int32 { + return p.FD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) SetFrontendFD(fd int32) { + p.FD = fd +} + +// +marshal +type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT struct { + Type uint32 // enum NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE + Data [12]byte // union +} + +// +marshal +type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS struct { + Object NV0000_CTRL_OS_UNIX_EXPORT_OBJECT + FD int32 + Flags uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) GetFrontendFD() int32 { + return p.FD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) SetFrontendFD(fd int32) { + p.FD = fd +} + +// +marshal +type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS struct { + FD int32 + Object NV0000_CTRL_OS_UNIX_EXPORT_OBJECT +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) GetFrontendFD() int32 { + return p.FD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) SetFrontendFD(fd int32) { + p.FD = fd +} + +// +marshal +type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS struct { + FD int32 + HDevice Handle + MaxObjects uint16 + Metadata [NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE]uint8 + Pad [2]byte + Objects [NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS]Handle + NumObjects uint16 + Index uint16 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) GetFrontendFD() int32 { + return p.FD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) SetFrontendFD(fd int32) { + p.FD = fd +} + +// +marshal +type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS struct { + FD int32 + HParent Handle + Objects [NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS]Handle + ObjectTypes [NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS]uint8 + NumObjects uint16 + Index uint16 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) GetFrontendFD() int32 { + return p.FD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) SetFrontendFD(fd int32) { + p.FD = fd +} + +// +marshal +type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS struct { + SizeOfStrings uint32 + Pad [4]byte + PDriverVersionBuffer P64 + PVersionBuffer P64 + PTitleBuffer P64 + ChangelistNumber uint32 + OfficialChangelistNumber uint32 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080dma.h. +const ( + NV0080_CTRL_CMD_DMA_ADV_SCHED_GET_VA_CAPS = 0x801806 + NV0080_CTRL_CMD_DMA_GET_CAPS = 0x80180d +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0041.h +const ( + NV0041_CTRL_CMD_GET_SURFACE_INFO = 0x410110 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fb.h: +const ( + NV0080_CTRL_CMD_FB_GET_CAPS = 0x801301 + NV0080_CTRL_CMD_FB_GET_CAPS_V2 = 0x801307 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fifo.h: +const ( + NV0080_CTRL_CMD_FIFO_GET_CAPS = 0x801701 + NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES = 0x801707 + NV0080_CTRL_CMD_FIFO_GET_CHANNELLIST = 0x80170d + NV0080_CTRL_CMD_FIFO_GET_CAPS_V2 = 0x801713 +) + +// +marshal +type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS struct { + NumChannels uint32 + Pad [4]byte + PChannelHandleList P64 + PChannelList P64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gpu.h: +const ( + NV0080_CTRL_CMD_GPU_GET_CLASSLIST = 0x800201 + NV0080_CTRL_CMD_GPU_GET_NUM_SUBDEVICES = 0x800280 + NV0080_CTRL_CMD_GPU_QUERY_SW_STATE_PERSISTENCE = 0x800288 + NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE = 0x800289 + NV0080_CTRL_CMD_GPU_GET_CLASSLIST_V2 = 0x800292 +) + +// RmapiParamNvU32List is used to represent the following types: +// - NV0080_CTRL_GPU_GET_CLASSLIST_PARAMS +// - NV2080_CTRL_GPU_GET_ENGINES_PARAMS +// +// +marshal +type RmapiParamNvU32List struct { + NumElems uint32 + Pad [4]byte + List P64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gr.h: +const ( + NV0080_CTRL_CMD_GR_GET_CAPS = 0x801102 + NV0080_CTRL_CMD_GR_GET_INFO = 0x801104 + NV0080_CTRL_CMD_GR_GET_CAPS_V2 = 0x801109 +) + +// NV0080_CTRL_GET_CAPS_PARAMS is used to represent the following: +// - NV0080_CTRL_FB_GET_CAPS_PARAMS +// - NV0080_CTRL_GR_GET_CAPS_PARAMS +// - NV0080_CTRL_FIFO_GET_CAPS_PARAMS +// - NV0080_CTRL_MSENC_GET_CAPS_PARAMS +// +// +marshal +type NV0080_CTRL_GET_CAPS_PARAMS struct { + CapsTblSize uint32 + Pad [4]byte + CapsTbl P64 +} + +// +marshal +type NV0080_CTRL_GR_ROUTE_INFO struct { + Flags uint32 + Pad [4]byte + Route uint64 +} + +const ( + // From src/common/sdk/nvidia/inc/ctrl/ctrl208f/ctrl208fgpu.h + NV208F_CTRL_CMD_GPU_VERIFY_INFOROM = 0x208f1105 +) + +// NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS is used to represent to determine if an InfoROM with a +// valid image is present. +// +// +marshal +type NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS struct { + Result uint32 + Checksum uint32 +} + +// NvxxxCtrlXxxGetInfoParams is used to represent the following: +// - NV0080_CTRL_GR_GET_INFO_PARAMS +// - NV2080_CTRL_FB_GET_INFO_PARAMS +// - NV0041_CTRL_GET_SURFACE_INFO_PARAMS +// - NV2080_CTRL_BIOS_GET_INFO_PARAMS +// - NV2080_CTRL_BUS_GET_INFO_PARAMS +// +// +marshal +type NvxxxCtrlXxxGetInfoParams struct { + InfoListSize uint32 + Pad [4]byte + InfoList P64 +} + +// ListSize implements HasCtrlInfoList.ListSize. +func (p *NvxxxCtrlXxxGetInfoParams) ListSize() uint32 { + return p.InfoListSize +} + +// SetCtrlInfoList implements HasCtrlInfoList.SetCtrlInfoList. +func (p *NvxxxCtrlXxxGetInfoParams) SetCtrlInfoList(ptr P64) { + p.InfoList = ptr +} + +// CtrlInfoList implements HasCtrlInfoList.CtrlInfoList. +func (p *NvxxxCtrlXxxGetInfoParams) CtrlInfoList() P64 { + return p.InfoList +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080host.h: +const ( + NV0080_CTRL_CMD_HOST_GET_CAPS_V2 = 0x801402 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080perf.h: +const ( + NV0080_CTRL_CMD_PERF_CUDA_LIMIT_SET_CONTROL = 0x801909 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080msenc.h: +const ( + NV0080_CTRL_CMD_MSENC_GET_CAPS = 0x801b01 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080bsp.h +const ( + NV0080_CTRL_CMD_BSP_GET_CAPS_V2 = 0x801c02 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080nvjpg.h +const ( + NV0080_CTRL_CMD_NVJPG_GET_CAPS_V2 = 0x801f02 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl00da.h: +const ( + NV_SEMAPHORE_SURFACE_CTRL_CMD_BIND_CHANNEL = 0xda0002 + NV_SEMAPHORE_SURFACE_CTRL_CMD_UNBIND_CHANNEL = 0xda0006 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl00de.h +const ( + NV00DE_CTRL_CMD_REQUEST_DATA_POLL = 0xde0001 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl00f8.h: +const ( + NV00F8_CTRL_CMD_ATTACH_MEM = 0xf80103 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl00fd.h: +const ( + NV00FD_CTRL_CMD_GET_INFO = 0xfd0101 + NV00FD_CTRL_CMD_ATTACH_MEM = 0xfd0102 + NV00FD_CTRL_CMD_ATTACH_GPU = 0xfd0104 + NV00FD_CTRL_CMD_DETACH_MEM = 0xfd0105 +) + +// +marshal +type NV00FD_CTRL_ATTACH_GPU_PARAMS struct { + HSubDevice Handle + Flags uint32 + DevDescriptor uint64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080bios.h: +const ( + NV2080_CTRL_CMD_BIOS_GET_INFO = 0x20800802 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080bus.h: +const ( + NV2080_CTRL_CMD_BUS_GET_PCI_INFO = 0x20801801 + NV2080_CTRL_CMD_BUS_GET_PCI_BAR_INFO = 0x20801803 + NV2080_CTRL_CMD_BUS_GET_INFO = 0x20801802 + NV2080_CTRL_CMD_BUS_GET_INFO_V2 = 0x20801823 + NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS = 0x2080182a + NV2080_CTRL_CMD_BUS_GET_C2C_INFO = 0x2080182b +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ce.h: +const ( + NV2080_CTRL_CMD_CE_GET_CE_PCE_MASK = 0x20802a02 + NV2080_CTRL_CMD_CE_GET_CAPS_V2 = 0x20802a03 + NV2080_CTRL_CMD_CE_GET_ALL_CAPS = 0x20802a0a +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h: +const ( + NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION = 0x20800301 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h: +const ( + NV2080_CTRL_CMD_FB_GET_INFO = 0x20801301 + NV2080_CTRL_CMD_FB_GET_INFO_V2 = 0x20801303 + NV2080_CTRL_CMD_FB_GET_GPU_CACHE_INFO = 0x20801315 + NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO = 0x20801320 + NV2080_CTRL_CMD_FB_GET_SEMAPHORE_SURFACE_LAYOUT = 0x20801352 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT = 0x20801358 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS = 0x20801359 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT_V575 = NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT - 1 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS_V575 = NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS - 1 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fifo.h: +const ( + NV2080_CTRL_CMD_FIFO_DISABLE_CHANNELS = 0x2080110b + + NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES = 64 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080flcn.h: +const ( + NV2080_CTRL_CMD_FLCN_GET_CTX_BUFFER_SIZE = 0x20803125 +) + +// +marshal +type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS struct { + BDisable uint8 + Pad1 [3]byte + NumChannels uint32 + BOnlyDisableScheduling uint8 + BRewindGpPut uint8 + Pad2 [6]byte + PRunlistPreemptEvent P64 + HClientList [NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES]Handle + HChannelList [NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES]Handle +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h: +const ( + NV2080_CTRL_CMD_GPU_GET_INFO_V2 = 0x20800102 + NV2080_CTRL_CMD_GPU_GET_NAME_STRING = 0x20800110 + NV2080_CTRL_CMD_GPU_GET_SHORT_NAME_STRING = 0x20800111 + NV2080_CTRL_CMD_GPU_GET_SIMULATION_INFO = 0x20800119 + NV2080_CTRL_CMD_GPU_GET_ENGINES = 0x20800123 + NV2080_CTRL_CMD_GPU_QUERY_ECC_STATUS = 0x2080012f + NV2080_CTRL_CMD_GPU_QUERY_COMPUTE_MODE_RULES = 0x20800131 + NV2080_CTRL_CMD_GPU_QUERY_ECC_CONFIGURATION = 0x20800133 + NV2080_CTRL_CMD_GPU_GET_OEM_BOARD_INFO = 0x2080013f + NV2080_CTRL_CMD_GPU_GET_ID = 0x20800142 + NV2080_CTRL_CMD_GPU_ACQUIRE_COMPUTE_MODE_RESERVATION = 0x20800145 // undocumented; paramSize == 0 + NV2080_CTRL_CMD_GPU_RELEASE_COMPUTE_MODE_RESERVATION = 0x20800146 // undocumented; paramSize == 0 + NV2080_CTRL_CMD_GPU_GET_ENGINE_PARTNERLIST = 0x20800147 + NV2080_CTRL_CMD_GPU_GET_GID_INFO = 0x2080014a + NV2080_CTRL_CMD_GPU_GET_INFOROM_OBJECT_VERSION = 0x2080014b + NV2080_CTRL_CMD_GPU_GET_INFOROM_IMAGE_VERSION = 0x20800156 + NV2080_CTRL_CMD_GPU_QUERY_INFOROM_ECC_SUPPORT = 0x20800157 + NV2080_CTRL_CMD_GPU_GET_ENCODER_CAPACITY = 0x2080016c + NV2080_CTRL_CMD_GPU_GET_ENGINES_V2 = 0x20800170 + NV2080_CTRL_CMD_GPU_GET_ACTIVE_PARTITION_IDS = 0x2080018b + NV2080_CTRL_CMD_GPU_GET_PIDS = 0x2080018d + NV2080_CTRL_CMD_GPU_GET_PID_INFO = 0x2080018e + NV2080_CTRL_CMD_GPU_GET_COMPUTE_POLICY_CONFIG = 0x20800195 + NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO = 0x208001a3 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gr.h: +const ( + NV2080_CTRL_CMD_GR_GET_INFO = 0x20801201 + NV2080_CTRL_CMD_GR_GET_ZCULL_INFO = 0x20801206 + NV2080_CTRL_CMD_GR_CTXSW_ZCULL_BIND = 0x20801208 + NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE = 0x20801210 + NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_SIZE = 0x20801218 + NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER = 0x2080121b + NV2080_CTRL_CMD_GR_GET_CAPS_V2 = 0x20801227 + NV2080_CTRL_CMD_GR_GET_GPC_MASK = 0x2080122a + NV2080_CTRL_CMD_GR_GET_TPC_MASK = 0x2080122b + NV2080_CTRL_CMD_GR_GET_SM_ISSUE_RATE_MODIFIER = 0x20801230 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080grmgr.h: +const ( + NV2080_CTRL_CMD_GRMGR_GET_GR_FS_INFO = 0x20803801 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gsp.h: +const ( + NV2080_CTRL_CMD_GSP_GET_FEATURES = 0x20803601 +) + +// +marshal +type NV2080_CTRL_GR_GET_INFO_PARAMS struct { + NvxxxCtrlXxxGetInfoParams + GRRouteInfo NV0080_CTRL_GR_ROUTE_INFO +} + +// ListSize implements HasCtrlInfoList.ListSize. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) ListSize() uint32 { + return p.InfoListSize +} + +// SetCtrlInfoList implements HasCtrlInfoList.SetCtrlInfoList. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) SetCtrlInfoList(ptr P64) { + p.InfoList = ptr +} + +// CtrlInfoList implements HasCtrlInfoList.CtrlInfoList. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) CtrlInfoList() P64 { + return p.InfoList +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080mc.h: +const ( + NV2080_CTRL_CMD_MC_GET_ARCH_INFO = 0x20801701 + NV2080_CTRL_CMD_MC_SERVICE_INTERRUPTS = 0x20801702 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080nvlink.h: +const ( + NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS = 0x20803001 + NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS = 0x20803002 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080perf.h: +const ( + NV2080_CTRL_CMD_PERF_BOOST = 0x2080200a + NV2080_CTRL_CMD_PERF_GET_CURRENT_PSTATE = 0x20802068 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080rc.h: +const ( + NV2080_CTRL_CMD_RC_GET_WATCHDOG_INFO = 0x20802209 + NV2080_CTRL_CMD_RC_RELEASE_WATCHDOG_REQUESTS = 0x2080220c + NV2080_CTRL_CMD_RC_SOFT_DISABLE_WATCHDOG = 0x20802210 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080tmr.h: +const ( + NV2080_CTRL_CMD_TIMER_GET_TIME = 0x20800403 + NV2080_CTRL_CMD_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO = 0x20800406 + NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ = 0x20800407 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080thermal.h: +const ( + NV2080_CTRL_CMD_THERMAL_SYSTEM_EXECUTE_V2 = 0x20800513 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080unix.h: +const ( + NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS = 0x20803d07 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl503c.h: +const ( + NV503C_CTRL_CMD_REGISTER_VA_SPACE = 0x503c0102 + NV503C_CTRL_CMD_REGISTER_VIDMEM = 0x503c0104 + NV503C_CTRL_CMD_UNREGISTER_VIDMEM = 0x503c0105 +) + +// +marshal +type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS struct { + HVASpace Handle + Pad [4]byte + VASpaceToken uint64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl83de/ctrl83dedebug.h: +const ( + NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK = 0x83de0309 + NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES = 0x83de030c + NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES = 0x83de0310 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlc36f.h: +const ( + NVC36F_CTRL_GET_CLASS_ENGINEID = 0xc36f0101 + NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN = 0xc36f0108 + NVC36F_CTRL_CMD_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX = 0xc36f010a +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlc56f.h: +const ( + NVC56F_CTRL_CMD_GET_KMB = 0xc56f010b +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl906f.h: +const ( + NV906F_CTRL_GET_CLASS_ENGINEID = 0x906f0101 + NV906F_CTRL_CMD_RESET_CHANNEL = 0x906f0102 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl9096.h: +const ( + NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE = 0x90960106 + NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_ENTRY = 0x90960107 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl90e6.h: +const ( + NV90E6_CTRL_CMD_MASTER_GET_VIRTUAL_FUNCTION_ERROR_CONT_INTR_MASK = 0x90e60102 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrla06c.h: +const ( + NVA06C_CTRL_CMD_GPFIFO_SCHEDULE = 0xa06c0101 + NVA06C_CTRL_CMD_SET_TIMESLICE = 0xa06c0103 + NVA06C_CTRL_CMD_PREEMPT = 0xa06c0105 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrla06f/ctrla06fgpfifo.h: +const ( + NVA06F_CTRL_CMD_GPFIFO_SCHEDULE = 0xa06f0103 + NVA06F_CTRL_CMD_BIND = 0xa06f0104 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlcb33.h: +const ( + NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES = 0xcb330101 + NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_GPUS_STATE = 0xcb330104 + NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS = 0xcb33010b + NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE = 0xcb33010c +) + +// The following IOCTLs are not found in the OSS repo. +const ( + UNKNOWN_CONTROL_COMMAND_80028B = 0x80028b +) diff --git a/pkg/abi/nvgpu/frontend.go b/pkg/abi/nvgpu/frontend.go new file mode 100644 index 0000000..f857392 --- /dev/null +++ b/pkg/abi/nvgpu/frontend.go @@ -0,0 +1,805 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +import ( + "github.com/sagernet/gvisor/pkg/marshal" +) + +// NV_IOCTL_MAGIC is the "canonical" IOC_TYPE for frontend ioctls. +// The driver ignores IOC_TYPE, allowing any value to be passed. +const NV_IOCTL_MAGIC = uint32('F') + +// Frontend ioctl numbers. +// Note that these are only the IOC_NR part of the ioctl command. +const ( + // From kernel-open/common/inc/nv-ioctl-numbers.h: + NV_IOCTL_BASE = 200 + NV_ESC_CARD_INFO = NV_IOCTL_BASE + 0 + NV_ESC_REGISTER_FD = NV_IOCTL_BASE + 1 + NV_ESC_ALLOC_OS_EVENT = NV_IOCTL_BASE + 6 + NV_ESC_FREE_OS_EVENT = NV_IOCTL_BASE + 7 + NV_ESC_CHECK_VERSION_STR = NV_IOCTL_BASE + 10 + NV_ESC_ATTACH_GPUS_TO_FD = NV_IOCTL_BASE + 12 + NV_ESC_SYS_PARAMS = NV_IOCTL_BASE + 14 + NV_ESC_WAIT_OPEN_COMPLETE = NV_IOCTL_BASE + 18 + + // From kernel-open/common/inc/nv-ioctl-numa.h: + NV_ESC_NUMA_INFO = NV_IOCTL_BASE + 15 + + // From src/nvidia/arch/nvalloc/unix/include/nv_escape.h: + NV_ESC_RM_ALLOC_MEMORY = 0x27 + NV_ESC_RM_FREE = 0x29 + NV_ESC_RM_CONTROL = 0x2a + NV_ESC_RM_ALLOC = 0x2b + NV_ESC_RM_DUP_OBJECT = 0x34 + NV_ESC_RM_SHARE = 0x35 + NV_ESC_RM_IDLE_CHANNELS = 0x41 + NV_ESC_RM_VID_HEAP_CONTROL = 0x4a + NV_ESC_RM_MAP_MEMORY = 0x4e + NV_ESC_RM_UNMAP_MEMORY = 0x4f + NV_ESC_RM_ALLOC_CONTEXT_DMA2 = 0x54 + NV_ESC_RM_MAP_MEMORY_DMA = 0x57 + NV_ESC_RM_UNMAP_MEMORY_DMA = 0x58 + NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO = 0x5e +) + +// Frontend ioctl parameter structs, from src/common/sdk/nvidia/inc/nvos.h or +// kernel-open/common/inc/nv-ioctl.h. + +// IoctlRegisterFD is the parameter type for NV_ESC_REGISTER_FD. +// +// +marshal +type IoctlRegisterFD struct { + CtlFD int32 +} + +// IoctlAllocOSEvent is the parameter type for NV_ESC_ALLOC_OS_EVENT. +// +// +marshal +type IoctlAllocOSEvent struct { + HClient Handle + HDevice Handle + FD uint32 + Status uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlAllocOSEvent) GetFrontendFD() int32 { + return int32(p.FD) +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlAllocOSEvent) SetFrontendFD(fd int32) { + p.FD = uint32(fd) +} + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlAllocOSEvent) GetStatus() uint32 { + return p.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlAllocOSEvent) SetStatus(status uint32) { + p.Status = status +} + +// IoctlFreeOSEvent is the parameter type for NV_ESC_FREE_OS_EVENT. +// +// +marshal +type IoctlFreeOSEvent struct { + HClient Handle + HDevice Handle + FD uint32 + Status uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlFreeOSEvent) GetFrontendFD() int32 { + return int32(p.FD) +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlFreeOSEvent) SetFrontendFD(fd int32) { + p.FD = uint32(fd) +} + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlFreeOSEvent) GetStatus() uint32 { + return p.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlFreeOSEvent) SetStatus(status uint32) { + p.Status = status +} + +// RMAPIVersion is the parameter type for NV_ESC_CHECK_VERSION_STR. +// +// +marshal +type RMAPIVersion struct { + Cmd uint32 + Reply uint32 + VersionString [64]byte +} + +// IoctlSysParams is the parameter type for NV_ESC_SYS_PARAMS. +// +// +marshal +type IoctlSysParams struct { + MemblockSize uint64 +} + +// IoctlWaitOpenComplete is the parameter type for NV_ESC_WAIT_OPEN_COMPLETE. +// +// +marshal +type IoctlWaitOpenComplete struct { + Rc int32 + AdapterStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlWaitOpenComplete) GetStatus() uint32 { + return p.AdapterStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlWaitOpenComplete) SetStatus(status uint32) { + p.AdapterStatus = status +} + +// IoctlNVOS02ParametersWithFD is the parameter type for NV_ESC_RM_ALLOC_MEMORY. +// +// +marshal +type IoctlNVOS02ParametersWithFD struct { + Params NVOS02_PARAMETERS + FD int32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlNVOS02ParametersWithFD) GetStatus() uint32 { + return p.Params.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlNVOS02ParametersWithFD) SetStatus(status uint32) { + p.Params.Status = status +} + +// +marshal +type NVOS02_PARAMETERS struct { + HRoot Handle + HObjectParent Handle + HObjectNew Handle + HClass ClassID + Flags uint32 + Pad0 [4]byte + PMemory P64 // address of application mapping, without indirection + Limit uint64 + Status uint32 + Pad1 [4]byte +} + +// Bitfields in NVOS02_PARAMETERS.Flags: +const ( + NVOS02_FLAGS_ALLOC_SHIFT = 16 + NVOS02_FLAGS_ALLOC_MASK = 0x3 + NVOS02_FLAGS_ALLOC_NONE = 0x00000001 + + NVOS02_FLAGS_MAPPING_SHIFT = 30 + NVOS02_FLAGS_MAPPING_MASK = 0x3 + NVOS02_FLAGS_MAPPING_NO_MAP = 0x00000001 +) + +// NVOS00_PARAMETERS is the parameter type for NV_ESC_RM_FREE. +// +// +marshal +type NVOS00_PARAMETERS struct { + HRoot Handle + HObjectParent Handle + HObjectOld Handle + Status uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *NVOS00_PARAMETERS) GetStatus() uint32 { + return p.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (p *NVOS00_PARAMETERS) SetStatus(status uint32) { + p.Status = status +} + +// RmAllocParamType should be implemented by all possible parameter types for +// NV_ESC_RM_ALLOC. +type RmAllocParamType interface { + GetHClass() ClassID + GetPAllocParms() P64 + GetPRightsRequested() P64 + SetPAllocParms(p P64) + SetPRightsRequested(p P64) + FromOS64(other NVOS64_PARAMETERS) + ToOS64() NVOS64_PARAMETERS + GetPointer() uintptr + HasStatus + marshal.Marshallable +} + +// GetRmAllocParamObj returns the appropriate implementation of +// RmAllocParamType based on passed parameters. +func GetRmAllocParamObj(isNVOS64 bool) RmAllocParamType { + if isNVOS64 { + return &NVOS64_PARAMETERS{} + } + return &NVOS21_PARAMETERS{} +} + +// NVOS21_PARAMETERS is one possible parameter type for NV_ESC_RM_ALLOC. +// +// +marshal +type NVOS21_PARAMETERS struct { + HRoot Handle + HObjectParent Handle + HObjectNew Handle + HClass ClassID + PAllocParms P64 + ParamsSize uint32 + Status uint32 +} + +// GetHClass implements RmAllocParamType.GetHClass. +func (n *NVOS21_PARAMETERS) GetHClass() ClassID { + return n.HClass +} + +// GetPAllocParms implements RmAllocParamType.GetPAllocParms. +func (n *NVOS21_PARAMETERS) GetPAllocParms() P64 { + return n.PAllocParms +} + +// GetPRightsRequested implements RmAllocParamType.GetPRightsRequested. +func (n *NVOS21_PARAMETERS) GetPRightsRequested() P64 { + return 0 +} + +// SetPAllocParms implements RmAllocParamType.SetPAllocParms. +func (n *NVOS21_PARAMETERS) SetPAllocParms(p P64) { n.PAllocParms = p } + +// SetPRightsRequested implements RmAllocParamType.SetPRightsRequested. +func (n *NVOS21_PARAMETERS) SetPRightsRequested(p P64) { + panic("impossible") +} + +// FromOS64 implements RmAllocParamType.FromOS64. +func (n *NVOS21_PARAMETERS) FromOS64(other NVOS64_PARAMETERS) { + n.HRoot = other.HRoot + n.HObjectParent = other.HObjectParent + n.HObjectNew = other.HObjectNew + n.HClass = other.HClass + n.PAllocParms = other.PAllocParms + n.ParamsSize = other.ParamsSize + n.Status = other.Status +} + +// ToOS64 implements RmAllocParamType.ToOS64. +func (n *NVOS21_PARAMETERS) ToOS64() NVOS64_PARAMETERS { + return NVOS64_PARAMETERS{ + HRoot: n.HRoot, + HObjectParent: n.HObjectParent, + HObjectNew: n.HObjectNew, + HClass: n.HClass, + PAllocParms: n.PAllocParms, + ParamsSize: n.ParamsSize, + Status: n.Status, + } +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS21_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS21_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS55_PARAMETERS is the parameter type for NV_ESC_RM_DUP_OBJECT. +// +// +marshal +type NVOS55_PARAMETERS struct { + HClient Handle + HParent Handle + HObject Handle + HClientSrc Handle + HObjectSrc Handle + Flags uint32 + Status uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS55_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS55_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS57_PARAMETERS is the parameter type for NV_ESC_RM_SHARE. +// +// +marshal +type NVOS57_PARAMETERS struct { + HClient Handle + HObject Handle + SharePolicy RS_SHARE_POLICY + Status uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS57_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS57_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS30_PARAMETERS is the parameter type for NV_ESC_RM_IDLE_CHANNELS. +// +// +marshal +type NVOS30_PARAMETERS struct { + Client Handle + Device Handle + Channel Handle + NumChannels uint32 + + Clients P64 + Devices P64 + Channels P64 + + Flags uint32 + Timeout uint32 + Status uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS30_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS30_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS32_PARAMETERS is the parameter type for NV_ESC_RM_VID_HEAP_CONTROL. +// +// +marshal +type NVOS32_PARAMETERS struct { + HRoot Handle + HObjectParent Handle + Function uint32 + HVASpace Handle + IVCHeapNumber int16 + Pad [2]byte + Status uint32 + Total uint64 + Free uint64 + Data [144]byte // union +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS32_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS32_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// Possible values for NVOS32Parameters.Function: +const ( + NVOS32_FUNCTION_ALLOC_SIZE = 2 +) + +// NVOS32AllocSize is the type of NVOS32Parameters.Data for +// NVOS32_FUNCTION_ALLOC_SIZE. +type NVOS32AllocSize struct { + Owner uint32 + HMemory Handle + Type uint32 + Flags uint32 + Attr uint32 + Format uint32 + ComprCovg uint32 + ZcullCovg uint32 + PartitionStride uint32 + Width uint32 + Height uint32 + Pad0 [4]byte + Size uint64 + Alignment uint64 + Offset uint64 + Limit uint64 + Address P64 + RangeBegin uint64 + RangeEnd uint64 + Attr2 uint32 + CtagOffset uint32 +} + +// Flags in NVOS32AllocSize.Flags: +const ( + NVOS32_ALLOC_FLAGS_VIRTUAL = 0x00080000 +) + +// Bitfields in NVOS32AllocSize.Attr: +const ( + NVOS32_ATTR_LOCATION_SHIFT = 25 + NVOS32_ATTR_LOCATION_MASK = 0x3 + NVOS32_ATTR_LOCATION_VIDMEM = 0 +) + +// Bitfields in NVOS32AllocSize.Attr2: +const ( + NVOS32_ATTR2_USE_EGM_SHIFT = 24 + NVOS32_ATTR2_USE_EGM_MASK = 0x1 + NVOS32_ATTR2_USE_EGM_FALSE = 0 + NVOS32_ATTR2_USE_EGM_TRUE = 1 +) + +// IoctlNVOS33ParametersWithFD is the parameter type for NV_ESC_RM_MAP_MEMORY, +// from src/nvidia/arch/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h. +// +// +marshal +type IoctlNVOS33ParametersWithFD struct { + Params NVOS33_PARAMETERS + FD int32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlNVOS33ParametersWithFD) GetStatus() uint32 { + return p.Params.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlNVOS33ParametersWithFD) SetStatus(status uint32) { + p.Params.Status = status +} + +// +marshal +type NVOS33_PARAMETERS struct { + HClient Handle + HDevice Handle + HMemory Handle + Pad0 [4]byte + Offset uint64 + Length uint64 + PLinearAddress P64 // address of application mapping, without indirection + Status uint32 + Flags uint32 +} + +// Bitfields in NVOS33_PARAMETERS.Flags: +const ( + NVOS33_FLAGS_CACHING_TYPE_SHIFT = 23 + NVOS33_FLAGS_CACHING_TYPE_MASK = 0x7 + NVOS33_FLAGS_CACHING_TYPE_CACHED = 0 + NVOS33_FLAGS_CACHING_TYPE_UNCACHED = 1 + NVOS33_FLAGS_CACHING_TYPE_WRITECOMBINED = 2 + NVOS33_FLAGS_CACHING_TYPE_WRITEBACK = 5 + NVOS33_FLAGS_CACHING_TYPE_DEFAULT = 6 + NVOS33_FLAGS_CACHING_TYPE_UNCACHED_WEAK = 7 +) + +// NVOS34_PARAMETERS is the parameter type for NV_ESC_RM_UNMAP_MEMORY. +// +// +marshal +type NVOS34_PARAMETERS struct { + HClient Handle + HDevice Handle + HMemory Handle + Pad0 [4]byte + PLinearAddress P64 // address of application mapping, without indirection + Status uint32 + Flags uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS34_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS34_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS39_PARAMETERS is the parameter type for NV_ESC_RM_ALLOC_CONTEXT_DMA2. +// +// +marshal +type NVOS39_PARAMETERS struct { + HObjectParent Handle + HSubDevice Handle + HObjectNew Handle + HClass ClassID + Flags uint32 + Selector uint32 + HMemory Handle + Pad0 [4]byte + Offset uint64 + Limit uint64 + Status uint32 + Pad1 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS39_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS39_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS46_PARAMETERS is the parameter type for NV_ESC_RM_MAP_MEMORY_DMA. +// +// +marshal +type NVOS46_PARAMETERS struct { + Client Handle + Device Handle + Dma Handle + Memory Handle + Offset uint64 + Length uint64 + Flags uint32 + Pad0 [4]byte + DmaOffset uint64 + Status uint32 + Pad1 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS46_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS46_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS46_PARAMETERS_V580 is the updated version of NVOS46_PARAMETERS since +// 580.65.06. +// +// +marshal +type NVOS46_PARAMETERS_V580 struct { + Client Handle + Device Handle + Dma Handle + Memory Handle + Offset uint64 + Length uint64 + Flags uint32 + Flags2 uint32 + KindOverride uint32 + Pad0 [4]byte + DmaOffset uint64 + Status uint32 + Pad1 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS46_PARAMETERS_V580) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS46_PARAMETERS_V580) SetStatus(status uint32) { + n.Status = status +} + +// NVOS47_PARAMETERS is the parameter type for NV_ESC_RM_UNMAP_MEMORY_DMA. +// +// +marshal +type NVOS47_PARAMETERS struct { + Client Handle + Device Handle + Dma Handle + Memory Handle + Flags uint32 + Pad0 [4]byte + DmaOffset uint64 + Status uint32 + Pad1 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS47_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS47_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS47_PARAMETERS_V550 is the updated version of NVOS47_PARAMETERS since +// 550.54.04. +// +// +marshal +type NVOS47_PARAMETERS_V550 struct { + Client Handle + Device Handle + Dma Handle + Memory Handle + Flags uint32 + Pad0 [4]byte + DmaOffset uint64 + Size uint64 + Status uint32 + Pad1 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS47_PARAMETERS_V550) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS47_PARAMETERS_V550) SetStatus(status uint32) { + n.Status = status +} + +// NVOS54_PARAMETERS is the parameter type for NV_ESC_RM_CONTROL. +// +// +marshal +type NVOS54_PARAMETERS struct { + HClient Handle + HObject Handle + Cmd uint32 + Flags uint32 + Params P64 + ParamsSize uint32 + Status uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS54_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS54_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS56_PARAMETERS is the parameter type for NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO. +// +// +marshal +type NVOS56_PARAMETERS struct { + HClient Handle + HDevice Handle + HMemory Handle + Pad0 [4]byte + POldCPUAddress P64 + PNewCPUAddress P64 + Status uint32 + Pad1 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS56_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS56_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// NVOS64_PARAMETERS is one possible parameter type for NV_ESC_RM_ALLOC. +// +// +marshal +// +stateify savable +type NVOS64_PARAMETERS struct { + HRoot Handle + HObjectParent Handle + HObjectNew Handle + HClass ClassID + PAllocParms P64 + PRightsRequested P64 + ParamsSize uint32 + Flags uint32 + Status uint32 + _ uint32 +} + +// GetHClass implements RmAllocParamType.GetHClass. +func (n *NVOS64_PARAMETERS) GetHClass() ClassID { + return n.HClass +} + +// GetPAllocParms implements RmAllocParamType.GetPAllocParms. +func (n *NVOS64_PARAMETERS) GetPAllocParms() P64 { + return n.PAllocParms +} + +// GetPRightsRequested implements RmAllocParamType.GetPRightsRequested. +func (n *NVOS64_PARAMETERS) GetPRightsRequested() P64 { + return n.PRightsRequested +} + +// SetPAllocParms implements RmAllocParamType.SetPAllocParms. +func (n *NVOS64_PARAMETERS) SetPAllocParms(p P64) { n.PAllocParms = p } + +// SetPRightsRequested implements RmAllocParamType.SetPRightsRequested. +func (n *NVOS64_PARAMETERS) SetPRightsRequested(p P64) { n.PRightsRequested = p } + +// FromOS64 implements RmAllocParamType.FromOS64. +func (n *NVOS64_PARAMETERS) FromOS64(other NVOS64_PARAMETERS) { *n = other } + +// ToOS64 implements RmAllocParamType.ToOS64. +func (n *NVOS64_PARAMETERS) ToOS64() NVOS64_PARAMETERS { return *n } + +// GetStatus implements HasStatus.GetStatus. +func (n *NVOS64_PARAMETERS) GetStatus() uint32 { + return n.Status +} + +// SetStatus implements HasStatus.SetStatus. +func (n *NVOS64_PARAMETERS) SetStatus(status uint32) { + n.Status = status +} + +// HasFrontendFD is a type constraint for parameter structs containing a +// frontend FD field. This is necessary because, as of this writing (Go 1.20), +// there is no way to enable field access using a Go type constraint. +type HasFrontendFD interface { + GetFrontendFD() int32 + SetFrontendFD(int32) +} + +// Frontend ioctl parameter struct sizes. +var ( + SizeofIoctlRegisterFD = uint32((*IoctlRegisterFD)(nil).SizeBytes()) + SizeofIoctlAllocOSEvent = uint32((*IoctlAllocOSEvent)(nil).SizeBytes()) + SizeofIoctlFreeOSEvent = uint32((*IoctlFreeOSEvent)(nil).SizeBytes()) + SizeofRMAPIVersion = uint32((*RMAPIVersion)(nil).SizeBytes()) + SizeofIoctlSysParams = uint32((*IoctlSysParams)(nil).SizeBytes()) + SizeofIoctlWaitOpenComplete = uint32((*IoctlWaitOpenComplete)(nil).SizeBytes()) + SizeofIoctlNVOS02ParametersWithFD = uint32((*IoctlNVOS02ParametersWithFD)(nil).SizeBytes()) + SizeofNVOS00Parameters = uint32((*NVOS00_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS21Parameters = uint32((*NVOS21_PARAMETERS)(nil).SizeBytes()) + SizeofIoctlNVOS33ParametersWithFD = uint32((*IoctlNVOS33ParametersWithFD)(nil).SizeBytes()) + SizeofNVOS30Parameters = uint32((*NVOS30_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS32Parameters = uint32((*NVOS32_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS34Parameters = uint32((*NVOS34_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS39Parameters = uint32((*NVOS39_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS54Parameters = uint32((*NVOS54_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS55Parameters = uint32((*NVOS55_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS56Parameters = uint32((*NVOS56_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS57Parameters = uint32((*NVOS57_PARAMETERS)(nil).SizeBytes()) + SizeofNVOS64Parameters = uint32((*NVOS64_PARAMETERS)(nil).SizeBytes()) +) diff --git a/pkg/abi/nvgpu/frontend_unsafe.go b/pkg/abi/nvgpu/frontend_unsafe.go new file mode 100644 index 0000000..7e0f244 --- /dev/null +++ b/pkg/abi/nvgpu/frontend_unsafe.go @@ -0,0 +1,27 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +import "unsafe" + +// GetPointer implements RmAllocParamType.GetPointer. +func (n *NVOS21_PARAMETERS) GetPointer() uintptr { + return uintptr(unsafe.Pointer(n)) +} + +// GetPointer implements RmAllocParamType.GetPointer. +func (n *NVOS64_PARAMETERS) GetPointer() uintptr { + return uintptr(unsafe.Pointer(n)) +} diff --git a/pkg/abi/nvgpu/nvgpu.go b/pkg/abi/nvgpu/nvgpu.go new file mode 100644 index 0000000..5d4df11 --- /dev/null +++ b/pkg/abi/nvgpu/nvgpu.go @@ -0,0 +1,92 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nvgpu tracks the ABI of the Nvidia GPU Linux kernel driver: +// https://github.com/NVIDIA/open-gpu-kernel-modules +package nvgpu + +import ( + "fmt" +) + +// Device numbers. +const ( + NV_MAJOR_DEVICE_NUMBER = 195 // from kernel-open/common/inc/nv.h + NV_CONTROL_DEVICE_MINOR = 255 // from kernel-open/common/inc/nv-linux.h + NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0 // from kernel-open/nvidia-uvm/uvm_common.h +) + +// Handle is NvHandle, from src/common/sdk/nvidia/inc/nvtypes.h. +// +// +marshal +// +stateify savable +type Handle struct { + Val uint32 +} + +// String implements fmt.Stringer.String. +func (h Handle) String() string { + return fmt.Sprintf("%#x", h.Val) +} + +// P64 is NvP64, from src/common/sdk/nvidia/inc/nvtypes.h. +// +// +marshal +type P64 uint64 + +// From src/common/sdk/nvidia/inc/nvlimits.h: +const ( + NV_MAX_DEVICES = 32 + NV_MAX_SUBDEVICES = 8 +) + +// From src/common/sdk/nvidia/inc/alloc/alloc_channel.h. +const ( + CC_CHAN_ALLOC_IV_SIZE_DWORD = 3 + CC_CHAN_ALLOC_NONCE_SIZE_DWORD = 8 +) + +// RS_ACCESS_MASK is RS_ACCESS_MASK, from +// src/common/sdk/nvidia/inc/rs_access.h. +// +// +marshal +// +stateify savable +type RS_ACCESS_MASK struct { + Limbs [SDK_RS_ACCESS_MAX_LIMBS]uint32 // RsAccessLimb +} + +const SDK_RS_ACCESS_MAX_LIMBS = 1 + +// RS_SHARE_POLICY is RS_SHARE_POLICY, from +// src/common/sdk/nvidia/inc/rs_access.h. +// +// +marshal +type RS_SHARE_POLICY struct { + Target uint32 + AccessMask RS_ACCESS_MASK + Type uint16 + Action uint8 + Pad [1]byte +} + +// NvUUID is defined in src/common/inc/nvCpuUuid.h. +// +// +marshal +type NvUUID [16]uint8 + +// HasStatus is an interface for parameter structs that have a Status field. +type HasStatus interface { + GetStatus() uint32 + SetStatus(status uint32) +} diff --git a/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go b/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go new file mode 100644 index 0000000..f3f9923 --- /dev/null +++ b/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go @@ -0,0 +1,17491 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +package nvgpu + +import ( + "io" + "reflect" + "runtime" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Marshallable types used by this file. +var ( + _ marshal.Marshallable = (*ClassID)(nil) + _ marshal.Marshallable = (*Handle)(nil) + _ marshal.Marshallable = (*IoctlAllocOSEvent)(nil) + _ marshal.Marshallable = (*IoctlFreeOSEvent)(nil) + _ marshal.Marshallable = (*IoctlNVOS02ParametersWithFD)(nil) + _ marshal.Marshallable = (*IoctlNVOS33ParametersWithFD)(nil) + _ marshal.Marshallable = (*IoctlRegisterFD)(nil) + _ marshal.Marshallable = (*IoctlSysParams)(nil) + _ marshal.Marshallable = (*IoctlWaitOpenComplete)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_GPU_GET_ID_INFO_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECT)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550)(nil) + _ marshal.Marshallable = (*NV0005_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV0080_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS)(nil) + _ marshal.Marshallable = (*NV0080_CTRL_GET_CAPS_PARAMS)(nil) + _ marshal.Marshallable = (*NV0080_CTRL_GR_ROUTE_INFO)(nil) + _ marshal.Marshallable = (*NV00DE_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV00DE_ALLOC_PARAMETERS_V545)(nil) + _ marshal.Marshallable = (*NV00F8_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV00FD_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV00FD_ALLOCATION_PARAMETERS_V545)(nil) + _ marshal.Marshallable = (*NV00FD_CTRL_ATTACH_GPU_PARAMS)(nil) + _ marshal.Marshallable = (*NV2080_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS)(nil) + _ marshal.Marshallable = (*NV2080_CTRL_GR_GET_INFO_PARAMS)(nil) + _ marshal.Marshallable = (*NV2081_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS)(nil) + _ marshal.Marshallable = (*NV503B_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV503B_BAR1_P2P_DMA_INFO)(nil) + _ marshal.Marshallable = (*NV503C_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV503C_CTRL_REGISTER_VA_SPACE_PARAMS)(nil) + _ marshal.Marshallable = (*NV83DE_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV9072_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVA0BC_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVB0B5_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS00_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS02_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS21_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS30_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS32_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS33_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS34_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS39_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS46_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS46_PARAMETERS_V580)(nil) + _ marshal.Marshallable = (*NVOS47_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS47_PARAMETERS_V550)(nil) + _ marshal.Marshallable = (*NVOS54_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS55_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS56_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS57_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVOS64_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVXXXX_CTRL_XXX_INFO)(nil) + _ marshal.Marshallable = (*NV_BSP_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_CHANNEL_ALLOC_PARAMS)(nil) + _ marshal.Marshallable = (*NV_CHANNEL_ALLOC_PARAMS_V570)(nil) + _ marshal.Marshallable = (*NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS)(nil) + _ marshal.Marshallable = (*NV_CONTEXT_DMA_ALLOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*NV_CTXSHARE_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_EXPORT_MEM_PACKET)(nil) + _ marshal.Marshallable = (*NV_GR_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_HOPPER_USERMODE_A_PARAMS)(nil) + _ marshal.Marshallable = (*NV_MEMORY_ALLOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*NV_MEMORY_ALLOCATION_PARAMS_V545)(nil) + _ marshal.Marshallable = (*NV_MEMORY_DESC_PARAMS)(nil) + _ marshal.Marshallable = (*NV_MEMORY_MAPPER_ALLOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550)(nil) + _ marshal.Marshallable = (*NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555)(nil) + _ marshal.Marshallable = (*NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*NV_MSENC_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_VASPACE_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_VASPACE_ALLOCATION_PARAMETERS_V580)(nil) + _ marshal.Marshallable = (*NvUUID)(nil) + _ marshal.Marshallable = (*NvxxxCtrlXxxGetInfoParams)(nil) + _ marshal.Marshallable = (*P64)(nil) + _ marshal.Marshallable = (*RMAPIVersion)(nil) + _ marshal.Marshallable = (*RS_ACCESS_MASK)(nil) + _ marshal.Marshallable = (*RS_SHARE_POLICY)(nil) + _ marshal.Marshallable = (*RmapiParamNvU32List)(nil) + _ marshal.Marshallable = (*UVM_ALLOC_SEMAPHORE_POOL_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550)(nil) + _ marshal.Marshallable = (*UVM_CREATE_EXTERNAL_RANGE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_CREATE_RANGE_GROUP_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_DESTROY_RANGE_GROUP_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_DISABLE_PEER_ACCESS_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_DISABLE_READ_DUPLICATION_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_ENABLE_PEER_ACCESS_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_FREE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_INITIALIZE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_MAP_EXTERNAL_ALLOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550)(nil) + _ marshal.Marshallable = (*UVM_MIGRATE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_MIGRATE_PARAMS_V550)(nil) + _ marshal.Marshallable = (*UVM_MIGRATE_RANGE_GROUP_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_MM_INITIALIZE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_PAGEABLE_MEM_ACCESS_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_REGISTER_CHANNEL_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_REGISTER_GPU_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_REGISTER_GPU_VASPACE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_SET_PREFERRED_LOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_SET_PREFERRED_LOCATION_PARAMS_V550)(nil) + _ marshal.Marshallable = (*UVM_SET_RANGE_GROUP_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNMAP_EXTERNAL_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNREGISTER_CHANNEL_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNREGISTER_GPU_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNREGISTER_GPU_VASPACE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNSET_ACCESSED_BY_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNSET_PREFERRED_LOCATION_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_VALIDATE_VA_RANGE_PARAMS)(nil) + _ marshal.Marshallable = (*UvmGpuMappingAttributes)(nil) + _ marshal.Marshallable = (*nv00f8Map)(nil) +) + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (id *ClassID) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (id *ClassID) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*id)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (id *ClassID) UnmarshalBytes(src []byte) []byte { + *id = ClassID(uint32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (id *ClassID) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (id *ClassID) MarshalUnsafe(dst []byte) []byte { + size := id.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(id), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (id *ClassID) UnmarshalUnsafe(src []byte) []byte { + size := id.SizeBytes() + gohacks.Memmove(unsafe.Pointer(id), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (id *ClassID) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(id))) + hdr.Len = id.SizeBytes() + hdr.Cap = id.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that id + // must live until the use above. + runtime.KeepAlive(id) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (id *ClassID) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return id.CopyOutN(cc, addr, id.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (id *ClassID) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(id))) + hdr.Len = id.SizeBytes() + hdr.Cap = id.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that id + // must live until the use above. + runtime.KeepAlive(id) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (id *ClassID) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return id.CopyInN(cc, addr, id.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (id *ClassID) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(id))) + hdr.Len = id.SizeBytes() + hdr.Cap = id.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that id + // must live until the use above. + runtime.KeepAlive(id) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0005_ALLOC_PARAMETERS) SizeBytes() int { + return 4 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*ClassID)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0005_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HParentClient.MarshalUnsafe(dst) + dst = n.HSrcResource.MarshalUnsafe(dst) + dst = n.HClass.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NotifyIndex)) + dst = dst[4:] + dst = n.Data.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0005_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HParentClient.UnmarshalUnsafe(src) + src = n.HSrcResource.UnmarshalUnsafe(src) + src = n.HClass.UnmarshalUnsafe(src) + n.NotifyIndex = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.Data.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0005_ALLOC_PARAMETERS) Packed() bool { + return n.Data.Packed() && n.HClass.Packed() && n.HParentClient.Packed() && n.HSrcResource.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0005_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.Data.Packed() && n.HClass.Packed() && n.HParentClient.Packed() && n.HSrcResource.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0005_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0005_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.Data.Packed() && n.HClass.Packed() && n.HParentClient.Packed() && n.HSrcResource.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0005_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0005_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Data.Packed() && n.HClass.Packed() && n.HParentClient.Packed() && n.HSrcResource.Packed() { + // Type NV0005_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0005_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0005_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Data.Packed() && n.HClass.Packed() && n.HParentClient.Packed() && n.HSrcResource.Packed() { + // Type NV0005_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0005_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0005_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.Data.Packed() && n.HClass.Packed() && n.HParentClient.Packed() && n.HSrcResource.Packed() { + // Type NV0005_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0080_ALLOC_PARAMETERS) SizeBytes() int { + return 36 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0080_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.DeviceID)) + dst = dst[4:] + dst = n.HClientShare.MarshalUnsafe(dst) + dst = n.HTargetClient.MarshalUnsafe(dst) + dst = n.HTargetDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VASpaceSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VAStartInternal)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VALimitInternal)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.VAMode)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0080_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.DeviceID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HClientShare.UnmarshalUnsafe(src) + src = n.HTargetClient.UnmarshalUnsafe(src) + src = n.HTargetDevice.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.VASpaceSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.VAStartInternal = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.VALimitInternal = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.VAMode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0080_ALLOC_PARAMETERS) Packed() bool { + return n.HClientShare.Packed() && n.HTargetClient.Packed() && n.HTargetDevice.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0080_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClientShare.Packed() && n.HTargetClient.Packed() && n.HTargetDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0080_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0080_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClientShare.Packed() && n.HTargetClient.Packed() && n.HTargetDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0080_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0080_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClientShare.Packed() && n.HTargetClient.Packed() && n.HTargetDevice.Packed() { + // Type NV0080_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0080_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0080_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClientShare.Packed() && n.HTargetClient.Packed() && n.HTargetDevice.Packed() { + // Type NV0080_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0080_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0080_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClientShare.Packed() && n.HTargetClient.Packed() && n.HTargetDevice.Packed() { + // Type NV0080_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00DE_ALLOC_PARAMETERS) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00DE_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Reserved)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00DE_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Reserved = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00DE_ALLOC_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00DE_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00DE_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00DE_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00DE_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00DE_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00DE_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00DE_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00DE_ALLOC_PARAMETERS_V545) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00DE_ALLOC_PARAMETERS_V545) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.PolledDataMask)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00DE_ALLOC_PARAMETERS_V545) UnmarshalBytes(src []byte) []byte { + n.PolledDataMask = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00DE_ALLOC_PARAMETERS_V545) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00DE_ALLOC_PARAMETERS_V545) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00DE_ALLOC_PARAMETERS_V545) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00DE_ALLOC_PARAMETERS_V545) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00DE_ALLOC_PARAMETERS_V545) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00DE_ALLOC_PARAMETERS_V545) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00DE_ALLOC_PARAMETERS_V545) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00DE_ALLOC_PARAMETERS_V545) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00F8_ALLOCATION_PARAMETERS) SizeBytes() int { + return 32 + + (*nv00f8Map)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00F8_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Alignment)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.AllocSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.PageSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.AllocFlags)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + dst = n.Map.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00F8_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Alignment = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.AllocSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.PageSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.AllocFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + src = n.Map.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00F8_ALLOCATION_PARAMETERS) Packed() bool { + return n.Map.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00F8_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.Map.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV00F8_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00F8_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.Map.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00F8_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00F8_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Map.Packed() { + // Type NV00F8_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00F8_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00F8_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Map.Packed() { + // Type NV00F8_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00F8_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00F8_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.Map.Packed() { + // Type NV00F8_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00FD_ALLOCATION_PARAMETERS) SizeBytes() int { + return 32 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00FD_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Alignment)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.AllocSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.PageSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.AllocFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumGPUs)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + dst = n.POsEvent.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00FD_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Alignment = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.AllocSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.PageSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.AllocFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.NumGPUs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + src = n.POsEvent.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00FD_ALLOCATION_PARAMETERS) Packed() bool { + return n.POsEvent.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00FD_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.POsEvent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00FD_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.POsEvent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00FD_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00FD_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00FD_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00FD_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00FD_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) SizeBytes() int { + return 2 + + (*NV_EXPORT_MEM_PACKET)(nil).SizeBytes() + + 1*6 + + (*NV00FD_ALLOCATION_PARAMETERS)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) MarshalBytes(dst []byte) []byte { + dst = n.ExpPacket.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.Index)) + dst = dst[2:] + // Padding: dst[:sizeof(byte)*6] ~= [6]byte{0} + dst = dst[1*(6):] + dst = n.NV00FD_ALLOCATION_PARAMETERS.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) UnmarshalBytes(src []byte) []byte { + src = n.ExpPacket.UnmarshalUnsafe(src) + n.Index = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([6]byte(n._), src[:sizeof(byte)*6]) + src = src[1*(6):] + src = n.NV00FD_ALLOCATION_PARAMETERS.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) Packed() bool { + return n.ExpPacket.Packed() && n.NV00FD_ALLOCATION_PARAMETERS.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) MarshalUnsafe(dst []byte) []byte { + if n.ExpPacket.Packed() && n.NV00FD_ALLOCATION_PARAMETERS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) UnmarshalUnsafe(src []byte) []byte { + if n.ExpPacket.Packed() && n.NV00FD_ALLOCATION_PARAMETERS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.ExpPacket.Packed() && n.NV00FD_ALLOCATION_PARAMETERS.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.ExpPacket.Packed() && n.NV00FD_ALLOCATION_PARAMETERS.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00FD_ALLOCATION_PARAMETERS_V545) WriteTo(writer io.Writer) (int64, error) { + if !n.ExpPacket.Packed() && n.NV00FD_ALLOCATION_PARAMETERS.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV2080_ALLOC_PARAMETERS) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV2080_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubDeviceID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV2080_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.SubDeviceID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV2080_ALLOC_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV2080_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV2080_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV2080_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV2080_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV2080_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV2080_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV2080_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV2081_ALLOC_PARAMETERS) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV2081_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Reserved)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV2081_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Reserved = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV2081_ALLOC_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV2081_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV2081_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV2081_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV2081_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV2081_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV2081_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV2081_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV503B_ALLOC_PARAMETERS) SizeBytes() int { + return 32 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*NV503B_BAR1_P2P_DMA_INFO)(nil).SizeBytes() + + (*NV503B_BAR1_P2P_DMA_INFO)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV503B_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HSubDevice.MarshalUnsafe(dst) + dst = n.HPeerSubDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubDevicePeerIDMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.PeerSubDevicePeerIDMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.MailboxBar1Addr)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.MailboxTotalSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubDeviceEgmPeerIDMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.PeerSubDeviceEgmPeerIDMask)) + dst = dst[4:] + dst = n.L2pBar1P2PDmaInfo.MarshalUnsafe(dst) + dst = n.P2lBar1P2PDmaInfo.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV503B_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HSubDevice.UnmarshalUnsafe(src) + src = n.HPeerSubDevice.UnmarshalUnsafe(src) + n.SubDevicePeerIDMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.PeerSubDevicePeerIDMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.MailboxBar1Addr = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.MailboxTotalSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubDeviceEgmPeerIDMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.PeerSubDeviceEgmPeerIDMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.L2pBar1P2PDmaInfo.UnmarshalUnsafe(src) + src = n.P2lBar1P2PDmaInfo.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV503B_ALLOC_PARAMETERS) Packed() bool { + return n.HPeerSubDevice.Packed() && n.HSubDevice.Packed() && n.L2pBar1P2PDmaInfo.Packed() && n.P2lBar1P2PDmaInfo.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV503B_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HPeerSubDevice.Packed() && n.HSubDevice.Packed() && n.L2pBar1P2PDmaInfo.Packed() && n.P2lBar1P2PDmaInfo.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV503B_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV503B_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HPeerSubDevice.Packed() && n.HSubDevice.Packed() && n.L2pBar1P2PDmaInfo.Packed() && n.P2lBar1P2PDmaInfo.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV503B_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV503B_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HPeerSubDevice.Packed() && n.HSubDevice.Packed() && n.L2pBar1P2PDmaInfo.Packed() && n.P2lBar1P2PDmaInfo.Packed() { + // Type NV503B_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV503B_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV503B_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HPeerSubDevice.Packed() && n.HSubDevice.Packed() && n.L2pBar1P2PDmaInfo.Packed() && n.P2lBar1P2PDmaInfo.Packed() { + // Type NV503B_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV503B_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV503B_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HPeerSubDevice.Packed() && n.HSubDevice.Packed() && n.L2pBar1P2PDmaInfo.Packed() && n.P2lBar1P2PDmaInfo.Packed() { + // Type NV503B_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV503B_BAR1_P2P_DMA_INFO) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV503B_BAR1_P2P_DMA_INFO) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DmaAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DmaSize)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV503B_BAR1_P2P_DMA_INFO) UnmarshalBytes(src []byte) []byte { + n.DmaAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.DmaSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV503B_BAR1_P2P_DMA_INFO) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV503B_BAR1_P2P_DMA_INFO) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV503B_BAR1_P2P_DMA_INFO) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV503B_BAR1_P2P_DMA_INFO) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV503B_BAR1_P2P_DMA_INFO) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV503B_BAR1_P2P_DMA_INFO) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV503B_BAR1_P2P_DMA_INFO) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV503B_BAR1_P2P_DMA_INFO) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV503C_ALLOC_PARAMETERS) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV503C_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV503C_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV503C_ALLOC_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV503C_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV503C_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV503C_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV503C_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV503C_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV503C_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV503C_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV83DE_ALLOC_PARAMETERS) SizeBytes() int { + return 0 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV83DE_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HDebuggerClient_Obsolete.MarshalUnsafe(dst) + dst = n.HAppClient.MarshalUnsafe(dst) + dst = n.HClass3DObject.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV83DE_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HDebuggerClient_Obsolete.UnmarshalUnsafe(src) + src = n.HAppClient.UnmarshalUnsafe(src) + src = n.HClass3DObject.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV83DE_ALLOC_PARAMETERS) Packed() bool { + return n.HAppClient.Packed() && n.HClass3DObject.Packed() && n.HDebuggerClient_Obsolete.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV83DE_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HAppClient.Packed() && n.HClass3DObject.Packed() && n.HDebuggerClient_Obsolete.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV83DE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV83DE_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HAppClient.Packed() && n.HClass3DObject.Packed() && n.HDebuggerClient_Obsolete.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV83DE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV83DE_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HAppClient.Packed() && n.HClass3DObject.Packed() && n.HDebuggerClient_Obsolete.Packed() { + // Type NV83DE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV83DE_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV83DE_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HAppClient.Packed() && n.HClass3DObject.Packed() && n.HDebuggerClient_Obsolete.Packed() { + // Type NV83DE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV83DE_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV83DE_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HAppClient.Packed() && n.HClass3DObject.Packed() && n.HDebuggerClient_Obsolete.Packed() { + // Type NV83DE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV9072_ALLOCATION_PARAMETERS) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV9072_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.LogicalHeadID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.DisplayMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Caps)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV9072_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.LogicalHeadID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.DisplayMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Caps = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV9072_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV9072_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV9072_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV9072_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV9072_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV9072_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV9072_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV9072_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVA0BC_ALLOC_PARAMETERS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVA0BC_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.CodecType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.HResolution)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.VResolution)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Version)) + dst = dst[4:] + dst = n.HMem.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVA0BC_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.CodecType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.HResolution = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.VResolution = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Version = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HMem.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVA0BC_ALLOC_PARAMETERS) Packed() bool { + return n.HMem.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVA0BC_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVA0BC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVA0BC_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVA0BC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVA0BC_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HMem.Packed() { + // Type NVA0BC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVA0BC_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVA0BC_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HMem.Packed() { + // Type NVA0BC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVA0BC_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVA0BC_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HMem.Packed() { + // Type NVA0BC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVB0B5_ALLOCATION_PARAMETERS) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVB0B5_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Version)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineType)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVB0B5_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Version = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.EngineType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVB0B5_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVB0B5_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVB0B5_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVB0B5_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVB0B5_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVB0B5_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVB0B5_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVB0B5_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_BSP_ALLOCATION_PARAMETERS) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_BSP_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ProhibitMultipleInstances)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineInstance)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_BSP_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ProhibitMultipleInstances = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.EngineInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_BSP_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_BSP_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_BSP_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_BSP_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_BSP_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_BSP_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_BSP_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_BSP_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS) SizeBytes() int { + return 40 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes()*NV_MAX_SUBDEVICES + + 8*NV_MAX_SUBDEVICES + + (*Handle)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + 4*CC_CHAN_ALLOC_IV_SIZE_DWORD + + 4*CC_CHAN_ALLOC_IV_SIZE_DWORD + + 4*CC_CHAN_ALLOC_NONCE_SIZE_DWORD +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS) MarshalBytes(dst []byte) []byte { + dst = n.HObjectError.MarshalUnsafe(dst) + dst = n.HObjectBuffer.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.GPFIFOOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GPFIFOEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + dst = n.HContextShare.MarshalUnsafe(dst) + dst = n.HVASpace.MarshalUnsafe(dst) + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + dst = n.HUserdMemory[idx].MarshalUnsafe(dst) + } + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.UserdOffset[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.CID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubDeviceID)) + dst = dst[4:] + dst = n.HObjectECCError.MarshalUnsafe(dst) + dst = n.InstanceMem.MarshalUnsafe(dst) + dst = n.UserdMem.MarshalUnsafe(dst) + dst = n.RamfcMem.MarshalUnsafe(dst) + dst = n.MthdbufMem.MarshalUnsafe(dst) + dst = n.HPhysChannelGroup.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.InternalFlags)) + dst = dst[4:] + dst = n.ErrorNotifierMem.MarshalUnsafe(dst) + dst = n.ECCErrorNotifierMem.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ProcessID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubProcessID)) + dst = dst[4:] + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EncryptIv[idx])) + dst = dst[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.DecryptIv[idx])) + dst = dst[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_NONCE_SIZE_DWORD; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.HmacNonce[idx])) + dst = dst[4:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS) UnmarshalBytes(src []byte) []byte { + src = n.HObjectError.UnmarshalUnsafe(src) + src = n.HObjectBuffer.UnmarshalUnsafe(src) + n.GPFIFOOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.GPFIFOEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HContextShare.UnmarshalUnsafe(src) + src = n.HVASpace.UnmarshalUnsafe(src) + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + src = n.HUserdMemory[idx].UnmarshalUnsafe(src) + } + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + n.UserdOffset[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + n.EngineType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.CID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubDeviceID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HObjectECCError.UnmarshalUnsafe(src) + src = n.InstanceMem.UnmarshalUnsafe(src) + src = n.UserdMem.UnmarshalUnsafe(src) + src = n.RamfcMem.UnmarshalUnsafe(src) + src = n.MthdbufMem.UnmarshalUnsafe(src) + src = n.HPhysChannelGroup.UnmarshalUnsafe(src) + n.InternalFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.ErrorNotifierMem.UnmarshalUnsafe(src) + src = n.ECCErrorNotifierMem.UnmarshalUnsafe(src) + n.ProcessID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubProcessID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + n.EncryptIv[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + n.DecryptIv[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_NONCE_SIZE_DWORD; idx++ { + n.HmacNonce[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CHANNEL_ALLOC_PARAMS) Packed() bool { + return n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CHANNEL_ALLOC_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CHANNEL_ALLOC_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CHANNEL_ALLOC_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CHANNEL_ALLOC_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CHANNEL_ALLOC_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CHANNEL_ALLOC_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CHANNEL_ALLOC_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CHANNEL_ALLOC_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CHANNEL_ALLOC_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) SizeBytes() int { + return 8 + + (*NV_CHANNEL_ALLOC_PARAMS)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) MarshalBytes(dst []byte) []byte { + dst = n.NV_CHANNEL_ALLOC_PARAMS.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.TPCConfigID)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) UnmarshalBytes(src []byte) []byte { + src = n.NV_CHANNEL_ALLOC_PARAMS.UnmarshalUnsafe(src) + n.TPCConfigID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) Packed() bool { + return n.NV_CHANNEL_ALLOC_PARAMS.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) MarshalUnsafe(dst []byte) []byte { + if n.NV_CHANNEL_ALLOC_PARAMS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CHANNEL_ALLOC_PARAMS_V570 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) UnmarshalUnsafe(src []byte) []byte { + if n.NV_CHANNEL_ALLOC_PARAMS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CHANNEL_ALLOC_PARAMS_V570 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_CHANNEL_ALLOC_PARAMS.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS_V570 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_CHANNEL_ALLOC_PARAMS.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS_V570 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CHANNEL_ALLOC_PARAMS_V570) WriteTo(writer io.Writer) (int64, error) { + if !n.NV_CHANNEL_ALLOC_PARAMS.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS_V570 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) SizeBytes() int { + return 5 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*3 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HObjectError.MarshalUnsafe(dst) + dst = n.HObjectECCError.MarshalUnsafe(dst) + dst = n.HVASpace.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineType)) + dst = dst[4:] + dst[0] = byte(n.BIsCallingContextVgpuPlugin) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HObjectError.UnmarshalUnsafe(src) + src = n.HObjectECCError.UnmarshalUnsafe(src) + src = n.HVASpace.UnmarshalUnsafe(src) + n.EngineType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.BIsCallingContextVgpuPlugin = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) Packed() bool { + return n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HVASpace.Packed() { + // Type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HVASpace.Packed() { + // Type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HVASpace.Packed() { + // Type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) SizeBytes() int { + return 0 + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) MarshalBytes(dst []byte) []byte { + dst = n.Handle.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) UnmarshalBytes(src []byte) []byte { + src = n.Handle.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) Packed() bool { + return n.Handle.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.Handle.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.Handle.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Handle.Packed() { + // Type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Handle.Packed() { + // Type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.Handle.Packed() { + // Type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) SizeBytes() int { + return 24 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + dst = n.HSubDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + dst = n.HMemory.MarshalUnsafe(dst) + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Limit)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + src = n.HSubDevice.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HMemory.UnmarshalUnsafe(src) + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Limit = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) Packed() bool { + return n.HMemory.Packed() && n.HSubDevice.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.HMemory.Packed() && n.HSubDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CONTEXT_DMA_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.HMemory.Packed() && n.HSubDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CONTEXT_DMA_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HMemory.Packed() && n.HSubDevice.Packed() { + // Type NV_CONTEXT_DMA_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HMemory.Packed() && n.HSubDevice.Packed() { + // Type NV_CONTEXT_DMA_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CONTEXT_DMA_ALLOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.HMemory.Packed() && n.HSubDevice.Packed() { + // Type NV_CONTEXT_DMA_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HVASpace.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubctxID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HVASpace.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubctxID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) Packed() bool { + return n.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CTXSHARE_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CTXSHARE_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HVASpace.Packed() { + // Type NV_CTXSHARE_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HVASpace.Packed() { + // Type NV_CTXSHARE_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CTXSHARE_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HVASpace.Packed() { + // Type NV_CTXSHARE_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_EXPORT_MEM_PACKET) SizeBytes() int { + return 0 + + 1*NV_MEM_EXPORT_UUID_LEN + + 1*16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_EXPORT_MEM_PACKET) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < NV_MEM_EXPORT_UUID_LEN; idx++ { + dst[0] = byte(n.UUID[idx]) + dst = dst[1:] + } + for idx := 0; idx < 16; idx++ { + dst[0] = byte(n.Opaque[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_EXPORT_MEM_PACKET) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < NV_MEM_EXPORT_UUID_LEN; idx++ { + n.UUID[idx] = uint8(src[0]) + src = src[1:] + } + for idx := 0; idx < 16; idx++ { + n.Opaque[idx] = uint8(src[0]) + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_EXPORT_MEM_PACKET) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_EXPORT_MEM_PACKET) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_EXPORT_MEM_PACKET) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_EXPORT_MEM_PACKET) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_EXPORT_MEM_PACKET) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_EXPORT_MEM_PACKET) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_EXPORT_MEM_PACKET) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_EXPORT_MEM_PACKET) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_GR_ALLOCATION_PARAMETERS) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_GR_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Version)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Caps)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_GR_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Version = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Caps = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_GR_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_GR_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_GR_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_GR_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_GR_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_GR_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_GR_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_GR_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_HOPPER_USERMODE_A_PARAMS) SizeBytes() int { + return 2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_HOPPER_USERMODE_A_PARAMS) MarshalBytes(dst []byte) []byte { + dst[0] = byte(n.Bar1Mapping) + dst = dst[1:] + dst[0] = byte(n.Priv) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_HOPPER_USERMODE_A_PARAMS) UnmarshalBytes(src []byte) []byte { + n.Bar1Mapping = uint8(src[0]) + src = src[1:] + n.Priv = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_HOPPER_USERMODE_A_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_HOPPER_USERMODE_A_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_HOPPER_USERMODE_A_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_HOPPER_USERMODE_A_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_HOPPER_USERMODE_A_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_HOPPER_USERMODE_A_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_HOPPER_USERMODE_A_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_HOPPER_USERMODE_A_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_ALLOCATION_PARAMS) SizeBytes() int { + return 108 + + (*P64)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_ALLOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Owner)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Type)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Width)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Height)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Pitch)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Attr)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Attr2)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Format)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ComprCovg)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ZcullCovg)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.RangeLo)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.RangeHi)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Alignment)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Limit)) + dst = dst[8:] + dst = n.Address.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.CtagOffset)) + dst = dst[4:] + dst = n.HVASpace.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.InternalFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Tag)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_ALLOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + n.Owner = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Type = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Width = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Height = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Pitch = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Attr = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Attr2 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Format = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ComprCovg = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ZcullCovg = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + n.RangeLo = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.RangeHi = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Alignment = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Limit = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = n.Address.UnmarshalUnsafe(src) + n.CtagOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HVASpace.UnmarshalUnsafe(src) + n.InternalFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Tag = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_ALLOCATION_PARAMS) Packed() bool { + return n.Address.Packed() && n.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_ALLOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.Address.Packed() && n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_MEMORY_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_ALLOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.Address.Packed() && n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_MEMORY_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_ALLOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Address.Packed() && n.HVASpace.Packed() { + // Type NV_MEMORY_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_ALLOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_ALLOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Address.Packed() && n.HVASpace.Packed() { + // Type NV_MEMORY_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_ALLOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_ALLOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.Address.Packed() && n.HVASpace.Packed() { + // Type NV_MEMORY_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) SizeBytes() int { + return 8 + + (*NV_MEMORY_ALLOCATION_PARAMS)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) MarshalBytes(dst []byte) []byte { + dst = n.NV_MEMORY_ALLOCATION_PARAMS.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumaNode)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) UnmarshalBytes(src []byte) []byte { + src = n.NV_MEMORY_ALLOCATION_PARAMS.UnmarshalUnsafe(src) + n.NumaNode = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) Packed() bool { + return n.NV_MEMORY_ALLOCATION_PARAMS.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) MarshalUnsafe(dst []byte) []byte { + if n.NV_MEMORY_ALLOCATION_PARAMS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_MEMORY_ALLOCATION_PARAMS_V545 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) UnmarshalUnsafe(src []byte) []byte { + if n.NV_MEMORY_ALLOCATION_PARAMS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_MEMORY_ALLOCATION_PARAMS_V545 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_MEMORY_ALLOCATION_PARAMS.Packed() { + // Type NV_MEMORY_ALLOCATION_PARAMS_V545 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_MEMORY_ALLOCATION_PARAMS.Packed() { + // Type NV_MEMORY_ALLOCATION_PARAMS_V545 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_ALLOCATION_PARAMS_V545) WriteTo(writer io.Writer) (int64, error) { + if !n.NV_MEMORY_ALLOCATION_PARAMS.Packed() { + // Type NV_MEMORY_ALLOCATION_PARAMS_V545 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_DESC_PARAMS) SizeBytes() int { + return 24 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_DESC_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.AddressSpace)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.CacheAttrib)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_DESC_PARAMS) UnmarshalBytes(src []byte) []byte { + n.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.AddressSpace = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.CacheAttrib = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_DESC_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_DESC_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_DESC_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_DESC_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_DESC_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_DESC_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_DESC_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_DESC_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) SizeBytes() int { + return 1 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + dst[0] = byte(n.unused) + dst = dst[1:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + n.unused = uint8(src[0]) + src = src[1:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) SizeBytes() int { + return 4 + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) MarshalBytes(dst []byte) []byte { + dst = n.HSemaphoreSurface.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.MaxQueueSize)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) UnmarshalBytes(src []byte) []byte { + src = n.HSemaphoreSurface.UnmarshalUnsafe(src) + n.MaxQueueSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) Packed() bool { + return n.HSemaphoreSurface.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) MarshalUnsafe(dst []byte) []byte { + if n.HSemaphoreSurface.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) UnmarshalUnsafe(src []byte) []byte { + if n.HSemaphoreSurface.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HSemaphoreSurface.Packed() { + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HSemaphoreSurface.Packed() { + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550) WriteTo(writer io.Writer) (int64, error) { + if !n.HSemaphoreSurface.Packed() { + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) SizeBytes() int { + return 12 + + (*NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) MarshalBytes(dst []byte) []byte { + dst = n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.MarshalUnsafe(dst) + dst = n.HNotificationMemory.MarshalUnsafe(dst) + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.NotificationOffset)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) UnmarshalBytes(src []byte) []byte { + src = n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.UnmarshalUnsafe(src) + src = n.HNotificationMemory.UnmarshalUnsafe(src) + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + n.NotificationOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) Packed() bool { + return n.HNotificationMemory.Packed() && n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) MarshalUnsafe(dst []byte) []byte { + if n.HNotificationMemory.Packed() && n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) UnmarshalUnsafe(src []byte) []byte { + if n.HNotificationMemory.Packed() && n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HNotificationMemory.Packed() && n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.Packed() { + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HNotificationMemory.Packed() && n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.Packed() { + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555) WriteTo(writer io.Writer) (int64, error) { + if !n.HNotificationMemory.Packed() && n.NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550.Packed() { + // Type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Limit)) + dst = dst[8:] + dst = n.HVASpace.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Limit = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = n.HVASpace.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) Packed() bool { + return n.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HVASpace.Packed() { + // Type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HVASpace.Packed() { + // Type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.HVASpace.Packed() { + // Type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ProhibitMultipleInstances)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineInstance)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ProhibitMultipleInstances = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.EngineInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_MSENC_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_MSENC_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HSemaphoreMem.MarshalUnsafe(dst) + dst = n.HMaxSubmittedMem.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.flags)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HSemaphoreMem.UnmarshalUnsafe(src) + src = n.HMaxSubmittedMem.UnmarshalUnsafe(src) + n.flags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) Packed() bool { + return n.HMaxSubmittedMem.Packed() && n.HSemaphoreMem.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HMaxSubmittedMem.Packed() && n.HSemaphoreMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HMaxSubmittedMem.Packed() && n.HSemaphoreMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HMaxSubmittedMem.Packed() && n.HSemaphoreMem.Packed() { + // Type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HMaxSubmittedMem.Packed() && n.HSemaphoreMem.Packed() { + // Type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HMaxSubmittedMem.Packed() && n.HSemaphoreMem.Packed() { + // Type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) SizeBytes() int { + return 44 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VASize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VAStartInternal)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VALimitInternal)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.BigPageSize)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VABase)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.VASize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.VAStartInternal = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.VALimitInternal = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.BigPageSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.VABase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) SizeBytes() int { + return 4 + + (*NV_VASPACE_ALLOCATION_PARAMETERS)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) MarshalBytes(dst []byte) []byte { + dst = n.NV_VASPACE_ALLOCATION_PARAMETERS.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Pasid)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) UnmarshalBytes(src []byte) []byte { + src = n.NV_VASPACE_ALLOCATION_PARAMETERS.UnmarshalUnsafe(src) + n.Pasid = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) Packed() bool { + return n.NV_VASPACE_ALLOCATION_PARAMETERS.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) MarshalUnsafe(dst []byte) []byte { + if n.NV_VASPACE_ALLOCATION_PARAMETERS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_VASPACE_ALLOCATION_PARAMETERS_V580 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) UnmarshalUnsafe(src []byte) []byte { + if n.NV_VASPACE_ALLOCATION_PARAMETERS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_VASPACE_ALLOCATION_PARAMETERS_V580 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_VASPACE_ALLOCATION_PARAMETERS.Packed() { + // Type NV_VASPACE_ALLOCATION_PARAMETERS_V580 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_VASPACE_ALLOCATION_PARAMETERS.Packed() { + // Type NV_VASPACE_ALLOCATION_PARAMETERS_V580 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_VASPACE_ALLOCATION_PARAMETERS_V580) WriteTo(writer io.Writer) (int64, error) { + if !n.NV_VASPACE_ALLOCATION_PARAMETERS.Packed() { + // Type NV_VASPACE_ALLOCATION_PARAMETERS_V580 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *nv00f8Map) SizeBytes() int { + return 12 + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *nv00f8Map) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.offset)) + dst = dst[8:] + dst = n.hVidMem.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *nv00f8Map) UnmarshalBytes(src []byte) []byte { + n.offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = n.hVidMem.UnmarshalUnsafe(src) + n.flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *nv00f8Map) Packed() bool { + return n.hVidMem.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *nv00f8Map) MarshalUnsafe(dst []byte) []byte { + if n.hVidMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type nv00f8Map doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *nv00f8Map) UnmarshalUnsafe(src []byte) []byte { + if n.hVidMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type nv00f8Map doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *nv00f8Map) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.hVidMem.Packed() { + // Type nv00f8Map doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *nv00f8Map) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *nv00f8Map) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.hVidMem.Packed() { + // Type nv00f8Map doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *nv00f8Map) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *nv00f8Map) WriteTo(writer io.Writer) (int64, error) { + if !n.hVidMem.Packed() { + // Type nv00f8Map doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) SizeBytes() int { + return 32 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GpuID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GpuFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.DeviceInstance)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubDeviceInstance)) + dst = dst[4:] + dst = n.SzName.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SliStatus)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.BoardID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GpuInstance)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumaID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) UnmarshalBytes(src []byte) []byte { + n.GpuID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.GpuFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.DeviceInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubDeviceInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.SzName.UnmarshalUnsafe(src) + n.SliStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.BoardID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.GpuInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.NumaID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) Packed() bool { + return n.SzName.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.SzName.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.SzName.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.SzName.Packed() { + // Type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.SzName.Packed() { + // Type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.SzName.Packed() { + // Type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) SizeBytes() int { + return 4 + + 1*12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Type)) + dst = dst[4:] + for idx := 0; idx < 12; idx++ { + dst[0] = byte(n.Data[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) UnmarshalBytes(src []byte) []byte { + n.Type = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 12; idx++ { + n.Data[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) SizeBytes() int { + return 10 + + (*Handle)(nil).SizeBytes() + + 1*NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE + + 1*2 + + (*Handle)(nil).SizeBytes()*NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + dst = p.HDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.MaxObjects)) + dst = dst[2:] + for idx := 0; idx < NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE; idx++ { + dst[0] = byte(p.Metadata[idx]) + dst = dst[1:] + } + for idx := 0; idx < 2; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + for idx := 0; idx < NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS; idx++ { + dst = p.Objects[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.NumObjects)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.Index)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HDevice.UnmarshalUnsafe(src) + p.MaxObjects = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE; idx++ { + p.Metadata[idx] = uint8(src[0]) + src = src[1:] + } + for idx := 0; idx < 2; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS; idx++ { + src = p.Objects[idx].UnmarshalUnsafe(src) + } + p.NumObjects = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + p.Index = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) Packed() bool { + return p.HDevice.Packed() && p.Objects[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.HDevice.Packed() && p.Objects[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.HDevice.Packed() && p.Objects[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HDevice.Packed() && p.Objects[0].Packed() { + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HDevice.Packed() && p.Objects[0].Packed() { + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.HDevice.Packed() && p.Objects[0].Packed() { + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) SizeBytes() int { + return 8 + + (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.Object.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.Object.UnmarshalUnsafe(src) + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) Packed() bool { + return p.Object.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.Object.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.Object.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Object.Packed() { + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Object.Packed() { + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.Object.Packed() { + // Type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) SizeBytes() int { + return 10 + + 1*NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE + + 1*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.DeviceInstance)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.MaxObjects)) + dst = dst[2:] + for idx := 0; idx < NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE; idx++ { + dst[0] = byte(p.Metadata[idx]) + dst = dst[1:] + } + for idx := 0; idx < 2; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.DeviceInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.MaxObjects = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE; idx++ { + p.Metadata[idx] = uint8(src[0]) + src = src[1:] + } + for idx := 0; idx < 2; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) SizeBytes() int { + return 14 + + 1*NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE + + 1*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.DeviceInstance)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.GpuInstanceID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.MaxObjects)) + dst = dst[2:] + for idx := 0; idx < NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE; idx++ { + dst[0] = byte(p.Metadata[idx]) + dst = dst[1:] + } + for idx := 0; idx < 2; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.DeviceInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.GpuInstanceID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.MaxObjects = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE; idx++ { + p.Metadata[idx] = uint8(src[0]) + src = src[1:] + } + for idx := 0; idx < 2; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes()*NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS + + 1*NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + dst = p.HParent.MarshalUnsafe(dst) + for idx := 0; idx < NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS; idx++ { + dst = p.Objects[idx].MarshalUnsafe(dst) + } + for idx := 0; idx < NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS; idx++ { + dst[0] = byte(p.ObjectTypes[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.NumObjects)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.Index)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HParent.UnmarshalUnsafe(src) + for idx := 0; idx < NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS; idx++ { + src = p.Objects[idx].UnmarshalUnsafe(src) + } + for idx := 0; idx < NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS; idx++ { + p.ObjectTypes[idx] = uint8(src[0]) + src = src[1:] + } + p.NumObjects = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + p.Index = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) Packed() bool { + return p.HParent.Packed() && p.Objects[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.HParent.Packed() && p.Objects[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.HParent.Packed() && p.Objects[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HParent.Packed() && p.Objects[0].Packed() { + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HParent.Packed() && p.Objects[0].Packed() { + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.HParent.Packed() && p.Objects[0].Packed() { + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) SizeBytes() int { + return 4 + + (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECT)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + dst = p.Object.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.Object.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) Packed() bool { + return p.Object.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.Object.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.Object.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Object.Packed() { + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Object.Packed() { + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.Object.Packed() { + // Type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) SizeBytes() int { + return 12 + + 1*4 + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SizeOfStrings)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad[idx]) + dst = dst[1:] + } + dst = n.PDriverVersionBuffer.MarshalUnsafe(dst) + dst = n.PVersionBuffer.MarshalUnsafe(dst) + dst = n.PTitleBuffer.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ChangelistNumber)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.OfficialChangelistNumber)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) UnmarshalBytes(src []byte) []byte { + n.SizeOfStrings = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad[idx] = src[0] + src = src[1:] + } + src = n.PDriverVersionBuffer.UnmarshalUnsafe(src) + src = n.PVersionBuffer.UnmarshalUnsafe(src) + src = n.PTitleBuffer.UnmarshalUnsafe(src) + n.ChangelistNumber = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.OfficialChangelistNumber = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) Packed() bool { + return n.PDriverVersionBuffer.Packed() && n.PTitleBuffer.Packed() && n.PVersionBuffer.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.PDriverVersionBuffer.Packed() && n.PTitleBuffer.Packed() && n.PVersionBuffer.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.PDriverVersionBuffer.Packed() && n.PTitleBuffer.Packed() && n.PVersionBuffer.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.PDriverVersionBuffer.Packed() && n.PTitleBuffer.Packed() && n.PVersionBuffer.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.PDriverVersionBuffer.Packed() && n.PTitleBuffer.Packed() && n.PVersionBuffer.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.PDriverVersionBuffer.Packed() && n.PTitleBuffer.Packed() && n.PVersionBuffer.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) SizeBytes() int { + return 16 + + 4*NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS + + 1*NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE + + 1*7 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GpuIDs[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GpuCount)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.P2PCaps)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.P2POptimalReadCEs)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.P2POptimalWriteCEs)) + dst = dst[4:] + for idx := 0; idx < NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE; idx++ { + dst[0] = byte(n.P2PCapsStatus[idx]) + dst = dst[1:] + } + // Padding: dst[:sizeof(byte)*7] ~= [7]byte{0} + dst = dst[1*(7):] + dst = n.BusPeerIDs.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS; idx++ { + n.GpuIDs[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + n.GpuCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.P2PCaps = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.P2POptimalReadCEs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.P2POptimalWriteCEs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE; idx++ { + n.P2PCapsStatus[idx] = uint8(src[0]) + src = src[1:] + } + // Padding: ~ copy([7]byte(n._), src[:sizeof(byte)*7]) + src = src[1*(7):] + src = n.BusPeerIDs.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) Packed() bool { + return n.BusPeerIDs.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.BusPeerIDs.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.BusPeerIDs.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.BusPeerIDs.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.BusPeerIDs.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.BusPeerIDs.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) SizeBytes() int { + return 0 + + (*NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) MarshalBytes(dst []byte) []byte { + dst = n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.MarshalUnsafe(dst) + dst = n.BusEgmPeerIDs.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) UnmarshalBytes(src []byte) []byte { + src = n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.UnmarshalUnsafe(src) + src = n.BusEgmPeerIDs.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) Packed() bool { + return n.BusEgmPeerIDs.Packed() && n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) MarshalUnsafe(dst []byte) []byte { + if n.BusEgmPeerIDs.Packed() && n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) UnmarshalUnsafe(src []byte) []byte { + if n.BusEgmPeerIDs.Packed() && n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.BusEgmPeerIDs.Packed() && n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.BusEgmPeerIDs.Packed() && n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550) WriteTo(writer io.Writer) (int64, error) { + if !n.BusEgmPeerIDs.Packed() && n.NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.Packed() { + // Type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) SizeBytes() int { + return 4 + + 1*4 + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumChannels)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad[idx]) + dst = dst[1:] + } + dst = n.PChannelHandleList.MarshalUnsafe(dst) + dst = n.PChannelList.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) UnmarshalBytes(src []byte) []byte { + n.NumChannels = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad[idx] = src[0] + src = src[1:] + } + src = n.PChannelHandleList.UnmarshalUnsafe(src) + src = n.PChannelList.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) Packed() bool { + return n.PChannelHandleList.Packed() && n.PChannelList.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.PChannelHandleList.Packed() && n.PChannelList.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.PChannelHandleList.Packed() && n.PChannelList.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.PChannelHandleList.Packed() && n.PChannelList.Packed() { + // Type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.PChannelHandleList.Packed() && n.PChannelList.Packed() { + // Type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.PChannelHandleList.Packed() && n.PChannelList.Packed() { + // Type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) SizeBytes() int { + return 4 + + 1*4 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.CapsTblSize)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad[idx]) + dst = dst[1:] + } + dst = n.CapsTbl.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) UnmarshalBytes(src []byte) []byte { + n.CapsTblSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad[idx] = src[0] + src = src[1:] + } + src = n.CapsTbl.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0080_CTRL_GET_CAPS_PARAMS) Packed() bool { + return n.CapsTbl.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.CapsTbl.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV0080_CTRL_GET_CAPS_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.CapsTbl.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV0080_CTRL_GET_CAPS_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.CapsTbl.Packed() { + // Type NV0080_CTRL_GET_CAPS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.CapsTbl.Packed() { + // Type NV0080_CTRL_GET_CAPS_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0080_CTRL_GET_CAPS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.CapsTbl.Packed() { + // Type NV0080_CTRL_GET_CAPS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0080_CTRL_GR_ROUTE_INFO) SizeBytes() int { + return 12 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0080_CTRL_GR_ROUTE_INFO) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Route)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0080_CTRL_GR_ROUTE_INFO) UnmarshalBytes(src []byte) []byte { + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad[idx] = src[0] + src = src[1:] + } + n.Route = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0080_CTRL_GR_ROUTE_INFO) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0080_CTRL_GR_ROUTE_INFO) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0080_CTRL_GR_ROUTE_INFO) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0080_CTRL_GR_ROUTE_INFO) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0080_CTRL_GR_ROUTE_INFO) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0080_CTRL_GR_ROUTE_INFO) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0080_CTRL_GR_ROUTE_INFO) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0080_CTRL_GR_ROUTE_INFO) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) SizeBytes() int { + return 12 + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) MarshalBytes(dst []byte) []byte { + dst = n.HSubDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DevDescriptor)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) UnmarshalBytes(src []byte) []byte { + src = n.HSubDevice.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.DevDescriptor = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) Packed() bool { + return n.HSubDevice.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.HSubDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV00FD_CTRL_ATTACH_GPU_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.HSubDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00FD_CTRL_ATTACH_GPU_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HSubDevice.Packed() { + // Type NV00FD_CTRL_ATTACH_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HSubDevice.Packed() { + // Type NV00FD_CTRL_ATTACH_GPU_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00FD_CTRL_ATTACH_GPU_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.HSubDevice.Packed() { + // Type NV00FD_CTRL_ATTACH_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) SizeBytes() int { + return 7 + + 1*3 + + 1*6 + + (*P64)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes()*NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES + + (*Handle)(nil).SizeBytes()*NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) MarshalBytes(dst []byte) []byte { + dst[0] = byte(n.BDisable) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumChannels)) + dst = dst[4:] + dst[0] = byte(n.BOnlyDisableScheduling) + dst = dst[1:] + dst[0] = byte(n.BRewindGpPut) + dst = dst[1:] + for idx := 0; idx < 6; idx++ { + dst[0] = byte(n.Pad2[idx]) + dst = dst[1:] + } + dst = n.PRunlistPreemptEvent.MarshalUnsafe(dst) + for idx := 0; idx < NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES; idx++ { + dst = n.HClientList[idx].MarshalUnsafe(dst) + } + for idx := 0; idx < NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES; idx++ { + dst = n.HChannelList[idx].MarshalUnsafe(dst) + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) UnmarshalBytes(src []byte) []byte { + n.BDisable = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + n.NumChannels = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.BOnlyDisableScheduling = uint8(src[0]) + src = src[1:] + n.BRewindGpPut = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 6; idx++ { + n.Pad2[idx] = src[0] + src = src[1:] + } + src = n.PRunlistPreemptEvent.UnmarshalUnsafe(src) + for idx := 0; idx < NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES; idx++ { + src = n.HClientList[idx].UnmarshalUnsafe(src) + } + for idx := 0; idx < NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES; idx++ { + src = n.HChannelList[idx].UnmarshalUnsafe(src) + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) Packed() bool { + return n.HChannelList[0].Packed() && n.HClientList[0].Packed() && n.PRunlistPreemptEvent.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.HChannelList[0].Packed() && n.HClientList[0].Packed() && n.PRunlistPreemptEvent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.HChannelList[0].Packed() && n.HClientList[0].Packed() && n.PRunlistPreemptEvent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HChannelList[0].Packed() && n.HClientList[0].Packed() && n.PRunlistPreemptEvent.Packed() { + // Type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HChannelList[0].Packed() && n.HClientList[0].Packed() && n.PRunlistPreemptEvent.Packed() { + // Type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.HChannelList[0].Packed() && n.HClientList[0].Packed() && n.PRunlistPreemptEvent.Packed() { + // Type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) SizeBytes() int { + return 0 + + (*NvxxxCtrlXxxGetInfoParams)(nil).SizeBytes() + + (*NV0080_CTRL_GR_ROUTE_INFO)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.NvxxxCtrlXxxGetInfoParams.MarshalUnsafe(dst) + dst = p.GRRouteInfo.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.NvxxxCtrlXxxGetInfoParams.UnmarshalUnsafe(src) + src = p.GRRouteInfo.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) Packed() bool { + return p.GRRouteInfo.Packed() && p.NvxxxCtrlXxxGetInfoParams.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GRRouteInfo.Packed() && p.NvxxxCtrlXxxGetInfoParams.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV2080_CTRL_GR_GET_INFO_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GRRouteInfo.Packed() && p.NvxxxCtrlXxxGetInfoParams.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV2080_CTRL_GR_GET_INFO_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GRRouteInfo.Packed() && p.NvxxxCtrlXxxGetInfoParams.Packed() { + // Type NV2080_CTRL_GR_GET_INFO_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GRRouteInfo.Packed() && p.NvxxxCtrlXxxGetInfoParams.Packed() { + // Type NV2080_CTRL_GR_GET_INFO_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GRRouteInfo.Packed() && p.NvxxxCtrlXxxGetInfoParams.Packed() { + // Type NV2080_CTRL_GR_GET_INFO_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Result)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Checksum)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) UnmarshalBytes(src []byte) []byte { + n.Result = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Checksum = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) MarshalBytes(dst []byte) []byte { + dst = n.HVASpace.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.VASpaceToken)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) UnmarshalBytes(src []byte) []byte { + src = n.HVASpace.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + n.Pad[idx] = src[0] + src = src[1:] + } + n.VASpaceToken = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) Packed() bool { + return n.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HVASpace.Packed() { + // Type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HVASpace.Packed() { + // Type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV503C_CTRL_REGISTER_VA_SPACE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.HVASpace.Packed() { + // Type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVXXXX_CTRL_XXX_INFO) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVXXXX_CTRL_XXX_INFO) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Data)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVXXXX_CTRL_XXX_INFO) UnmarshalBytes(src []byte) []byte { + n.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Data = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVXXXX_CTRL_XXX_INFO) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVXXXX_CTRL_XXX_INFO) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVXXXX_CTRL_XXX_INFO) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVXXXX_CTRL_XXX_INFO) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVXXXX_CTRL_XXX_INFO) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVXXXX_CTRL_XXX_INFO) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVXXXX_CTRL_XXX_INFO) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVXXXX_CTRL_XXX_INFO) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NvxxxCtrlXxxGetInfoParams) SizeBytes() int { + return 4 + + 1*4 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NvxxxCtrlXxxGetInfoParams) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.InfoListSize)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + dst = p.InfoList.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NvxxxCtrlXxxGetInfoParams) UnmarshalBytes(src []byte) []byte { + p.InfoListSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + src = p.InfoList.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NvxxxCtrlXxxGetInfoParams) Packed() bool { + return p.InfoList.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NvxxxCtrlXxxGetInfoParams) MarshalUnsafe(dst []byte) []byte { + if p.InfoList.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NvxxxCtrlXxxGetInfoParams doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NvxxxCtrlXxxGetInfoParams) UnmarshalUnsafe(src []byte) []byte { + if p.InfoList.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NvxxxCtrlXxxGetInfoParams doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NvxxxCtrlXxxGetInfoParams) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.InfoList.Packed() { + // Type NvxxxCtrlXxxGetInfoParams doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NvxxxCtrlXxxGetInfoParams) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NvxxxCtrlXxxGetInfoParams) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.InfoList.Packed() { + // Type NvxxxCtrlXxxGetInfoParams doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NvxxxCtrlXxxGetInfoParams) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NvxxxCtrlXxxGetInfoParams) WriteTo(writer io.Writer) (int64, error) { + if !p.InfoList.Packed() { + // Type NvxxxCtrlXxxGetInfoParams doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RmapiParamNvU32List) SizeBytes() int { + return 4 + + 1*4 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RmapiParamNvU32List) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.NumElems)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(r.Pad[idx]) + dst = dst[1:] + } + dst = r.List.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RmapiParamNvU32List) UnmarshalBytes(src []byte) []byte { + r.NumElems = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + r.Pad[idx] = src[0] + src = src[1:] + } + src = r.List.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RmapiParamNvU32List) Packed() bool { + return r.List.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RmapiParamNvU32List) MarshalUnsafe(dst []byte) []byte { + if r.List.Packed() { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] + } + // Type RmapiParamNvU32List doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RmapiParamNvU32List) UnmarshalUnsafe(src []byte) []byte { + if r.List.Packed() { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type RmapiParamNvU32List doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RmapiParamNvU32List) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !r.List.Packed() { + // Type RmapiParamNvU32List doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RmapiParamNvU32List) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RmapiParamNvU32List) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !r.List.Packed() { + // Type RmapiParamNvU32List doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RmapiParamNvU32List) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RmapiParamNvU32List) WriteTo(writer io.Writer) (int64, error) { + if !r.List.Packed() { + // Type RmapiParamNvU32List doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlAllocOSEvent) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlAllocOSEvent) MarshalBytes(dst []byte) []byte { + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlAllocOSEvent) UnmarshalBytes(src []byte) []byte { + src = p.HClient.UnmarshalUnsafe(src) + src = p.HDevice.UnmarshalUnsafe(src) + p.FD = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlAllocOSEvent) Packed() bool { + return p.HClient.Packed() && p.HDevice.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlAllocOSEvent) MarshalUnsafe(dst []byte) []byte { + if p.HClient.Packed() && p.HDevice.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlAllocOSEvent doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlAllocOSEvent) UnmarshalUnsafe(src []byte) []byte { + if p.HClient.Packed() && p.HDevice.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlAllocOSEvent doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlAllocOSEvent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.HDevice.Packed() { + // Type IoctlAllocOSEvent doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlAllocOSEvent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlAllocOSEvent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.HDevice.Packed() { + // Type IoctlAllocOSEvent doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlAllocOSEvent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlAllocOSEvent) WriteTo(writer io.Writer) (int64, error) { + if !p.HClient.Packed() && p.HDevice.Packed() { + // Type IoctlAllocOSEvent doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlFreeOSEvent) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlFreeOSEvent) MarshalBytes(dst []byte) []byte { + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HDevice.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlFreeOSEvent) UnmarshalBytes(src []byte) []byte { + src = p.HClient.UnmarshalUnsafe(src) + src = p.HDevice.UnmarshalUnsafe(src) + p.FD = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlFreeOSEvent) Packed() bool { + return p.HClient.Packed() && p.HDevice.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlFreeOSEvent) MarshalUnsafe(dst []byte) []byte { + if p.HClient.Packed() && p.HDevice.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlFreeOSEvent doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlFreeOSEvent) UnmarshalUnsafe(src []byte) []byte { + if p.HClient.Packed() && p.HDevice.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlFreeOSEvent doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlFreeOSEvent) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.HDevice.Packed() { + // Type IoctlFreeOSEvent doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlFreeOSEvent) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlFreeOSEvent) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.HDevice.Packed() { + // Type IoctlFreeOSEvent doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlFreeOSEvent) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlFreeOSEvent) WriteTo(writer io.Writer) (int64, error) { + if !p.HClient.Packed() && p.HDevice.Packed() { + // Type IoctlFreeOSEvent doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlNVOS02ParametersWithFD) SizeBytes() int { + return 4 + + (*NVOS02_PARAMETERS)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlNVOS02ParametersWithFD) MarshalBytes(dst []byte) []byte { + dst = p.Params.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlNVOS02ParametersWithFD) UnmarshalBytes(src []byte) []byte { + src = p.Params.UnmarshalUnsafe(src) + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlNVOS02ParametersWithFD) Packed() bool { + return p.Params.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlNVOS02ParametersWithFD) MarshalUnsafe(dst []byte) []byte { + if p.Params.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlNVOS02ParametersWithFD doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlNVOS02ParametersWithFD) UnmarshalUnsafe(src []byte) []byte { + if p.Params.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlNVOS02ParametersWithFD doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlNVOS02ParametersWithFD) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Params.Packed() { + // Type IoctlNVOS02ParametersWithFD doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlNVOS02ParametersWithFD) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlNVOS02ParametersWithFD) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Params.Packed() { + // Type IoctlNVOS02ParametersWithFD doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlNVOS02ParametersWithFD) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlNVOS02ParametersWithFD) WriteTo(writer io.Writer) (int64, error) { + if !p.Params.Packed() { + // Type IoctlNVOS02ParametersWithFD doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlNVOS33ParametersWithFD) SizeBytes() int { + return 4 + + (*NVOS33_PARAMETERS)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlNVOS33ParametersWithFD) MarshalBytes(dst []byte) []byte { + dst = p.Params.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlNVOS33ParametersWithFD) UnmarshalBytes(src []byte) []byte { + src = p.Params.UnmarshalUnsafe(src) + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlNVOS33ParametersWithFD) Packed() bool { + return p.Params.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlNVOS33ParametersWithFD) MarshalUnsafe(dst []byte) []byte { + if p.Params.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlNVOS33ParametersWithFD doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlNVOS33ParametersWithFD) UnmarshalUnsafe(src []byte) []byte { + if p.Params.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlNVOS33ParametersWithFD doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlNVOS33ParametersWithFD) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Params.Packed() { + // Type IoctlNVOS33ParametersWithFD doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlNVOS33ParametersWithFD) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlNVOS33ParametersWithFD) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.Params.Packed() { + // Type IoctlNVOS33ParametersWithFD doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlNVOS33ParametersWithFD) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlNVOS33ParametersWithFD) WriteTo(writer io.Writer) (int64, error) { + if !p.Params.Packed() { + // Type IoctlNVOS33ParametersWithFD doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IoctlRegisterFD) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IoctlRegisterFD) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.CtlFD)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IoctlRegisterFD) UnmarshalBytes(src []byte) []byte { + i.CtlFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IoctlRegisterFD) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IoctlRegisterFD) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IoctlRegisterFD) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IoctlRegisterFD) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IoctlRegisterFD) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IoctlRegisterFD) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IoctlRegisterFD) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IoctlRegisterFD) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IoctlSysParams) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IoctlSysParams) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.MemblockSize)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IoctlSysParams) UnmarshalBytes(src []byte) []byte { + i.MemblockSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IoctlSysParams) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IoctlSysParams) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IoctlSysParams) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IoctlSysParams) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IoctlSysParams) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IoctlSysParams) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IoctlSysParams) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IoctlSysParams) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlWaitOpenComplete) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlWaitOpenComplete) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Rc)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.AdapterStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlWaitOpenComplete) UnmarshalBytes(src []byte) []byte { + p.Rc = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.AdapterStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlWaitOpenComplete) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlWaitOpenComplete) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlWaitOpenComplete) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlWaitOpenComplete) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlWaitOpenComplete) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlWaitOpenComplete) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlWaitOpenComplete) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlWaitOpenComplete) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NVOS00_PARAMETERS) SizeBytes() int { + return 4 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NVOS00_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = p.HRoot.MarshalUnsafe(dst) + dst = p.HObjectParent.MarshalUnsafe(dst) + dst = p.HObjectOld.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NVOS00_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = p.HRoot.UnmarshalUnsafe(src) + src = p.HObjectParent.UnmarshalUnsafe(src) + src = p.HObjectOld.UnmarshalUnsafe(src) + p.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NVOS00_PARAMETERS) Packed() bool { + return p.HObjectOld.Packed() && p.HObjectParent.Packed() && p.HRoot.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NVOS00_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if p.HObjectOld.Packed() && p.HObjectParent.Packed() && p.HRoot.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NVOS00_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NVOS00_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if p.HObjectOld.Packed() && p.HObjectParent.Packed() && p.HRoot.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS00_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NVOS00_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HObjectOld.Packed() && p.HObjectParent.Packed() && p.HRoot.Packed() { + // Type NVOS00_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NVOS00_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NVOS00_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HObjectOld.Packed() && p.HObjectParent.Packed() && p.HRoot.Packed() { + // Type NVOS00_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NVOS00_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NVOS00_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !p.HObjectOld.Packed() && p.HObjectParent.Packed() && p.HRoot.Packed() { + // Type NVOS00_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS02_PARAMETERS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*ClassID)(nil).SizeBytes() + + 1*4 + + (*P64)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS02_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HRoot.MarshalUnsafe(dst) + dst = n.HObjectParent.MarshalUnsafe(dst) + dst = n.HObjectNew.MarshalUnsafe(dst) + dst = n.HClass.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + dst = n.PMemory.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Limit)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS02_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HRoot.UnmarshalUnsafe(src) + src = n.HObjectParent.UnmarshalUnsafe(src) + src = n.HObjectNew.UnmarshalUnsafe(src) + src = n.HClass.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + src = n.PMemory.UnmarshalUnsafe(src) + n.Limit = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS02_PARAMETERS) Packed() bool { + return n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PMemory.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS02_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PMemory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS02_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS02_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PMemory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS02_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS02_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PMemory.Packed() { + // Type NVOS02_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS02_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS02_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PMemory.Packed() { + // Type NVOS02_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS02_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS02_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PMemory.Packed() { + // Type NVOS02_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS21_PARAMETERS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*ClassID)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS21_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HRoot.MarshalUnsafe(dst) + dst = n.HObjectParent.MarshalUnsafe(dst) + dst = n.HObjectNew.MarshalUnsafe(dst) + dst = n.HClass.MarshalUnsafe(dst) + dst = n.PAllocParms.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ParamsSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS21_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HRoot.UnmarshalUnsafe(src) + src = n.HObjectParent.UnmarshalUnsafe(src) + src = n.HObjectNew.UnmarshalUnsafe(src) + src = n.HClass.UnmarshalUnsafe(src) + src = n.PAllocParms.UnmarshalUnsafe(src) + n.ParamsSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS21_PARAMETERS) Packed() bool { + return n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS21_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS21_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS21_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS21_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS21_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() { + // Type NVOS21_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS21_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS21_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() { + // Type NVOS21_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS21_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS21_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() { + // Type NVOS21_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS30_PARAMETERS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS30_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.Client.MarshalUnsafe(dst) + dst = n.Device.MarshalUnsafe(dst) + dst = n.Channel.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumChannels)) + dst = dst[4:] + dst = n.Clients.MarshalUnsafe(dst) + dst = n.Devices.MarshalUnsafe(dst) + dst = n.Channels.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Timeout)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS30_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.Client.UnmarshalUnsafe(src) + src = n.Device.UnmarshalUnsafe(src) + src = n.Channel.UnmarshalUnsafe(src) + n.NumChannels = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.Clients.UnmarshalUnsafe(src) + src = n.Devices.UnmarshalUnsafe(src) + src = n.Channels.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Timeout = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS30_PARAMETERS) Packed() bool { + return n.Channel.Packed() && n.Channels.Packed() && n.Client.Packed() && n.Clients.Packed() && n.Device.Packed() && n.Devices.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS30_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.Channel.Packed() && n.Channels.Packed() && n.Client.Packed() && n.Clients.Packed() && n.Device.Packed() && n.Devices.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS30_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS30_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.Channel.Packed() && n.Channels.Packed() && n.Client.Packed() && n.Clients.Packed() && n.Device.Packed() && n.Devices.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS30_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS30_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Channel.Packed() && n.Channels.Packed() && n.Client.Packed() && n.Clients.Packed() && n.Device.Packed() && n.Devices.Packed() { + // Type NVOS30_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS30_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS30_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Channel.Packed() && n.Channels.Packed() && n.Client.Packed() && n.Clients.Packed() && n.Device.Packed() && n.Devices.Packed() { + // Type NVOS30_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS30_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS30_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.Channel.Packed() && n.Channels.Packed() && n.Client.Packed() && n.Clients.Packed() && n.Device.Packed() && n.Devices.Packed() { + // Type NVOS30_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS32_PARAMETERS) SizeBytes() int { + return 26 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*2 + + 1*144 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS32_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HRoot.MarshalUnsafe(dst) + dst = n.HObjectParent.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Function)) + dst = dst[4:] + dst = n.HVASpace.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.IVCHeapNumber)) + dst = dst[2:] + for idx := 0; idx < 2; idx++ { + dst[0] = byte(n.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Total)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Free)) + dst = dst[8:] + for idx := 0; idx < 144; idx++ { + dst[0] = byte(n.Data[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS32_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HRoot.UnmarshalUnsafe(src) + src = n.HObjectParent.UnmarshalUnsafe(src) + n.Function = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HVASpace.UnmarshalUnsafe(src) + n.IVCHeapNumber = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < 2; idx++ { + n.Pad[idx] = src[0] + src = src[1:] + } + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Total = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Free = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < 144; idx++ { + n.Data[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS32_PARAMETERS) Packed() bool { + return n.HObjectParent.Packed() && n.HRoot.Packed() && n.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS32_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HObjectParent.Packed() && n.HRoot.Packed() && n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS32_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS32_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HObjectParent.Packed() && n.HRoot.Packed() && n.HVASpace.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS32_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS32_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HObjectParent.Packed() && n.HRoot.Packed() && n.HVASpace.Packed() { + // Type NVOS32_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS32_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS32_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HObjectParent.Packed() && n.HRoot.Packed() && n.HVASpace.Packed() { + // Type NVOS32_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS32_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS32_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HObjectParent.Packed() && n.HRoot.Packed() && n.HVASpace.Packed() { + // Type NVOS32_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS33_PARAMETERS) SizeBytes() int { + return 24 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS33_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClient.MarshalUnsafe(dst) + dst = n.HDevice.MarshalUnsafe(dst) + dst = n.HMemory.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Length)) + dst = dst[8:] + dst = n.PLinearAddress.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS33_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClient.UnmarshalUnsafe(src) + src = n.HDevice.UnmarshalUnsafe(src) + src = n.HMemory.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = n.PLinearAddress.UnmarshalUnsafe(src) + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS33_PARAMETERS) Packed() bool { + return n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS33_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS33_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS33_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS33_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS33_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + // Type NVOS33_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS33_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS33_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + // Type NVOS33_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS33_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS33_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + // Type NVOS33_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS34_PARAMETERS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS34_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClient.MarshalUnsafe(dst) + dst = n.HDevice.MarshalUnsafe(dst) + dst = n.HMemory.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + dst = n.PLinearAddress.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS34_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClient.UnmarshalUnsafe(src) + src = n.HDevice.UnmarshalUnsafe(src) + src = n.HMemory.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + src = n.PLinearAddress.UnmarshalUnsafe(src) + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS34_PARAMETERS) Packed() bool { + return n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS34_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS34_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS34_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS34_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS34_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + // Type NVOS34_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS34_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS34_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + // Type NVOS34_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS34_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS34_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PLinearAddress.Packed() { + // Type NVOS34_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS39_PARAMETERS) SizeBytes() int { + return 28 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*ClassID)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS39_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HObjectParent.MarshalUnsafe(dst) + dst = n.HSubDevice.MarshalUnsafe(dst) + dst = n.HObjectNew.MarshalUnsafe(dst) + dst = n.HClass.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Selector)) + dst = dst[4:] + dst = n.HMemory.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Limit)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS39_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HObjectParent.UnmarshalUnsafe(src) + src = n.HSubDevice.UnmarshalUnsafe(src) + src = n.HObjectNew.UnmarshalUnsafe(src) + src = n.HClass.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Selector = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HMemory.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Limit = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS39_PARAMETERS) Packed() bool { + return n.HClass.Packed() && n.HMemory.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HSubDevice.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS39_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClass.Packed() && n.HMemory.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HSubDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS39_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS39_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClass.Packed() && n.HMemory.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HSubDevice.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS39_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS39_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HMemory.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HSubDevice.Packed() { + // Type NVOS39_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS39_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS39_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HMemory.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HSubDevice.Packed() { + // Type NVOS39_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS39_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS39_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClass.Packed() && n.HMemory.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HSubDevice.Packed() { + // Type NVOS39_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS46_PARAMETERS) SizeBytes() int { + return 32 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS46_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.Client.MarshalUnsafe(dst) + dst = n.Device.MarshalUnsafe(dst) + dst = n.Dma.MarshalUnsafe(dst) + dst = n.Memory.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DmaOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS46_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.Client.UnmarshalUnsafe(src) + src = n.Device.UnmarshalUnsafe(src) + src = n.Dma.UnmarshalUnsafe(src) + src = n.Memory.UnmarshalUnsafe(src) + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.DmaOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS46_PARAMETERS) Packed() bool { + return n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS46_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS46_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS46_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS46_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS46_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS46_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS46_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS46_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS46_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS46_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS46_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS46_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS46_PARAMETERS_V580) SizeBytes() int { + return 40 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS46_PARAMETERS_V580) MarshalBytes(dst []byte) []byte { + dst = n.Client.MarshalUnsafe(dst) + dst = n.Device.MarshalUnsafe(dst) + dst = n.Dma.MarshalUnsafe(dst) + dst = n.Memory.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Offset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags2)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.KindOverride)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DmaOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS46_PARAMETERS_V580) UnmarshalBytes(src []byte) []byte { + src = n.Client.UnmarshalUnsafe(src) + src = n.Device.UnmarshalUnsafe(src) + src = n.Dma.UnmarshalUnsafe(src) + src = n.Memory.UnmarshalUnsafe(src) + n.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags2 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.KindOverride = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.DmaOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS46_PARAMETERS_V580) Packed() bool { + return n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS46_PARAMETERS_V580) MarshalUnsafe(dst []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS46_PARAMETERS_V580 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS46_PARAMETERS_V580) UnmarshalUnsafe(src []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS46_PARAMETERS_V580 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS46_PARAMETERS_V580) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS46_PARAMETERS_V580 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS46_PARAMETERS_V580) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS46_PARAMETERS_V580) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS46_PARAMETERS_V580 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS46_PARAMETERS_V580) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS46_PARAMETERS_V580) WriteTo(writer io.Writer) (int64, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS46_PARAMETERS_V580 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS47_PARAMETERS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS47_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.Client.MarshalUnsafe(dst) + dst = n.Device.MarshalUnsafe(dst) + dst = n.Dma.MarshalUnsafe(dst) + dst = n.Memory.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DmaOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS47_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.Client.UnmarshalUnsafe(src) + src = n.Device.UnmarshalUnsafe(src) + src = n.Dma.UnmarshalUnsafe(src) + src = n.Memory.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.DmaOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS47_PARAMETERS) Packed() bool { + return n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS47_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS47_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS47_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS47_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS47_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS47_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS47_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS47_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS47_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS47_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS47_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS47_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS47_PARAMETERS_V550) SizeBytes() int { + return 24 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS47_PARAMETERS_V550) MarshalBytes(dst []byte) []byte { + dst = n.Client.MarshalUnsafe(dst) + dst = n.Device.MarshalUnsafe(dst) + dst = n.Dma.MarshalUnsafe(dst) + dst = n.Memory.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.DmaOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS47_PARAMETERS_V550) UnmarshalBytes(src []byte) []byte { + src = n.Client.UnmarshalUnsafe(src) + src = n.Device.UnmarshalUnsafe(src) + src = n.Dma.UnmarshalUnsafe(src) + src = n.Memory.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.DmaOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS47_PARAMETERS_V550) Packed() bool { + return n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS47_PARAMETERS_V550) MarshalUnsafe(dst []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS47_PARAMETERS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS47_PARAMETERS_V550) UnmarshalUnsafe(src []byte) []byte { + if n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS47_PARAMETERS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS47_PARAMETERS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS47_PARAMETERS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS47_PARAMETERS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS47_PARAMETERS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS47_PARAMETERS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS47_PARAMETERS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS47_PARAMETERS_V550) WriteTo(writer io.Writer) (int64, error) { + if !n.Client.Packed() && n.Device.Packed() && n.Dma.Packed() && n.Memory.Packed() { + // Type NVOS47_PARAMETERS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS54_PARAMETERS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS54_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClient.MarshalUnsafe(dst) + dst = n.HObject.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Cmd)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + dst = n.Params.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ParamsSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS54_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClient.UnmarshalUnsafe(src) + src = n.HObject.UnmarshalUnsafe(src) + n.Cmd = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.Params.UnmarshalUnsafe(src) + n.ParamsSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS54_PARAMETERS) Packed() bool { + return n.HClient.Packed() && n.HObject.Packed() && n.Params.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS54_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClient.Packed() && n.HObject.Packed() && n.Params.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS54_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS54_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClient.Packed() && n.HObject.Packed() && n.Params.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS54_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS54_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HObject.Packed() && n.Params.Packed() { + // Type NVOS54_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS54_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS54_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HObject.Packed() && n.Params.Packed() { + // Type NVOS54_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS54_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS54_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClient.Packed() && n.HObject.Packed() && n.Params.Packed() { + // Type NVOS54_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS55_PARAMETERS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS55_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClient.MarshalUnsafe(dst) + dst = n.HParent.MarshalUnsafe(dst) + dst = n.HObject.MarshalUnsafe(dst) + dst = n.HClientSrc.MarshalUnsafe(dst) + dst = n.HObjectSrc.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS55_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClient.UnmarshalUnsafe(src) + src = n.HParent.UnmarshalUnsafe(src) + src = n.HObject.UnmarshalUnsafe(src) + src = n.HClientSrc.UnmarshalUnsafe(src) + src = n.HObjectSrc.UnmarshalUnsafe(src) + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS55_PARAMETERS) Packed() bool { + return n.HClient.Packed() && n.HClientSrc.Packed() && n.HObject.Packed() && n.HObjectSrc.Packed() && n.HParent.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS55_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClient.Packed() && n.HClientSrc.Packed() && n.HObject.Packed() && n.HObjectSrc.Packed() && n.HParent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS55_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS55_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClient.Packed() && n.HClientSrc.Packed() && n.HObject.Packed() && n.HObjectSrc.Packed() && n.HParent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS55_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS55_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HClientSrc.Packed() && n.HObject.Packed() && n.HObjectSrc.Packed() && n.HParent.Packed() { + // Type NVOS55_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS55_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS55_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HClientSrc.Packed() && n.HObject.Packed() && n.HObjectSrc.Packed() && n.HParent.Packed() { + // Type NVOS55_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS55_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS55_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClient.Packed() && n.HClientSrc.Packed() && n.HObject.Packed() && n.HObjectSrc.Packed() && n.HParent.Packed() { + // Type NVOS55_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS56_PARAMETERS) SizeBytes() int { + return 4 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS56_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClient.MarshalUnsafe(dst) + dst = n.HDevice.MarshalUnsafe(dst) + dst = n.HMemory.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + dst = n.POldCPUAddress.MarshalUnsafe(dst) + dst = n.PNewCPUAddress.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS56_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClient.UnmarshalUnsafe(src) + src = n.HDevice.UnmarshalUnsafe(src) + src = n.HMemory.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + src = n.POldCPUAddress.UnmarshalUnsafe(src) + src = n.PNewCPUAddress.UnmarshalUnsafe(src) + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS56_PARAMETERS) Packed() bool { + return n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PNewCPUAddress.Packed() && n.POldCPUAddress.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS56_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PNewCPUAddress.Packed() && n.POldCPUAddress.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS56_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS56_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PNewCPUAddress.Packed() && n.POldCPUAddress.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS56_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS56_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PNewCPUAddress.Packed() && n.POldCPUAddress.Packed() { + // Type NVOS56_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS56_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS56_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PNewCPUAddress.Packed() && n.POldCPUAddress.Packed() { + // Type NVOS56_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS56_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS56_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClient.Packed() && n.HDevice.Packed() && n.HMemory.Packed() && n.PNewCPUAddress.Packed() && n.POldCPUAddress.Packed() { + // Type NVOS56_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS57_PARAMETERS) SizeBytes() int { + return 4 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*RS_SHARE_POLICY)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS57_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClient.MarshalUnsafe(dst) + dst = n.HObject.MarshalUnsafe(dst) + dst = n.SharePolicy.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS57_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClient.UnmarshalUnsafe(src) + src = n.HObject.UnmarshalUnsafe(src) + src = n.SharePolicy.UnmarshalUnsafe(src) + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS57_PARAMETERS) Packed() bool { + return n.HClient.Packed() && n.HObject.Packed() && n.SharePolicy.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS57_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClient.Packed() && n.HObject.Packed() && n.SharePolicy.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS57_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS57_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClient.Packed() && n.HObject.Packed() && n.SharePolicy.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS57_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS57_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HObject.Packed() && n.SharePolicy.Packed() { + // Type NVOS57_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS57_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS57_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClient.Packed() && n.HObject.Packed() && n.SharePolicy.Packed() { + // Type NVOS57_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS57_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS57_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClient.Packed() && n.HObject.Packed() && n.SharePolicy.Packed() { + // Type NVOS57_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVOS64_PARAMETERS) SizeBytes() int { + return 16 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*ClassID)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVOS64_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HRoot.MarshalUnsafe(dst) + dst = n.HObjectParent.MarshalUnsafe(dst) + dst = n.HObjectNew.MarshalUnsafe(dst) + dst = n.HClass.MarshalUnsafe(dst) + dst = n.PAllocParms.MarshalUnsafe(dst) + dst = n.PRightsRequested.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ParamsSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Status)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVOS64_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HRoot.UnmarshalUnsafe(src) + src = n.HObjectParent.UnmarshalUnsafe(src) + src = n.HObjectNew.UnmarshalUnsafe(src) + src = n.HClass.UnmarshalUnsafe(src) + src = n.PAllocParms.UnmarshalUnsafe(src) + src = n.PRightsRequested.UnmarshalUnsafe(src) + n.ParamsSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVOS64_PARAMETERS) Packed() bool { + return n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() && n.PRightsRequested.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVOS64_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() && n.PRightsRequested.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVOS64_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVOS64_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() && n.PRightsRequested.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVOS64_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVOS64_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() && n.PRightsRequested.Packed() { + // Type NVOS64_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVOS64_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVOS64_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() && n.PRightsRequested.Packed() { + // Type NVOS64_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVOS64_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVOS64_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClass.Packed() && n.HObjectNew.Packed() && n.HObjectParent.Packed() && n.HRoot.Packed() && n.PAllocParms.Packed() && n.PRightsRequested.Packed() { + // Type NVOS64_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RMAPIVersion) SizeBytes() int { + return 8 + + 1*64 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RMAPIVersion) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Cmd)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Reply)) + dst = dst[4:] + for idx := 0; idx < 64; idx++ { + dst[0] = byte(r.VersionString[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RMAPIVersion) UnmarshalBytes(src []byte) []byte { + r.Cmd = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + r.Reply = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 64; idx++ { + r.VersionString[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RMAPIVersion) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RMAPIVersion) MarshalUnsafe(dst []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RMAPIVersion) UnmarshalUnsafe(src []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RMAPIVersion) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RMAPIVersion) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RMAPIVersion) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RMAPIVersion) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RMAPIVersion) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (h *Handle) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (h *Handle) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(h.Val)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (h *Handle) UnmarshalBytes(src []byte) []byte { + h.Val = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (h *Handle) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (h *Handle) MarshalUnsafe(dst []byte) []byte { + size := h.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(h), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (h *Handle) UnmarshalUnsafe(src []byte) []byte { + size := h.SizeBytes() + gohacks.Memmove(unsafe.Pointer(h), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (h *Handle) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(h))) + hdr.Len = h.SizeBytes() + hdr.Cap = h.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that h + // must live until the use above. + runtime.KeepAlive(h) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (h *Handle) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return h.CopyOutN(cc, addr, h.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (h *Handle) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(h))) + hdr.Len = h.SizeBytes() + hdr.Cap = h.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that h + // must live until the use above. + runtime.KeepAlive(h) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (h *Handle) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return h.CopyInN(cc, addr, h.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (h *Handle) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(h))) + hdr.Len = h.SizeBytes() + hdr.Cap = h.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that h + // must live until the use above. + runtime.KeepAlive(h) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (n *NvUUID) SizeBytes() int { + return 1 * 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NvUUID) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < 16; idx++ { + dst[0] = byte(n[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NvUUID) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < 16; idx++ { + n[idx] = uint8(src[0]) + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NvUUID) Packed() bool { + // Array newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NvUUID) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&n[0]), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NvUUID) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NvUUID) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NvUUID) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NvUUID) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NvUUID) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NvUUID) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (p *P64) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *P64) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*p)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *P64) UnmarshalBytes(src []byte) []byte { + *p = P64(uint64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *P64) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *P64) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *P64) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *P64) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *P64) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *P64) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *P64) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *P64) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RS_ACCESS_MASK) SizeBytes() int { + return 0 + + 4*SDK_RS_ACCESS_MAX_LIMBS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RS_ACCESS_MASK) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < SDK_RS_ACCESS_MAX_LIMBS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Limbs[idx])) + dst = dst[4:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RS_ACCESS_MASK) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < SDK_RS_ACCESS_MAX_LIMBS; idx++ { + r.Limbs[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RS_ACCESS_MASK) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RS_ACCESS_MASK) MarshalUnsafe(dst []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RS_ACCESS_MASK) UnmarshalUnsafe(src []byte) []byte { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RS_ACCESS_MASK) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RS_ACCESS_MASK) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RS_ACCESS_MASK) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RS_ACCESS_MASK) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RS_ACCESS_MASK) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *RS_SHARE_POLICY) SizeBytes() int { + return 7 + + (*RS_ACCESS_MASK)(nil).SizeBytes() + + 1*1 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *RS_SHARE_POLICY) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(r.Target)) + dst = dst[4:] + dst = r.AccessMask.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(r.Type)) + dst = dst[2:] + dst[0] = byte(r.Action) + dst = dst[1:] + for idx := 0; idx < 1; idx++ { + dst[0] = byte(r.Pad[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *RS_SHARE_POLICY) UnmarshalBytes(src []byte) []byte { + r.Target = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = r.AccessMask.UnmarshalUnsafe(src) + r.Type = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + r.Action = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 1; idx++ { + r.Pad[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *RS_SHARE_POLICY) Packed() bool { + return r.AccessMask.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *RS_SHARE_POLICY) MarshalUnsafe(dst []byte) []byte { + if r.AccessMask.Packed() { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(r), uintptr(size)) + return dst[size:] + } + // Type RS_SHARE_POLICY doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *RS_SHARE_POLICY) UnmarshalUnsafe(src []byte) []byte { + if r.AccessMask.Packed() { + size := r.SizeBytes() + gohacks.Memmove(unsafe.Pointer(r), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type RS_SHARE_POLICY doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (r *RS_SHARE_POLICY) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !r.AccessMask.Packed() { + // Type RS_SHARE_POLICY doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *RS_SHARE_POLICY) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (r *RS_SHARE_POLICY) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !r.AccessMask.Packed() { + // Type RS_SHARE_POLICY doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *RS_SHARE_POLICY) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *RS_SHARE_POLICY) WriteTo(writer io.Writer) (int64, error) { + if !r.AccessMask.Packed() { + // Type RS_SHARE_POLICY doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(r))) + hdr.Len = r.SizeBytes() + hdr.Cap = r.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that r + // must live until the use above. + runtime.KeepAlive(r) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) SizeBytes() int { + return 28 + + (*UvmGpuMappingAttributes)(nil).SizeBytes()*UVM_MAX_GPUS + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + for idx := 0; idx < UVM_MAX_GPUS; idx++ { + dst = p.PerGPUAttributes[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.GPUAttributesCount)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < UVM_MAX_GPUS; idx++ { + src = p.PerGPUAttributes[idx].UnmarshalUnsafe(src) + } + p.GPUAttributesCount = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) Packed() bool { + return p.PerGPUAttributes[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) SizeBytes() int { + return 28 + + (*UvmGpuMappingAttributes)(nil).SizeBytes()*UVM_MAX_GPUS_V2 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + for idx := 0; idx < UVM_MAX_GPUS_V2; idx++ { + dst = p.PerGPUAttributes[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.GPUAttributesCount)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < UVM_MAX_GPUS_V2; idx++ { + src = p.PerGPUAttributes[idx].UnmarshalUnsafe(src) + } + p.GPUAttributesCount = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) Packed() bool { + return p.PerGPUAttributes[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) MarshalUnsafe(dst []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) UnmarshalUnsafe(src []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) WriteTo(writer io.Writer) (int64, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) SizeBytes() int { + return 20 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) SizeBytes() int { + return 12 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RangeGroupID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RangeGroupID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) SizeBytes() int { + return 12 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RangeGroupID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RangeGroupID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) SizeBytes() int { + return 4 + + (*NvUUID)(nil).SizeBytes() + + (*NvUUID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUIDA.MarshalUnsafe(dst) + dst = p.GPUUUIDB.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUIDA.UnmarshalUnsafe(src) + src = p.GPUUUIDB.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) Packed() bool { + return p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_DISABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_DISABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + // Type UVM_DISABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + // Type UVM_DISABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + // Type UVM_DISABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) SizeBytes() int { + return 20 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) SizeBytes() int { + return 4 + + (*NvUUID)(nil).SizeBytes() + + (*NvUUID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUIDA.MarshalUnsafe(dst) + dst = p.GPUUUIDB.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUIDA.UnmarshalUnsafe(src) + src = p.GPUUUIDB.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) Packed() bool { + return p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_ENABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_ENABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + // Type UVM_ENABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + // Type UVM_ENABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUIDA.Packed() && p.GPUUUIDB.Packed() { + // Type UVM_ENABLE_PEER_ACCESS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_FREE_PARAMS) SizeBytes() int { + return 20 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_FREE_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_FREE_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_FREE_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_FREE_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_FREE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_FREE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_FREE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_FREE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_FREE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_FREE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_INITIALIZE_PARAMS) SizeBytes() int { + return 12 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_INITIALIZE_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Flags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_INITIALIZE_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Flags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_INITIALIZE_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_INITIALIZE_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_INITIALIZE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_INITIALIZE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_INITIALIZE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_INITIALIZE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_INITIALIZE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_INITIALIZE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) SizeBytes() int { + return 20 + + (*NvUUID)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.GPUUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) Packed() bool { + return p.GPUUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() { + // Type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) SizeBytes() int { + return 48 + + (*UvmGpuMappingAttributes)(nil).SizeBytes()*UVM_MAX_GPUS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Offset)) + dst = dst[8:] + for idx := 0; idx < UVM_MAX_GPUS; idx++ { + dst = p.PerGPUAttributes[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.GPUAttributesCount)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMCtrlFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.HClient)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.HMemory)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < UVM_MAX_GPUS; idx++ { + src = p.PerGPUAttributes[idx].UnmarshalUnsafe(src) + } + p.GPUAttributesCount = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMCtrlFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.HClient = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.HMemory = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) Packed() bool { + return p.PerGPUAttributes[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) SizeBytes() int { + return 48 + + (*UvmGpuMappingAttributes)(nil).SizeBytes()*UVM_MAX_GPUS_V2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Offset)) + dst = dst[8:] + for idx := 0; idx < UVM_MAX_GPUS_V2; idx++ { + dst = p.PerGPUAttributes[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.GPUAttributesCount)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMCtrlFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.HClient)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.HMemory)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Offset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < UVM_MAX_GPUS_V2; idx++ { + src = p.PerGPUAttributes[idx].UnmarshalUnsafe(src) + } + p.GPUAttributesCount = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMCtrlFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.HClient = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.HMemory = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) Packed() bool { + return p.PerGPUAttributes[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) MarshalUnsafe(dst []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) UnmarshalUnsafe(src []byte) []byte { + if p.PerGPUAttributes[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) WriteTo(writer io.Writer) (int64, error) { + if !p.PerGPUAttributes[0].Packed() { + // Type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MIGRATE_PARAMS) SizeBytes() int { + return 56 + + (*NvUUID)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MIGRATE_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.DestinationUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Flags)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.SemaphoreAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.SemaphorePayload)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.CPUNumaNode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.UserSpaceStart)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.UserSpaceLength)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MIGRATE_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.DestinationUUID.UnmarshalUnsafe(src) + p.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(p._), src[:sizeof(byte)*4]) + src = src[1*(4):] + p.SemaphoreAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.SemaphorePayload = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.CPUNumaNode = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.UserSpaceStart = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.UserSpaceLength = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(p._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MIGRATE_PARAMS) Packed() bool { + return p.DestinationUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MIGRATE_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.DestinationUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_MIGRATE_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MIGRATE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.DestinationUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_MIGRATE_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MIGRATE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MIGRATE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MIGRATE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MIGRATE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MIGRATE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MIGRATE_PARAMS_V550) SizeBytes() int { + return 56 + + (*NvUUID)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MIGRATE_PARAMS_V550) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.DestinationUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Flags)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.SemaphoreAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.SemaphorePayload)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.CPUNumaNode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.UserSpaceStart)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.UserSpaceLength)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MIGRATE_PARAMS_V550) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.DestinationUUID.UnmarshalUnsafe(src) + p.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(p._), src[:sizeof(byte)*4]) + src = src[1*(4):] + p.SemaphoreAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.SemaphorePayload = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.CPUNumaNode = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.UserSpaceStart = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.UserSpaceLength = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: ~ copy([4]byte(p._), src[:sizeof(byte)*4]) + src = src[1*(4):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MIGRATE_PARAMS_V550) Packed() bool { + return p.DestinationUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MIGRATE_PARAMS_V550) MarshalUnsafe(dst []byte) []byte { + if p.DestinationUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_MIGRATE_PARAMS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MIGRATE_PARAMS_V550) UnmarshalUnsafe(src []byte) []byte { + if p.DestinationUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_MIGRATE_PARAMS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MIGRATE_PARAMS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MIGRATE_PARAMS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MIGRATE_PARAMS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_PARAMS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MIGRATE_PARAMS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MIGRATE_PARAMS_V550) WriteTo(writer io.Writer) (int64, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) SizeBytes() int { + return 12 + + (*NvUUID)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RangeGroupID)) + dst = dst[8:] + dst = p.DestinationUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RangeGroupID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.DestinationUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) Packed() bool { + return p.DestinationUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.DestinationUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_MIGRATE_RANGE_GROUP_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.DestinationUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_MIGRATE_RANGE_GROUP_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_RANGE_GROUP_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_RANGE_GROUP_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.DestinationUUID.Packed() { + // Type UVM_MIGRATE_RANGE_GROUP_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_MM_INITIALIZE_PARAMS) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_MM_INITIALIZE_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.UvmFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_MM_INITIALIZE_PARAMS) UnmarshalBytes(src []byte) []byte { + p.UvmFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_MM_INITIALIZE_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_MM_INITIALIZE_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_MM_INITIALIZE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_MM_INITIALIZE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_MM_INITIALIZE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_MM_INITIALIZE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_MM_INITIALIZE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_MM_INITIALIZE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) SizeBytes() int { + return 5 + + (*NvUUID)(nil).SizeBytes() + + 1*3 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + dst[0] = byte(p.PageableMemAccess) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + p.PageableMemAccess = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) Packed() bool { + return p.GPUUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() { + // Type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) SizeBytes() int { + return 5 + + 1*3 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) MarshalBytes(dst []byte) []byte { + dst[0] = byte(p.PageableMemAccess) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) UnmarshalBytes(src []byte) []byte { + p.PageableMemAccess = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_REGISTER_CHANNEL_PARAMS) SizeBytes() int { + return 24 + + (*NvUUID)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 1*4 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_REGISTER_CHANNEL_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMCtrlFD)) + dst = dst[4:] + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HChannel.MarshalUnsafe(dst) + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_REGISTER_CHANNEL_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + p.RMCtrlFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HClient.UnmarshalUnsafe(src) + src = p.HChannel.UnmarshalUnsafe(src) + for idx := 0; idx < 4; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_REGISTER_CHANNEL_PARAMS) Packed() bool { + return p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_REGISTER_CHANNEL_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_REGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_REGISTER_CHANNEL_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_REGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_REGISTER_CHANNEL_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_REGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_REGISTER_CHANNEL_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_REGISTER_CHANNEL_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_REGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_REGISTER_CHANNEL_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_REGISTER_CHANNEL_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_REGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_REGISTER_GPU_PARAMS) SizeBytes() int { + return 13 + + (*NvUUID)(nil).SizeBytes() + + 1*3 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_REGISTER_GPU_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + dst[0] = byte(p.NumaEnabled) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(p.Pad[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.NumaNodeID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMCtrlFD)) + dst = dst[4:] + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HSMCPartRef.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_REGISTER_GPU_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + p.NumaEnabled = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + p.Pad[idx] = src[0] + src = src[1:] + } + p.NumaNodeID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.RMCtrlFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HClient.UnmarshalUnsafe(src) + src = p.HSMCPartRef.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_REGISTER_GPU_PARAMS) Packed() bool { + return p.GPUUUID.Packed() && p.HClient.Packed() && p.HSMCPartRef.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_REGISTER_GPU_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() && p.HClient.Packed() && p.HSMCPartRef.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_REGISTER_GPU_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_REGISTER_GPU_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() && p.HClient.Packed() && p.HSMCPartRef.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_REGISTER_GPU_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_REGISTER_GPU_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HClient.Packed() && p.HSMCPartRef.Packed() { + // Type UVM_REGISTER_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_REGISTER_GPU_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_REGISTER_GPU_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HClient.Packed() && p.HSMCPartRef.Packed() { + // Type UVM_REGISTER_GPU_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_REGISTER_GPU_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_REGISTER_GPU_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() && p.HClient.Packed() && p.HSMCPartRef.Packed() { + // Type UVM_REGISTER_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) SizeBytes() int { + return 8 + + (*NvUUID)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMCtrlFD)) + dst = dst[4:] + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HVASpace.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + p.RMCtrlFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HClient.UnmarshalUnsafe(src) + src = p.HVASpace.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) Packed() bool { + return p.GPUUUID.Packed() && p.HClient.Packed() && p.HVASpace.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() && p.HClient.Packed() && p.HVASpace.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_REGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() && p.HClient.Packed() && p.HVASpace.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_REGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HClient.Packed() && p.HVASpace.Packed() { + // Type UVM_REGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HClient.Packed() && p.HVASpace.Packed() { + // Type UVM_REGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() && p.HClient.Packed() && p.HVASpace.Packed() { + // Type UVM_REGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) SizeBytes() int { + return 20 + + (*NvUUID)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.PreferredLocation.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.PreferredLocation.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) Packed() bool { + return p.PreferredLocation.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.PreferredLocation.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_SET_PREFERRED_LOCATION_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.PreferredLocation.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_SET_PREFERRED_LOCATION_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PreferredLocation.Packed() { + // Type UVM_SET_PREFERRED_LOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PreferredLocation.Packed() { + // Type UVM_SET_PREFERRED_LOCATION_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.PreferredLocation.Packed() { + // Type UVM_SET_PREFERRED_LOCATION_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) SizeBytes() int { + return 24 + + (*NvUUID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.PreferredLocation.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.PreferredCPUNumaNode)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.PreferredLocation.UnmarshalUnsafe(src) + p.PreferredCPUNumaNode = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) Packed() bool { + return p.PreferredLocation.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) MarshalUnsafe(dst []byte) []byte { + if p.PreferredLocation.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) UnmarshalUnsafe(src []byte) []byte { + if p.PreferredLocation.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PreferredLocation.Packed() { + // Type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.PreferredLocation.Packed() { + // Type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) WriteTo(writer io.Writer) (int64, error) { + if !p.PreferredLocation.Packed() { + // Type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_SET_RANGE_GROUP_PARAMS) SizeBytes() int { + return 28 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_SET_RANGE_GROUP_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RangeGroupID)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_SET_RANGE_GROUP_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RangeGroupID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_SET_RANGE_GROUP_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_SET_RANGE_GROUP_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_SET_RANGE_GROUP_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_SET_RANGE_GROUP_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_SET_RANGE_GROUP_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_SET_RANGE_GROUP_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_SET_RANGE_GROUP_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_SET_RANGE_GROUP_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) SizeBytes() int { + return 36 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Buffer)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.TargetVA)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.BytesRead)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Buffer = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.TargetVA = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.BytesRead = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) SizeBytes() int { + return 36 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Buffer)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Size)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.TargetVA)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.BytesWritten)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Buffer = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.TargetVA = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.BytesWritten = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) SizeBytes() int { + return 20 + + (*NvUUID)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.GPUUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNMAP_EXTERNAL_PARAMS) Packed() bool { + return p.GPUUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_UNMAP_EXTERNAL_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_UNMAP_EXTERNAL_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNMAP_EXTERNAL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNMAP_EXTERNAL_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNMAP_EXTERNAL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) SizeBytes() int { + return 4 + + (*NvUUID)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HChannel.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + src = p.HClient.UnmarshalUnsafe(src) + src = p.HChannel.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) Packed() bool { + return p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_UNREGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_UNREGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_UNREGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_UNREGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() && p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_UNREGISTER_CHANNEL_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNREGISTER_GPU_PARAMS) SizeBytes() int { + return 4 + + (*NvUUID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNREGISTER_GPU_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNREGISTER_GPU_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNREGISTER_GPU_PARAMS) Packed() bool { + return p.GPUUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNREGISTER_GPU_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_UNREGISTER_GPU_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNREGISTER_GPU_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_UNREGISTER_GPU_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNREGISTER_GPU_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNREGISTER_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNREGISTER_GPU_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNREGISTER_GPU_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNREGISTER_GPU_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNREGISTER_GPU_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNREGISTER_GPU_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNREGISTER_GPU_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) SizeBytes() int { + return 4 + + (*NvUUID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) MarshalBytes(dst []byte) []byte { + dst = p.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) UnmarshalBytes(src []byte) []byte { + src = p.GPUUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) Packed() bool { + return p.GPUUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_UNREGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.GPUUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_UNREGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNREGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNREGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.GPUUUID.Packed() { + // Type UVM_UNREGISTER_GPU_VASPACE_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) SizeBytes() int { + return 20 + + (*NvUUID)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.AccessedByUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.AccessedByUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) Packed() bool { + return p.AccessedByUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.AccessedByUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_UNSET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.AccessedByUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_UNSET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.AccessedByUUID.Packed() { + // Type UVM_UNSET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.AccessedByUUID.Packed() { + // Type UVM_UNSET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.AccessedByUUID.Packed() { + // Type UVM_UNSET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) SizeBytes() int { + return 20 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) SizeBytes() int { + return 20 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (u *UvmGpuMappingAttributes) SizeBytes() int { + return 20 + + (*NvUUID)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *UvmGpuMappingAttributes) MarshalBytes(dst []byte) []byte { + dst = u.GPUUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(u.GPUMappingType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(u.GPUCachingType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(u.GPUFormatType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(u.GPUElementBits)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(u.GPUCompressionType)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *UvmGpuMappingAttributes) UnmarshalBytes(src []byte) []byte { + src = u.GPUUUID.UnmarshalUnsafe(src) + u.GPUMappingType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + u.GPUCachingType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + u.GPUFormatType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + u.GPUElementBits = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + u.GPUCompressionType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *UvmGpuMappingAttributes) Packed() bool { + return u.GPUUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *UvmGpuMappingAttributes) MarshalUnsafe(dst []byte) []byte { + if u.GPUUUID.Packed() { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] + } + // Type UvmGpuMappingAttributes doesn't have a packed layout in memory, fallback to MarshalBytes. + return u.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *UvmGpuMappingAttributes) UnmarshalUnsafe(src []byte) []byte { + if u.GPUUUID.Packed() { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UvmGpuMappingAttributes doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return u.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *UvmGpuMappingAttributes) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !u.GPUUUID.Packed() { + // Type UvmGpuMappingAttributes doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(u.SizeBytes()) // escapes: okay. + u.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *UvmGpuMappingAttributes) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *UvmGpuMappingAttributes) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !u.GPUUUID.Packed() { + // Type UvmGpuMappingAttributes doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(u.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + u.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *UvmGpuMappingAttributes) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *UvmGpuMappingAttributes) WriteTo(writer io.Writer) (int64, error) { + if !u.GPUUUID.Packed() { + // Type UvmGpuMappingAttributes doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, u.SizeBytes()) + u.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} diff --git a/pkg/abi/nvgpu/nvgpu_state_autogen.go b/pkg/abi/nvgpu/nvgpu_state_autogen.go new file mode 100644 index 0000000..dd5dcbd --- /dev/null +++ b/pkg/abi/nvgpu/nvgpu_state_autogen.go @@ -0,0 +1,114 @@ +// automatically generated by stateify. + +package nvgpu + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (n *NVOS64_PARAMETERS) StateTypeName() string { + return "pkg/abi/nvgpu.NVOS64_PARAMETERS" +} + +func (n *NVOS64_PARAMETERS) StateFields() []string { + return []string{ + "HRoot", + "HObjectParent", + "HObjectNew", + "HClass", + "PAllocParms", + "PRightsRequested", + "ParamsSize", + "Flags", + "Status", + } +} + +func (n *NVOS64_PARAMETERS) beforeSave() {} + +// +checklocksignore +func (n *NVOS64_PARAMETERS) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.HRoot) + stateSinkObject.Save(1, &n.HObjectParent) + stateSinkObject.Save(2, &n.HObjectNew) + stateSinkObject.Save(3, &n.HClass) + stateSinkObject.Save(4, &n.PAllocParms) + stateSinkObject.Save(5, &n.PRightsRequested) + stateSinkObject.Save(6, &n.ParamsSize) + stateSinkObject.Save(7, &n.Flags) + stateSinkObject.Save(8, &n.Status) +} + +func (n *NVOS64_PARAMETERS) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NVOS64_PARAMETERS) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.HRoot) + stateSourceObject.Load(1, &n.HObjectParent) + stateSourceObject.Load(2, &n.HObjectNew) + stateSourceObject.Load(3, &n.HClass) + stateSourceObject.Load(4, &n.PAllocParms) + stateSourceObject.Load(5, &n.PRightsRequested) + stateSourceObject.Load(6, &n.ParamsSize) + stateSourceObject.Load(7, &n.Flags) + stateSourceObject.Load(8, &n.Status) +} + +func (h *Handle) StateTypeName() string { + return "pkg/abi/nvgpu.Handle" +} + +func (h *Handle) StateFields() []string { + return []string{ + "Val", + } +} + +func (h *Handle) beforeSave() {} + +// +checklocksignore +func (h *Handle) StateSave(stateSinkObject state.Sink) { + h.beforeSave() + stateSinkObject.Save(0, &h.Val) +} + +func (h *Handle) afterLoad(context.Context) {} + +// +checklocksignore +func (h *Handle) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &h.Val) +} + +func (r *RS_ACCESS_MASK) StateTypeName() string { + return "pkg/abi/nvgpu.RS_ACCESS_MASK" +} + +func (r *RS_ACCESS_MASK) StateFields() []string { + return []string{ + "Limbs", + } +} + +func (r *RS_ACCESS_MASK) beforeSave() {} + +// +checklocksignore +func (r *RS_ACCESS_MASK) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.Limbs) +} + +func (r *RS_ACCESS_MASK) afterLoad(context.Context) {} + +// +checklocksignore +func (r *RS_ACCESS_MASK) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.Limbs) +} + +func init() { + state.Register((*NVOS64_PARAMETERS)(nil)) + state.Register((*Handle)(nil)) + state.Register((*RS_ACCESS_MASK)(nil)) +} diff --git a/pkg/abi/nvgpu/nvgpu_unsafe_abi_autogen_unsafe.go b/pkg/abi/nvgpu/nvgpu_unsafe_abi_autogen_unsafe.go new file mode 100644 index 0000000..138ea25 --- /dev/null +++ b/pkg/abi/nvgpu/nvgpu_unsafe_abi_autogen_unsafe.go @@ -0,0 +1,3 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +package nvgpu diff --git a/pkg/abi/nvgpu/nvgpu_unsafe_state_autogen.go b/pkg/abi/nvgpu/nvgpu_unsafe_state_autogen.go new file mode 100644 index 0000000..9c4c9d4 --- /dev/null +++ b/pkg/abi/nvgpu/nvgpu_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package nvgpu diff --git a/pkg/abi/nvgpu/status.go b/pkg/abi/nvgpu/status.go new file mode 100644 index 0000000..a781fb0 --- /dev/null +++ b/pkg/abi/nvgpu/status.go @@ -0,0 +1,26 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// Status codes, from src/common/sdk/nvidia/inc/nvstatuscodes.h. +const ( + NV_OK = 0x00000000 + NV_ERR_INVALID_ADDRESS = 0x0000001e + NV_ERR_INVALID_ARGUMENT = 0x0000001f + NV_ERR_INVALID_CLASS = 0x00000022 + NV_ERR_INVALID_CLIENT = 0x00000023 + NV_ERR_INVALID_LIMIT = 0x0000002e + NV_ERR_NOT_SUPPORTED = 0x00000056 +) diff --git a/pkg/abi/nvgpu/uvm.go b/pkg/abi/nvgpu/uvm.go new file mode 100644 index 0000000..0226a4e --- /dev/null +++ b/pkg/abi/nvgpu/uvm.go @@ -0,0 +1,778 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// UVM ioctl commands. +const ( + // From kernel-open/nvidia-uvm/uvm_linux_ioctl.h: + UVM_INITIALIZE = 0x30000001 + UVM_DEINITIALIZE = 0x30000002 + + // From kernel-open/nvidia-uvm/uvm_ioctl.h: + UVM_CREATE_RANGE_GROUP = 23 + UVM_DESTROY_RANGE_GROUP = 24 + UVM_REGISTER_GPU_VASPACE = 25 + UVM_UNREGISTER_GPU_VASPACE = 26 + UVM_REGISTER_CHANNEL = 27 + UVM_UNREGISTER_CHANNEL = 28 + UVM_ENABLE_PEER_ACCESS = 29 + UVM_DISABLE_PEER_ACCESS = 30 + UVM_SET_RANGE_GROUP = 31 + UVM_MAP_EXTERNAL_ALLOCATION = 33 + UVM_FREE = 34 + UVM_REGISTER_GPU = 37 + UVM_UNREGISTER_GPU = 38 + UVM_PAGEABLE_MEM_ACCESS = 39 + UVM_SET_PREFERRED_LOCATION = 42 + UVM_UNSET_PREFERRED_LOCATION = 43 + UVM_DISABLE_READ_DUPLICATION = 45 + UVM_UNSET_ACCESSED_BY = 47 + UVM_MIGRATE = 51 + UVM_MIGRATE_RANGE_GROUP = 53 + UVM_TOOLS_READ_PROCESS_MEMORY = 62 + UVM_TOOLS_WRITE_PROCESS_MEMORY = 63 + UVM_MAP_DYNAMIC_PARALLELISM_REGION = 65 + UVM_UNMAP_EXTERNAL = 66 + UVM_ALLOC_SEMAPHORE_POOL = 68 + UVM_PAGEABLE_MEM_ACCESS_ON_GPU = 70 + UVM_VALIDATE_VA_RANGE = 72 + UVM_CREATE_EXTERNAL_RANGE = 73 + UVM_MM_INITIALIZE = 75 +) + +// +marshal +type UVM_INITIALIZE_PARAMS struct { + Flags uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_INITIALIZE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_INITIALIZE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// UVM_INITIALIZE_PARAMS flags, from kernel-open/nvidia-uvm/uvm_types.h. +const ( + UVM_INIT_FLAGS_MULTI_PROCESS_SHARING_MODE = 0x2 +) + +// +marshal +type UVM_CREATE_RANGE_GROUP_PARAMS struct { + RangeGroupID uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_CREATE_RANGE_GROUP_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_DESTROY_RANGE_GROUP_PARAMS struct { + RangeGroupID uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_REGISTER_GPU_VASPACE_PARAMS struct { + GPUUUID NvUUID + RMCtrlFD int32 + HClient Handle + HVASpace Handle + RMStatus uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) GetFrontendFD() int32 { + return p.RMCtrlFD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) SetFrontendFD(fd int32) { + p.RMCtrlFD = fd +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_UNREGISTER_GPU_VASPACE_PARAMS struct { + GPUUUID NvUUID + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_REGISTER_CHANNEL_PARAMS struct { + GPUUUID NvUUID + RMCtrlFD int32 + HClient Handle + HChannel Handle + Pad [4]byte + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *UVM_REGISTER_CHANNEL_PARAMS) GetFrontendFD() int32 { + return p.RMCtrlFD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *UVM_REGISTER_CHANNEL_PARAMS) SetFrontendFD(fd int32) { + p.RMCtrlFD = fd +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_REGISTER_CHANNEL_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_REGISTER_CHANNEL_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_UNREGISTER_CHANNEL_PARAMS struct { + GPUUUID NvUUID + HClient Handle + HChannel Handle + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_ENABLE_PEER_ACCESS_PARAMS struct { + GPUUUIDA NvUUID + GPUUUIDB NvUUID + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_DISABLE_PEER_ACCESS_PARAMS struct { + GPUUUIDA NvUUID + GPUUUIDB NvUUID + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_SET_RANGE_GROUP_PARAMS struct { + RangeGroupID uint64 + RequestedBase uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_SET_RANGE_GROUP_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_SET_RANGE_GROUP_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS struct { + Base uint64 + Length uint64 + Offset uint64 + PerGPUAttributes [UVM_MAX_GPUS]UvmGpuMappingAttributes + GPUAttributesCount uint64 + RMCtrlFD int32 + HClient uint32 // These are treated like NvHandle, but the driver uses NvU32. + HMemory uint32 // These are treated like NvHandle, but the driver uses NvU32. + RMStatus uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) GetFrontendFD() int32 { + return p.RMCtrlFD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) SetFrontendFD(fd int32) { + p.RMCtrlFD = fd +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 struct { + Base uint64 + Length uint64 + Offset uint64 + PerGPUAttributes [UVM_MAX_GPUS_V2]UvmGpuMappingAttributes + GPUAttributesCount uint64 + RMCtrlFD int32 + HClient uint32 // These are treated like NvHandle, but the driver uses NvU32. + HMemory uint32 // These are treated like NvHandle, but the driver uses NvU32. + RMStatus uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) GetFrontendFD() int32 { + return p.RMCtrlFD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) SetFrontendFD(fd int32) { + p.RMCtrlFD = fd +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_FREE_PARAMS struct { + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_FREE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_FREE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_REGISTER_GPU_PARAMS struct { + GPUUUID NvUUID + NumaEnabled uint8 + Pad [3]byte + NumaNodeID int32 + RMCtrlFD int32 + HClient Handle + HSMCPartRef Handle + RMStatus uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *UVM_REGISTER_GPU_PARAMS) GetFrontendFD() int32 { + return p.RMCtrlFD +} + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *UVM_REGISTER_GPU_PARAMS) SetFrontendFD(fd int32) { + p.RMCtrlFD = fd +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_REGISTER_GPU_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_REGISTER_GPU_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_UNREGISTER_GPU_PARAMS struct { + GPUUUID NvUUID + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNREGISTER_GPU_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNREGISTER_GPU_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_PAGEABLE_MEM_ACCESS_PARAMS struct { + PageableMemAccess uint8 + Pad [3]byte + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_SET_PREFERRED_LOCATION_PARAMS struct { + RequestedBase uint64 + Length uint64 + PreferredLocation NvUUID + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 struct { + RequestedBase uint64 + Length uint64 + PreferredLocation NvUUID + PreferredCPUNumaNode int32 + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_UNSET_PREFERRED_LOCATION_PARAMS struct { + RequestedBase uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_DISABLE_READ_DUPLICATION_PARAMS struct { + RequestedBase uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_UNSET_ACCESSED_BY_PARAMS struct { + RequestedBase uint64 + Length uint64 + AccessedByUUID NvUUID + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNSET_ACCESSED_BY_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_MIGRATE_PARAMS struct { + Base uint64 + Length uint64 + DestinationUUID NvUUID + Flags uint32 + _ [4]byte + SemaphoreAddress uint64 + SemaphorePayload uint32 + CPUNumaNode uint32 + UserSpaceStart uint64 + UserSpaceLength uint64 + RMStatus uint32 + _ [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MIGRATE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MIGRATE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// UVM_MIGRATE_PARAMS_V550 is the updated version of +// UVM_MIGRATE_PARAMS since 550.40.07. +// +// +marshal +type UVM_MIGRATE_PARAMS_V550 struct { + Base uint64 + Length uint64 + DestinationUUID NvUUID + Flags uint32 + _ [4]byte + SemaphoreAddress uint64 + SemaphorePayload uint32 + CPUNumaNode int32 + UserSpaceStart uint64 + UserSpaceLength uint64 + RMStatus uint32 + _ [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MIGRATE_PARAMS_V550) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MIGRATE_PARAMS_V550) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_MIGRATE_RANGE_GROUP_PARAMS struct { + RangeGroupID uint64 + DestinationUUID NvUUID + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS struct { + Buffer uint64 + Size uint64 + TargetVA uint64 + BytesRead uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS struct { + Buffer uint64 + Size uint64 + TargetVA uint64 + BytesWritten uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS struct { + Base uint64 + Length uint64 + GPUUUID NvUUID + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_UNMAP_EXTERNAL_PARAMS struct { + Base uint64 + Length uint64 + GPUUUID NvUUID + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNMAP_EXTERNAL_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_ALLOC_SEMAPHORE_POOL_PARAMS struct { + Base uint64 + Length uint64 + PerGPUAttributes [UVM_MAX_GPUS]UvmGpuMappingAttributes + GPUAttributesCount uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 struct { + Base uint64 + Length uint64 + PerGPUAttributes [UVM_MAX_GPUS_V2]UvmGpuMappingAttributes + GPUAttributesCount uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS struct { + GPUUUID NvUUID + PageableMemAccess uint8 + Pad [3]byte + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_VALIDATE_VA_RANGE_PARAMS struct { + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_VALIDATE_VA_RANGE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_CREATE_EXTERNAL_RANGE_PARAMS struct { + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// +marshal +type UVM_MM_INITIALIZE_PARAMS struct { + UvmFD int32 + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_MM_INITIALIZE_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_MM_INITIALIZE_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + +// From kernel-open/nvidia-uvm/uvm_types.h: + +const ( + UVM_MAX_GPUS = NV_MAX_DEVICES + UVM_MAX_GPUS_V2 = NV_MAX_DEVICES * NV_MAX_SUBDEVICES +) + +// +marshal +type UvmGpuMappingAttributes struct { + GPUUUID NvUUID + GPUMappingType uint32 + GPUCachingType uint32 + GPUFormatType uint32 + GPUElementBits uint32 + GPUCompressionType uint32 +} diff --git a/pkg/abi/sentry/sentry.go b/pkg/abi/sentry/sentry.go new file mode 100644 index 0000000..84f765e --- /dev/null +++ b/pkg/abi/sentry/sentry.go @@ -0,0 +1,16 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sentry contains ABI-related constants for the gVisor sentry. +package sentry diff --git a/pkg/abi/sentry/sentry_state_autogen.go b/pkg/abi/sentry/sentry_state_autogen.go new file mode 100644 index 0000000..bb49925 --- /dev/null +++ b/pkg/abi/sentry/sentry_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package sentry diff --git a/pkg/abi/sentry/syscall.go b/pkg/abi/sentry/syscall.go new file mode 100644 index 0000000..71391b5 --- /dev/null +++ b/pkg/abi/sentry/syscall.go @@ -0,0 +1,20 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sentry + +// MaxSyscallNum is the largest-numbered syscall that is supported. +// Having this as a constant allows allocating per-syscall data structures +// that are of fixed size throughout the codebase. +const MaxSyscallNum = 2000 diff --git a/pkg/abi/tpu/tpu.go b/pkg/abi/tpu/tpu.go new file mode 100644 index 0000000..e31c647 --- /dev/null +++ b/pkg/abi/tpu/tpu.go @@ -0,0 +1,155 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tpu defines constants used to interact with TPUs. The constants are +// derived from those listed in https://github.com/tensorflow/tpu/blob/master/tools/driver/drivers/char/tpu_common +package tpu + +const ( + // SizeOfTPUV4InterruptList is the total number of valid + // (BAR Index, Register Offset) pairs. + SizeOfTPUV4InterruptList = uint64(45) + + // NumberOfTPUV4PageTables is the number of gasket page tables used by the + // TPU V4 driver. + NumberOfTPUV4PageTables = uint64(1) + + // TPUV4DeviceID is the PCI device ID of TPU V4 hardware. + TPUV4DeviceID = 0x005E + + // SizeOfTPUV4liteInterruptList is the total number of valid + // (BAR Index, Register Offset) pairs. + SizeOfTPUV4liteInterruptList = uint64(37) + + // NumberOfTPUV4litePageTables is the number of gasket page tables used by the + // TPU V4 driver + NumberOfTPUV4litePageTables = uint64(1) + + // TPUV4liteDeviceID is the PCI device ID of TPU V4lite hardware. + TPUV4liteDeviceID = 0x0056 + + // TPUV5eDeviceID is the PCI device ID of TPU V5e hardware. + TPUV5eDeviceID = 0x0063 + + // TPUV5pDeviceID is the PCI device ID of TPU V5p hardware. + TPUV5pDeviceID = 0x0062 + + // TPUV6eDeviceID is the PCI device ID of TPU V6e hardware. + TPUV6eDeviceID = 0x006f + + // TPUV6ePFDeviceID is PCI device ID of TPU V6e hardware PF. + TPUV6ePFDeviceID = 0x006e + + // TPUV6pDeviceID is the PCI device ID of TPU V6p hardware. + TPUV6pDeviceID = 0x0075 + + // TPUV6pPFDeviceID is the PCI device ID of TPU V6p hardware PF. + TPUV6pPFDeviceID = 0x0076 +) + +// TPUV4InterruptsMap maps BAR indices to valid register offsets. +var ( + TPUV4InterruptsMap = map[uint64]map[uint64]struct{}{ + 2: { + 0x15b0008: {}, + 0x15b0000: {}, + 0x16b0008: {}, + 0x16b0000: {}, + 0x17b0008: {}, + 0x17b0000: {}, + 0x18b0008: {}, + 0x18b0000: {}, + 0x19b0020: {}, + 0x19b0000: {}, + 0x19b0008: {}, + 0x19b0010: {}, + 0x19b0018: {}, + 0x1ab0020: {}, + 0x1ab0000: {}, + 0x1ab0008: {}, + 0x1ab0010: {}, + 0x1ab0018: {}, + 0x4720000: {}, + 0x1bb0000: {}, + 0x1bb0008: {}, + 0x1bb0010: {}, + 0x1bb0018: {}, + 0x90000: {}, + 0xb0000: {}, + 0xd0000: {}, + 0xf0000: {}, + 0x110000: {}, + 0x130000: {}, + 0x150000: {}, + 0x170000: {}, + 0x190000: {}, + 0x1b0000: {}, + 0x1d0000: {}, + 0x1f0000: {}, + 0x210000: {}, + 0x230000: {}, + 0x250000: {}, + 0x270000: {}, + 0x290000: {}, + 0x2b0000: {}, + 0x2d0000: {}, + 0x2f0000: {}, + 0x310000: {}, + 0x4720018: {}, + }, + } + + // TPUV4liteInterruptsMap maps BAR indices to valid register offsets. + TPUV4liteInterruptsMap = map[uint64]map[uint64]struct{}{ + 2: { + 0x19b0020: {}, + 0x19b0000: {}, + 0x19b0008: {}, + 0x19b0010: {}, + 0x19b0018: {}, + 0x1ab0020: {}, + 0x1ab0000: {}, + 0x1ab0008: {}, + 0x1ab0010: {}, + 0x1ab0018: {}, + 0x4720000: {}, + 0x1bb0000: {}, + 0x1bb0008: {}, + 0x1bb0010: {}, + 0x1bb0018: {}, + 0x90000: {}, + 0xb0000: {}, + 0xd0000: {}, + 0xf0000: {}, + 0x110000: {}, + 0x130000: {}, + 0x150000: {}, + 0x170000: {}, + 0x190000: {}, + 0x1b0000: {}, + 0x1d0000: {}, + 0x1f0000: {}, + 0x210000: {}, + 0x230000: {}, + 0x250000: {}, + 0x270000: {}, + 0x290000: {}, + 0x2b0000: {}, + 0x2d0000: {}, + 0x2f0000: {}, + 0x310000: {}, + 0x4720018: {}, + }, + } +) diff --git a/pkg/abi/tpu/tpu_state_autogen.go b/pkg/abi/tpu/tpu_state_autogen.go new file mode 100644 index 0000000..86a07da --- /dev/null +++ b/pkg/abi/tpu/tpu_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package tpu diff --git a/pkg/atomicbitops/32b_32bit.go b/pkg/atomicbitops/32b_32bit.go new file mode 100644 index 0000000..dffd27a --- /dev/null +++ b/pkg/atomicbitops/32b_32bit.go @@ -0,0 +1,289 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm || mips || mipsle || 386 +// +build arm mips mipsle 386 + +package atomicbitops + +import ( + "sync/atomic" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Note that this file is *identical* to 32b_64bit.go, as go_stateify gets +// confused about build tags if these are not separated. + +// LINT.IfChange + +// Int32 is an atomic int32. +// +// The default value is zero. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// +stateify savable +type Int32 struct { + _ sync.NoCopy + value int32 +} + +// FromInt32 returns an Int32 initialized to value v. +// +//go:nosplit +func FromInt32(v int32) Int32 { + return Int32{value: v} +} + +// Load is analogous to atomic.LoadInt32. +// +//go:nosplit +func (i *Int32) Load() int32 { + return atomic.LoadInt32(&i.value) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int32) RacyLoad() int32 { + return i.value +} + +// Store is analogous to atomic.StoreInt32. +// +//go:nosplit +func (i *Int32) Store(v int32) { + atomic.StoreInt32(&i.value, v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int32) RacyStore(v int32) { + i.value = v +} + +// Add is analogous to atomic.AddInt32. +// +//go:nosplit +func (i *Int32) Add(v int32) int32 { + return atomic.AddInt32(&i.value, v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int32) RacyAdd(v int32) int32 { + i.value += v + return i.value +} + +// Swap is analogous to atomic.SwapInt32. +// +//go:nosplit +func (i *Int32) Swap(v int32) int32 { + return atomic.SwapInt32(&i.value, v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapInt32. +// +//go:nosplit +func (i *Int32) CompareAndSwap(oldVal, newVal int32) bool { + return atomic.CompareAndSwapInt32(&i.value, oldVal, newVal) +} + +//go:nosplit +func (i *Int32) ptr() *int32 { + return &i.value +} + +// Uint32 is an atomic uint32. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// See aligned_unsafe.go in this directory for justification. +// +// +stateify savable +type Uint32 struct { + _ sync.NoCopy + value uint32 +} + +// FromUint32 returns an Uint32 initialized to value v. +// +//go:nosplit +func FromUint32(v uint32) Uint32 { + return Uint32{value: v} +} + +// Load is analogous to atomic.LoadUint32. +// +//go:nosplit +func (u *Uint32) Load() uint32 { + return atomic.LoadUint32(&u.value) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint32) RacyLoad() uint32 { + return u.value +} + +// Store is analogous to atomic.StoreUint32. +// +//go:nosplit +func (u *Uint32) Store(v uint32) { + atomic.StoreUint32(&u.value, v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint32) RacyStore(v uint32) { + u.value = v +} + +// Add is analogous to atomic.AddUint32. +// +//go:nosplit +func (u *Uint32) Add(v uint32) uint32 { + return atomic.AddUint32(&u.value, v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint32) RacyAdd(v uint32) uint32 { + u.value += v + return u.value +} + +// Swap is analogous to atomic.SwapUint32. +// +//go:nosplit +func (u *Uint32) Swap(v uint32) uint32 { + return atomic.SwapUint32(&u.value, v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapUint32. +// +//go:nosplit +func (u *Uint32) CompareAndSwap(oldVal, newVal uint32) bool { + return atomic.CompareAndSwapUint32(&u.value, oldVal, newVal) +} + +//go:nosplit +func (u *Uint32) ptr() *uint32 { + return &u.value +} + +// Bool is an atomic Boolean. +// +// It is implemented by a Uint32, with value 0 indicating false, and 1 +// indicating true. +// +// +stateify savable +type Bool struct { + Uint32 +} + +// b32 returns a uint32 0 or 1 representing b. +func b32(b bool) uint32 { + if b { + return 1 + } + return 0 +} + +// FromBool returns a Bool initialized to value val. +// +//go:nosplit +func FromBool(val bool) Bool { + return Bool{ + Uint32: FromUint32(b32(val)), + } +} + +// Load is analogous to atomic.LoadBool, if such a thing existed. +// +//go:nosplit +func (b *Bool) Load() bool { + return b.Uint32.Load() != 0 +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (b *Bool) RacyLoad() bool { + return b.Uint32.RacyLoad() != 0 +} + +// Store is analogous to atomic.StoreBool, if such a thing existed. +// +//go:nosplit +func (b *Bool) Store(val bool) { + b.Uint32.Store(b32(val)) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (b *Bool) RacyStore(val bool) { + b.Uint32.RacyStore(b32(val)) +} + +// Swap is analogous to atomic.SwapBool, if such a thing existed. +// +//go:nosplit +func (b *Bool) Swap(val bool) bool { + return b.Uint32.Swap(b32(val)) != 0 +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapBool, if such a thing +// existed. +// +//go:nosplit +func (b *Bool) CompareAndSwap(oldVal, newVal bool) bool { + return b.Uint32.CompareAndSwap(b32(oldVal), b32(newVal)) +} + +// LINT.ThenChange(32b_64bit.go) diff --git a/pkg/atomicbitops/32b_64bit.go b/pkg/atomicbitops/32b_64bit.go new file mode 100644 index 0000000..25db96f --- /dev/null +++ b/pkg/atomicbitops/32b_64bit.go @@ -0,0 +1,289 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !arm && !mips && !mipsle && !386 +// +build !arm,!mips,!mipsle,!386 + +package atomicbitops + +import ( + "sync/atomic" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Note that this file is *identical* to 32b_32bit.go, as go_stateify gets +// confused about build tags if these are not separated. + +// LINT.IfChange + +// Int32 is an atomic int32. +// +// The default value is zero. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// +stateify savable +type Int32 struct { + _ sync.NoCopy + value int32 +} + +// FromInt32 returns an Int32 initialized to value v. +// +//go:nosplit +func FromInt32(v int32) Int32 { + return Int32{value: v} +} + +// Load is analogous to atomic.LoadInt32. +// +//go:nosplit +func (i *Int32) Load() int32 { + return atomic.LoadInt32(&i.value) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int32) RacyLoad() int32 { + return i.value +} + +// Store is analogous to atomic.StoreInt32. +// +//go:nosplit +func (i *Int32) Store(v int32) { + atomic.StoreInt32(&i.value, v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int32) RacyStore(v int32) { + i.value = v +} + +// Add is analogous to atomic.AddInt32. +// +//go:nosplit +func (i *Int32) Add(v int32) int32 { + return atomic.AddInt32(&i.value, v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int32) RacyAdd(v int32) int32 { + i.value += v + return i.value +} + +// Swap is analogous to atomic.SwapInt32. +// +//go:nosplit +func (i *Int32) Swap(v int32) int32 { + return atomic.SwapInt32(&i.value, v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapInt32. +// +//go:nosplit +func (i *Int32) CompareAndSwap(oldVal, newVal int32) bool { + return atomic.CompareAndSwapInt32(&i.value, oldVal, newVal) +} + +//go:nosplit +func (i *Int32) ptr() *int32 { + return &i.value +} + +// Uint32 is an atomic uint32. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// See aligned_unsafe.go in this directory for justification. +// +// +stateify savable +type Uint32 struct { + _ sync.NoCopy + value uint32 +} + +// FromUint32 returns an Uint32 initialized to value v. +// +//go:nosplit +func FromUint32(v uint32) Uint32 { + return Uint32{value: v} +} + +// Load is analogous to atomic.LoadUint32. +// +//go:nosplit +func (u *Uint32) Load() uint32 { + return atomic.LoadUint32(&u.value) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint32) RacyLoad() uint32 { + return u.value +} + +// Store is analogous to atomic.StoreUint32. +// +//go:nosplit +func (u *Uint32) Store(v uint32) { + atomic.StoreUint32(&u.value, v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint32) RacyStore(v uint32) { + u.value = v +} + +// Add is analogous to atomic.AddUint32. +// +//go:nosplit +func (u *Uint32) Add(v uint32) uint32 { + return atomic.AddUint32(&u.value, v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint32) RacyAdd(v uint32) uint32 { + u.value += v + return u.value +} + +// Swap is analogous to atomic.SwapUint32. +// +//go:nosplit +func (u *Uint32) Swap(v uint32) uint32 { + return atomic.SwapUint32(&u.value, v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapUint32. +// +//go:nosplit +func (u *Uint32) CompareAndSwap(oldVal, newVal uint32) bool { + return atomic.CompareAndSwapUint32(&u.value, oldVal, newVal) +} + +//go:nosplit +func (u *Uint32) ptr() *uint32 { + return &u.value +} + +// Bool is an atomic Boolean. +// +// It is implemented by a Uint32, with value 0 indicating false, and 1 +// indicating true. +// +// +stateify savable +type Bool struct { + Uint32 +} + +// b32 returns a uint32 0 or 1 representing b. +func b32(b bool) uint32 { + if b { + return 1 + } + return 0 +} + +// FromBool returns a Bool initialized to value val. +// +//go:nosplit +func FromBool(val bool) Bool { + return Bool{ + Uint32: FromUint32(b32(val)), + } +} + +// Load is analogous to atomic.LoadBool, if such a thing existed. +// +//go:nosplit +func (b *Bool) Load() bool { + return b.Uint32.Load() != 0 +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (b *Bool) RacyLoad() bool { + return b.Uint32.RacyLoad() != 0 +} + +// Store is analogous to atomic.StoreBool, if such a thing existed. +// +//go:nosplit +func (b *Bool) Store(val bool) { + b.Uint32.Store(b32(val)) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (b *Bool) RacyStore(val bool) { + b.Uint32.RacyStore(b32(val)) +} + +// Swap is analogous to atomic.SwapBool, if such a thing existed. +// +//go:nosplit +func (b *Bool) Swap(val bool) bool { + return b.Uint32.Swap(b32(val)) != 0 +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapBool, if such a thing +// existed. +// +//go:nosplit +func (b *Bool) CompareAndSwap(oldVal, newVal bool) bool { + return b.Uint32.CompareAndSwap(b32(oldVal), b32(newVal)) +} + +// LINT.ThenChange(32b_32bit.go) diff --git a/pkg/atomicbitops/aligned_32bit_unsafe.go b/pkg/atomicbitops/aligned_32bit_unsafe.go new file mode 100644 index 0000000..d2c3e45 --- /dev/null +++ b/pkg/atomicbitops/aligned_32bit_unsafe.go @@ -0,0 +1,231 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm || mips || mipsle || 386 +// +build arm mips mipsle 386 + +package atomicbitops + +import ( + "sync/atomic" + "unsafe" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Int64 is an atomic int64 that is guaranteed to be 64-bit +// aligned, even on 32-bit systems. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// Per https://golang.org/pkg/sync/atomic/#pkg-note-BUG: +// +// "On ARM, 386, and 32-bit MIPS, it is the caller's responsibility to arrange +// for 64-bit alignment of 64-bit words accessed atomically. The first word in +// a variable or in an allocated struct, array, or slice can be relied upon to +// be 64-bit aligned." +// +// +stateify savable +type Int64 struct { + _ sync.NoCopy + value int64 + value32 int32 +} + +//go:nosplit +func (i *Int64) ptr() *int64 { + // On 32-bit systems, i.value is guaranteed to be 32-bit aligned. It means + // that in the 12-byte i.value, there are guaranteed to be 8 contiguous bytes + // with 64-bit alignment. + return (*int64)(unsafe.Pointer((uintptr(unsafe.Pointer(&i.value)) + 4) &^ 7)) +} + +// FromInt64 returns an Int64 initialized to value v. +// +//go:nosplit +func FromInt64(v int64) Int64 { + var i Int64 + *i.ptr() = v + return i +} + +// Load is analogous to atomic.LoadInt64. +// +//go:nosplit +func (i *Int64) Load() int64 { + return atomic.LoadInt64(i.ptr()) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int64) RacyLoad() int64 { + return *i.ptr() +} + +// Store is analogous to atomic.StoreInt64. +// +//go:nosplit +func (i *Int64) Store(v int64) { + atomic.StoreInt64(i.ptr(), v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int64) RacyStore(v int64) { + *i.ptr() = v +} + +// Add is analogous to atomic.AddInt64. +// +//go:nosplit +func (i *Int64) Add(v int64) int64 { + return atomic.AddInt64(i.ptr(), v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int64) RacyAdd(v int64) int64 { + *i.ptr() += v + return *i.ptr() +} + +// Swap is analogous to atomic.SwapInt64. +// +//go:nosplit +func (i *Int64) Swap(v int64) int64 { + return atomic.SwapInt64(i.ptr(), v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapInt64. +// +//go:nosplit +func (i *Int64) CompareAndSwap(oldVal, newVal int64) bool { + return atomic.CompareAndSwapInt64(&i.value, oldVal, newVal) +} + +// Uint64 is an atomic uint64 that is guaranteed to be 64-bit +// aligned, even on 32-bit systems. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// Per https://golang.org/pkg/sync/atomic/#pkg-note-BUG: +// +// "On ARM, 386, and 32-bit MIPS, it is the caller's responsibility to arrange +// for 64-bit alignment of 64-bit words accessed atomically. The first word in +// a variable or in an allocated struct, array, or slice can be relied upon to +// be 64-bit aligned." +// +// +stateify savable +type Uint64 struct { + _ sync.NoCopy + value uint64 + value32 uint32 +} + +//go:nosplit +func (u *Uint64) ptr() *uint64 { + // On 32-bit systems, i.value is guaranteed to be 32-bit aligned. It means + // that in the 12-byte i.value, there are guaranteed to be 8 contiguous bytes + // with 64-bit alignment. + return (*uint64)(unsafe.Pointer((uintptr(unsafe.Pointer(&u.value)) + 4) &^ 7)) +} + +// FromUint64 returns an Uint64 initialized to value v. +// +//go:nosplit +func FromUint64(v uint64) Uint64 { + var u Uint64 + *u.ptr() = v + return u +} + +// Load is analogous to atomic.LoadUint64. +// +//go:nosplit +func (u *Uint64) Load() uint64 { + return atomic.LoadUint64(u.ptr()) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint64) RacyLoad() uint64 { + return *u.ptr() +} + +// Store is analogous to atomic.StoreUint64. +// +//go:nosplit +func (u *Uint64) Store(v uint64) { + atomic.StoreUint64(u.ptr(), v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint64) RacyStore(v uint64) { + *u.ptr() = v +} + +// Add is analogous to atomic.AddUint64. +// +//go:nosplit +func (u *Uint64) Add(v uint64) uint64 { + return atomic.AddUint64(u.ptr(), v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint64) RacyAdd(v uint64) uint64 { + *u.ptr() += v + return *u.ptr() +} + +// Swap is analogous to atomic.SwapUint64. +// +//go:nosplit +func (u *Uint64) Swap(v uint64) uint64 { + return atomic.SwapUint64(u.ptr(), v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapUint64. +// +//go:nosplit +func (u *Uint64) CompareAndSwap(oldVal, newVal uint64) bool { + return atomic.CompareAndSwapUint64(u.ptr(), oldVal, newVal) +} diff --git a/pkg/atomicbitops/aligned_64bit.go b/pkg/atomicbitops/aligned_64bit.go new file mode 100644 index 0000000..053e614 --- /dev/null +++ b/pkg/atomicbitops/aligned_64bit.go @@ -0,0 +1,212 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !arm && !mips && !mipsle && !386 +// +build !arm,!mips,!mipsle,!386 + +package atomicbitops + +import ( + "sync/atomic" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Int64 is an atomic int64 that is guaranteed to be 64-bit +// aligned, even on 32-bit systems. On most architectures, it's just a regular +// int64. +// +// The default value is zero. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// See aligned_32bit_unsafe.go in this directory for justification. +// +// +stateify savable +type Int64 struct { + _ sync.NoCopy + value int64 +} + +// FromInt64 returns an Int64 initialized to value v. +// +//go:nosplit +func FromInt64(v int64) Int64 { + return Int64{value: v} +} + +// Load is analogous to atomic.LoadInt64. +// +//go:nosplit +func (i *Int64) Load() int64 { + return atomic.LoadInt64(&i.value) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int64) RacyLoad() int64 { + return i.value +} + +// Store is analogous to atomic.StoreInt64. +// +//go:nosplit +func (i *Int64) Store(v int64) { + atomic.StoreInt64(&i.value, v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int64) RacyStore(v int64) { + i.value = v +} + +// Add is analogous to atomic.AddInt64. +// +//go:nosplit +func (i *Int64) Add(v int64) int64 { + return atomic.AddInt64(&i.value, v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (i *Int64) RacyAdd(v int64) int64 { + i.value += v + return i.value +} + +// Swap is analogous to atomic.SwapInt64. +// +//go:nosplit +func (i *Int64) Swap(v int64) int64 { + return atomic.SwapInt64(&i.value, v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapInt64. +// +//go:nosplit +func (i *Int64) CompareAndSwap(oldVal, newVal int64) bool { + return atomic.CompareAndSwapInt64(&i.value, oldVal, newVal) +} + +//go:nosplit +func (i *Int64) ptr() *int64 { + return &i.value +} + +// Uint64 is an atomic uint64 that is guaranteed to be 64-bit +// aligned, even on 32-bit systems. On most architectures, it's just a regular +// uint64. +// +// Don't add fields to this struct. It is important that it remain the same +// size as its builtin analogue. +// +// See aligned_unsafe.go in this directory for justification. +// +// +stateify savable +type Uint64 struct { + _ sync.NoCopy + value uint64 +} + +// FromUint64 returns an Uint64 initialized to value v. +// +//go:nosplit +func FromUint64(v uint64) Uint64 { + return Uint64{value: v} +} + +// Load is analogous to atomic.LoadUint64. +// +//go:nosplit +func (u *Uint64) Load() uint64 { + return atomic.LoadUint64(&u.value) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint64) RacyLoad() uint64 { + return u.value +} + +// Store is analogous to atomic.StoreUint64. +// +//go:nosplit +func (u *Uint64) Store(v uint64) { + atomic.StoreUint64(&u.value, v) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint64) RacyStore(v uint64) { + u.value = v +} + +// Add is analogous to atomic.AddUint64. +// +//go:nosplit +func (u *Uint64) Add(v uint64) uint64 { + return atomic.AddUint64(&u.value, v) +} + +// RacyAdd is analogous to adding to an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (u *Uint64) RacyAdd(v uint64) uint64 { + u.value += v + return u.value +} + +// Swap is analogous to atomic.SwapUint64. +// +//go:nosplit +func (u *Uint64) Swap(v uint64) uint64 { + return atomic.SwapUint64(&u.value, v) +} + +// CompareAndSwap is analogous to atomic.CompareAndSwapUint64. +// +//go:nosplit +func (u *Uint64) CompareAndSwap(oldVal, newVal uint64) bool { + return atomic.CompareAndSwapUint64(&u.value, oldVal, newVal) +} + +//go:nosplit +func (u *Uint64) ptr() *uint64 { + return &u.value +} diff --git a/pkg/atomicbitops/atomicbitops.go b/pkg/atomicbitops/atomicbitops.go new file mode 100644 index 0000000..36620b3 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops.go @@ -0,0 +1,82 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +// Package atomicbitops provides extensions to the sync/atomic package. +// +// All read-modify-write operations implemented by this package have +// acquire-release memory ordering (like sync/atomic). +// +// +checkalignedignore +package atomicbitops + +// AndUint32 atomically applies bitwise AND operation to *addr with val. +func AndUint32(addr *Uint32, val uint32) { + andUint32(&addr.value, val) +} + +func andUint32(addr *uint32, val uint32) + +// OrUint32 atomically applies bitwise OR operation to *addr with val. +func OrUint32(addr *Uint32, val uint32) { + orUint32(&addr.value, val) +} + +func orUint32(addr *uint32, val uint32) + +// XorUint32 atomically applies bitwise XOR operation to *addr with val. +func XorUint32(addr *Uint32, val uint32) { + xorUint32(&addr.value, val) +} + +func xorUint32(addr *uint32, val uint32) + +// CompareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns +// the value previously stored at addr. +func CompareAndSwapUint32(addr *Uint32, old, new uint32) uint32 { + return compareAndSwapUint32(&addr.value, old, new) +} + +func compareAndSwapUint32(addr *uint32, old, new uint32) uint32 + +// AndUint64 atomically applies bitwise AND operation to *addr with val. +func AndUint64(addr *Uint64, val uint64) { + andUint64(&addr.value, val) +} + +func andUint64(addr *uint64, val uint64) + +// OrUint64 atomically applies bitwise OR operation to *addr with val. +func OrUint64(addr *Uint64, val uint64) { + orUint64(&addr.value, val) +} + +func orUint64(addr *uint64, val uint64) + +// XorUint64 atomically applies bitwise XOR operation to *addr with val. +func XorUint64(addr *Uint64, val uint64) { + xorUint64(&addr.value, val) +} + +func xorUint64(addr *uint64, val uint64) + +// CompareAndSwapUint64 is like sync/atomic.CompareAndSwapUint64, but returns +// the value previously stored at addr. +func CompareAndSwapUint64(addr *Uint64, old, new uint64) uint64 { + return compareAndSwapUint64(&addr.value, old, new) +} + +func compareAndSwapUint64(addr *uint64, old, new uint64) uint64 diff --git a/pkg/atomicbitops/atomicbitops_32bit_state_autogen.go b/pkg/atomicbitops/atomicbitops_32bit_state_autogen.go new file mode 100644 index 0000000..8de79fa --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_32bit_state_autogen.go @@ -0,0 +1,93 @@ +// automatically generated by stateify. + +//go:build arm || mips || mipsle || 386 +// +build arm mips mipsle 386 + +package atomicbitops + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (i *Int32) StateTypeName() string { + return "pkg/atomicbitops.Int32" +} + +func (i *Int32) StateFields() []string { + return []string{ + "value", + } +} + +func (i *Int32) beforeSave() {} + +// +checklocksignore +func (i *Int32) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.value) +} + +func (i *Int32) afterLoad(context.Context) {} + +// +checklocksignore +func (i *Int32) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.value) +} + +func (u *Uint32) StateTypeName() string { + return "pkg/atomicbitops.Uint32" +} + +func (u *Uint32) StateFields() []string { + return []string{ + "value", + } +} + +func (u *Uint32) beforeSave() {} + +// +checklocksignore +func (u *Uint32) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.value) +} + +func (u *Uint32) afterLoad(context.Context) {} + +// +checklocksignore +func (u *Uint32) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.value) +} + +func (b *Bool) StateTypeName() string { + return "pkg/atomicbitops.Bool" +} + +func (b *Bool) StateFields() []string { + return []string{ + "Uint32", + } +} + +func (b *Bool) beforeSave() {} + +// +checklocksignore +func (b *Bool) StateSave(stateSinkObject state.Sink) { + b.beforeSave() + stateSinkObject.Save(0, &b.Uint32) +} + +func (b *Bool) afterLoad(context.Context) {} + +// +checklocksignore +func (b *Bool) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &b.Uint32) +} + +func init() { + state.Register((*Int32)(nil)) + state.Register((*Uint32)(nil)) + state.Register((*Bool)(nil)) +} diff --git a/pkg/atomicbitops/atomicbitops_32bit_unsafe_state_autogen.go b/pkg/atomicbitops/atomicbitops_32bit_unsafe_state_autogen.go new file mode 100644 index 0000000..8691ce8 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_32bit_unsafe_state_autogen.go @@ -0,0 +1,73 @@ +// automatically generated by stateify. + +//go:build arm || mips || mipsle || 386 +// +build arm mips mipsle 386 + +package atomicbitops + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (i *Int64) StateTypeName() string { + return "pkg/atomicbitops.Int64" +} + +func (i *Int64) StateFields() []string { + return []string{ + "value", + "value32", + } +} + +func (i *Int64) beforeSave() {} + +// +checklocksignore +func (i *Int64) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.value) + stateSinkObject.Save(1, &i.value32) +} + +func (i *Int64) afterLoad(context.Context) {} + +// +checklocksignore +func (i *Int64) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.value) + stateSourceObject.Load(1, &i.value32) +} + +func (u *Uint64) StateTypeName() string { + return "pkg/atomicbitops.Uint64" +} + +func (u *Uint64) StateFields() []string { + return []string{ + "value", + "value32", + } +} + +func (u *Uint64) beforeSave() {} + +// +checklocksignore +func (u *Uint64) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.value) + stateSinkObject.Save(1, &u.value32) +} + +func (u *Uint64) afterLoad(context.Context) {} + +// +checklocksignore +func (u *Uint64) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.value) + stateSourceObject.Load(1, &u.value32) +} + +func init() { + state.Register((*Int64)(nil)) + state.Register((*Uint64)(nil)) +} diff --git a/pkg/atomicbitops/atomicbitops_64bit_state_autogen.go b/pkg/atomicbitops/atomicbitops_64bit_state_autogen.go new file mode 100644 index 0000000..0af99eb --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_64bit_state_autogen.go @@ -0,0 +1,145 @@ +// automatically generated by stateify. + +//go:build !arm && !mips && !mipsle && !386 && !arm && !mips && !mipsle && !386 +// +build !arm,!mips,!mipsle,!386,!arm,!mips,!mipsle,!386 + +package atomicbitops + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (i *Int32) StateTypeName() string { + return "pkg/atomicbitops.Int32" +} + +func (i *Int32) StateFields() []string { + return []string{ + "value", + } +} + +func (i *Int32) beforeSave() {} + +// +checklocksignore +func (i *Int32) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.value) +} + +func (i *Int32) afterLoad(context.Context) {} + +// +checklocksignore +func (i *Int32) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.value) +} + +func (u *Uint32) StateTypeName() string { + return "pkg/atomicbitops.Uint32" +} + +func (u *Uint32) StateFields() []string { + return []string{ + "value", + } +} + +func (u *Uint32) beforeSave() {} + +// +checklocksignore +func (u *Uint32) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.value) +} + +func (u *Uint32) afterLoad(context.Context) {} + +// +checklocksignore +func (u *Uint32) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.value) +} + +func (b *Bool) StateTypeName() string { + return "pkg/atomicbitops.Bool" +} + +func (b *Bool) StateFields() []string { + return []string{ + "Uint32", + } +} + +func (b *Bool) beforeSave() {} + +// +checklocksignore +func (b *Bool) StateSave(stateSinkObject state.Sink) { + b.beforeSave() + stateSinkObject.Save(0, &b.Uint32) +} + +func (b *Bool) afterLoad(context.Context) {} + +// +checklocksignore +func (b *Bool) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &b.Uint32) +} + +func (i *Int64) StateTypeName() string { + return "pkg/atomicbitops.Int64" +} + +func (i *Int64) StateFields() []string { + return []string{ + "value", + } +} + +func (i *Int64) beforeSave() {} + +// +checklocksignore +func (i *Int64) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.value) +} + +func (i *Int64) afterLoad(context.Context) {} + +// +checklocksignore +func (i *Int64) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.value) +} + +func (u *Uint64) StateTypeName() string { + return "pkg/atomicbitops.Uint64" +} + +func (u *Uint64) StateFields() []string { + return []string{ + "value", + } +} + +func (u *Uint64) beforeSave() {} + +// +checklocksignore +func (u *Uint64) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.value) +} + +func (u *Uint64) afterLoad(context.Context) {} + +// +checklocksignore +func (u *Uint64) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.value) +} + +func init() { + state.Register((*Int32)(nil)) + state.Register((*Uint32)(nil)) + state.Register((*Bool)(nil)) + state.Register((*Int64)(nil)) + state.Register((*Uint64)(nil)) +} diff --git a/pkg/atomicbitops/atomicbitops_amd64.s b/pkg/atomicbitops/atomicbitops_amd64.s new file mode 100644 index 0000000..c38f1cb --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_amd64.s @@ -0,0 +1,77 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build amd64 + +#include "textflag.h" + +TEXT ·andUint32(SB),NOSPLIT|NOFRAME,$0-12 + MOVQ addr+0(FP), BX + MOVL val+8(FP), AX + LOCK + ANDL AX, 0(BX) + RET + +TEXT ·orUint32(SB),NOSPLIT|NOFRAME,$0-12 + MOVQ addr+0(FP), BX + MOVL val+8(FP), AX + LOCK + ORL AX, 0(BX) + RET + +TEXT ·xorUint32(SB),NOSPLIT|NOFRAME,$0-12 + MOVQ addr+0(FP), BX + MOVL val+8(FP), AX + LOCK + XORL AX, 0(BX) + RET + +TEXT ·compareAndSwapUint32(SB),NOSPLIT|NOFRAME,$0-20 + MOVQ addr+0(FP), DI + MOVL old+8(FP), AX + MOVL new+12(FP), DX + LOCK + CMPXCHGL DX, 0(DI) + MOVL AX, ret+16(FP) + RET + +TEXT ·andUint64(SB),NOSPLIT|NOFRAME,$0-16 + MOVQ addr+0(FP), BX + MOVQ val+8(FP), AX + LOCK + ANDQ AX, 0(BX) + RET + +TEXT ·orUint64(SB),NOSPLIT|NOFRAME,$0-16 + MOVQ addr+0(FP), BX + MOVQ val+8(FP), AX + LOCK + ORQ AX, 0(BX) + RET + +TEXT ·xorUint64(SB),NOSPLIT|NOFRAME,$0-16 + MOVQ addr+0(FP), BX + MOVQ val+8(FP), AX + LOCK + XORQ AX, 0(BX) + RET + +TEXT ·compareAndSwapUint64(SB),NOSPLIT|NOFRAME,$0-32 + MOVQ addr+0(FP), DI + MOVQ old+8(FP), AX + MOVQ new+16(FP), DX + LOCK + CMPXCHGQ DX, 0(DI) + MOVQ AX, ret+24(FP) + RET diff --git a/pkg/atomicbitops/atomicbitops_arm64.go b/pkg/atomicbitops/atomicbitops_arm64.go new file mode 100644 index 0000000..d4f7073 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_arm64.go @@ -0,0 +1,40 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package atomicbitops + +import ( + "runtime" + + "github.com/sagernet/gvisor/pkg/cpuid" + "golang.org/x/sys/cpu" +) + +var arm64HasATOMICS bool + +func init() { + // The gvisor cpuid package only works on Linux. + // For all other operating systems, use Go's x/sys/cpu package + // to get the one bit we care about here. + // + // See https://github.com/google/gvisor/issues/7849. + if runtime.GOOS == "linux" { + arm64HasATOMICS = cpuid.HostFeatureSet().HasFeature(cpuid.ARM64FeatureATOMICS) + } else { + arm64HasATOMICS = cpu.ARM64.HasATOMICS + } +} diff --git a/pkg/atomicbitops/atomicbitops_arm64.s b/pkg/atomicbitops/atomicbitops_arm64.s new file mode 100644 index 0000000..cf92211 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_arm64.s @@ -0,0 +1,141 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build arm64 + +#include "textflag.h" + +TEXT ·andUint32(SB),NOSPLIT,$0-12 + MOVD addr+0(FP), R0 + MOVW val+8(FP), R1 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + MVN R1, R2 + LDCLRALW R2, (R0), R3 + RET +load_store_loop: + LDAXRW (R0), R2 + ANDW R1, R2 + STLXRW R2, (R0), R3 + CBNZ R3, load_store_loop + RET + +TEXT ·orUint32(SB),NOSPLIT,$0-12 + MOVD addr+0(FP), R0 + MOVW val+8(FP), R1 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDORALW R1, (R0), R2 + RET +load_store_loop: + LDAXRW (R0), R2 + ORRW R1, R2 + STLXRW R2, (R0), R3 + CBNZ R3, load_store_loop + RET + +TEXT ·xorUint32(SB),NOSPLIT,$0-12 + MOVD addr+0(FP), R0 + MOVW val+8(FP), R1 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDEORALW R1, (R0), R2 + RET +load_store_loop: + LDAXRW (R0), R2 + EORW R1, R2 + STLXRW R2, (R0), R3 + CBNZ R3, load_store_loop + RET + +TEXT ·compareAndSwapUint32(SB),NOSPLIT,$0-20 + MOVD addr+0(FP), R0 + MOVW old+8(FP), R1 + MOVW new+12(FP), R2 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + CASALW R1, (R0), R2 + MOVW R1, ret+16(FP) + RET +load_store_loop: + LDAXRW (R0), R3 + CMPW R1, R3 + BNE ok + STLXRW R2, (R0), R4 + CBNZ R4, load_store_loop +ok: + MOVW R3, ret+16(FP) + RET + +TEXT ·andUint64(SB),NOSPLIT,$0-16 + MOVD addr+0(FP), R0 + MOVD val+8(FP), R1 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + MVN R1, R2 + LDCLRALD R2, (R0), R3 + RET +load_store_loop: + LDAXR (R0), R2 + AND R1, R2 + STLXR R2, (R0), R3 + CBNZ R3, load_store_loop + RET + +TEXT ·orUint64(SB),NOSPLIT,$0-16 + MOVD addr+0(FP), R0 + MOVD val+8(FP), R1 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDORALD R1, (R0), R2 + RET +load_store_loop: + LDAXR (R0), R2 + ORR R1, R2 + STLXR R2, (R0), R3 + CBNZ R3, load_store_loop + RET + +TEXT ·xorUint64(SB),NOSPLIT,$0-16 + MOVD addr+0(FP), R0 + MOVD val+8(FP), R1 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDEORALD R1, (R0), R2 + RET +load_store_loop: + LDAXR (R0), R2 + EOR R1, R2 + STLXR R2, (R0), R3 + CBNZ R3, load_store_loop + RET + +TEXT ·compareAndSwapUint64(SB),NOSPLIT,$0-32 + MOVD addr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R2 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + CASALD R1, (R0), R2 + MOVD R1, ret+24(FP) + RET +load_store_loop: + LDAXR (R0), R3 + CMP R1, R3 + BNE ok + STLXR R2, (R0), R4 + CBNZ R4, load_store_loop +ok: + MOVD R3, ret+24(FP) + RET diff --git a/pkg/atomicbitops/atomicbitops_arm64_state_autogen.go b/pkg/atomicbitops/atomicbitops_arm64_state_autogen.go new file mode 100644 index 0000000..d134333 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_arm64_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build arm64 +// +build arm64 + +package atomicbitops diff --git a/pkg/atomicbitops/atomicbitops_float64.go b/pkg/atomicbitops/atomicbitops_float64.go new file mode 100644 index 0000000..cd3c3cc --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_float64.go @@ -0,0 +1,105 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atomicbitops + +import ( + "math" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Float64 is an atomic 64-bit floating-point number. +// +// +stateify savable +type Float64 struct { + _ sync.NoCopy + // bits stores the bit of a 64-bit floating point number. + // It is not (and should not be interpreted as) a real uint64. + bits Uint64 +} + +// FromFloat64 returns a Float64 initialized to value v. +// +//go:nosplit +func FromFloat64(v float64) Float64 { + return Float64{bits: FromUint64(math.Float64bits(v))} +} + +// Load loads the floating-point value. +// +//go:nosplit +func (f *Float64) Load() float64 { + return math.Float64frombits(f.bits.Load()) +} + +// RacyLoad is analogous to reading an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (f *Float64) RacyLoad() float64 { + return math.Float64frombits(f.bits.RacyLoad()) +} + +// Store stores the given floating-point value in the Float64. +// +//go:nosplit +func (f *Float64) Store(v float64) { + f.bits.Store(math.Float64bits(v)) +} + +// RacyStore is analogous to setting an atomic value without using +// synchronization. +// +// It may be helpful to document why a racy operation is permitted. +// +//go:nosplit +func (f *Float64) RacyStore(v float64) { + f.bits.RacyStore(math.Float64bits(v)) +} + +// Swap stores the given value and returns the previously-stored one. +// +//go:nosplit +func (f *Float64) Swap(v float64) float64 { + return math.Float64frombits(f.bits.Swap(math.Float64bits(v))) +} + +// CompareAndSwap does a compare-and-swap operation on the float64 value. +// Note that unlike typical IEEE 754 semantics, this function will treat NaN +// as equal to itself if all of its bits exactly match. +// +//go:nosplit +func (f *Float64) CompareAndSwap(oldVal, newVal float64) bool { + return f.bits.CompareAndSwap(math.Float64bits(oldVal), math.Float64bits(newVal)) +} + +// Add increments the float by the given value. +// Note that unlike an atomic integer, this requires spin-looping until we win +// the compare-and-swap race, so this may take an indeterminate amount of time. +// +//go:nosplit +func (f *Float64) Add(v float64) { + // We do a racy load here because we optimistically think it may pass the + // compare-and-swap operation. If it doesn't, we'll load it safely, so this + // is OK and not a race for the overall intent of the user to add a number. + sync.RaceDisable() + oldVal := f.RacyLoad() + for !f.CompareAndSwap(oldVal, oldVal+v) { + oldVal = f.Load() + } + sync.RaceEnable() +} diff --git a/pkg/atomicbitops/atomicbitops_noasm.go b/pkg/atomicbitops/atomicbitops_noasm.go new file mode 100644 index 0000000..db8ca46 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_noasm.go @@ -0,0 +1,112 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !amd64 && !arm64 +// +build !amd64,!arm64 + +package atomicbitops + +import "sync/atomic" + +//go:nosplit +func AndUint32(addr *Uint32, val uint32) { + for { + o := addr.Load() + n := o & val + if atomic.CompareAndSwapUint32(&addr.value, o, n) { + break + } + } +} + +//go:nosplit +func OrUint32(addr *Uint32, val uint32) { + for { + o := addr.Load() + n := o | val + if atomic.CompareAndSwapUint32(&addr.value, o, n) { + break + } + } +} + +//go:nosplit +func XorUint32(addr *Uint32, val uint32) { + for { + o := addr.Load() + n := o ^ val + if atomic.CompareAndSwapUint32(&addr.value, o, n) { + break + } + } +} + +//go:nosplit +func CompareAndSwapUint32(addr *Uint32, old, new uint32) (prev uint32) { + for { + prev = addr.Load() + if prev != old { + return + } + if atomic.CompareAndSwapUint32(&addr.value, old, new) { + return + } + } +} + +//go:nosplit +func AndUint64(addr *Uint64, val uint64) { + for { + o := atomic.LoadUint64(addr.ptr()) + n := o & val + if atomic.CompareAndSwapUint64(addr.ptr(), o, n) { + break + } + } +} + +//go:nosplit +func OrUint64(addr *Uint64, val uint64) { + for { + o := atomic.LoadUint64(addr.ptr()) + n := o | val + if atomic.CompareAndSwapUint64(addr.ptr(), o, n) { + break + } + } +} + +//go:nosplit +func XorUint64(addr *Uint64, val uint64) { + for { + o := atomic.LoadUint64(addr.ptr()) + n := o ^ val + if atomic.CompareAndSwapUint64(addr.ptr(), o, n) { + break + } + } +} + +//go:nosplit +func CompareAndSwapUint64(addr *Uint64, old, new uint64) (prev uint64) { + for { + prev = atomic.LoadUint64(addr.ptr()) + if prev != old { + return + } + if atomic.CompareAndSwapUint64(addr.ptr(), old, new) { + return + } + } +} diff --git a/pkg/atomicbitops/atomicbitops_state_autogen.go b/pkg/atomicbitops/atomicbitops_state_autogen.go new file mode 100644 index 0000000..bd9da61 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_state_autogen.go @@ -0,0 +1,43 @@ +// automatically generated by stateify. + +//go:build (amd64 || arm64) && !amd64 && !arm64 +// +build amd64 arm64 +// +build !amd64 +// +build !arm64 + +package atomicbitops + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (f *Float64) StateTypeName() string { + return "pkg/atomicbitops.Float64" +} + +func (f *Float64) StateFields() []string { + return []string{ + "bits", + } +} + +func (f *Float64) beforeSave() {} + +// +checklocksignore +func (f *Float64) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.bits) +} + +func (f *Float64) afterLoad(context.Context) {} + +// +checklocksignore +func (f *Float64) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.bits) +} + +func init() { + state.Register((*Float64)(nil)) +} diff --git a/pkg/binary/binary.go b/pkg/binary/binary.go new file mode 100644 index 0000000..28b6ff6 --- /dev/null +++ b/pkg/binary/binary.go @@ -0,0 +1,266 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package binary translates between select fixed-sized types and a binary +// representation. +package binary + +import ( + "encoding/binary" + "fmt" + "io" + "reflect" +) + +// LittleEndian is the same as encoding/binary.LittleEndian. +// +// It is included here as a convenience. +var LittleEndian = binary.LittleEndian + +// BigEndian is the same as encoding/binary.BigEndian. +// +// It is included here as a convenience. +var BigEndian = binary.BigEndian + +// AppendUint16 appends the binary representation of a uint16 to buf. +func AppendUint16(buf []byte, order binary.ByteOrder, num uint16) []byte { + buf = append(buf, make([]byte, 2)...) + order.PutUint16(buf[len(buf)-2:], num) + return buf +} + +// AppendUint32 appends the binary representation of a uint32 to buf. +func AppendUint32(buf []byte, order binary.ByteOrder, num uint32) []byte { + buf = append(buf, make([]byte, 4)...) + order.PutUint32(buf[len(buf)-4:], num) + return buf +} + +// AppendUint64 appends the binary representation of a uint64 to buf. +func AppendUint64(buf []byte, order binary.ByteOrder, num uint64) []byte { + buf = append(buf, make([]byte, 8)...) + order.PutUint64(buf[len(buf)-8:], num) + return buf +} + +// Marshal appends a binary representation of data to buf. +// +// data must only contain fixed-length signed and unsigned ints, arrays, +// slices, structs and compositions of said types. data may be a pointer, +// but cannot contain pointers. +func Marshal(buf []byte, order binary.ByteOrder, data any) []byte { + return marshal(buf, order, reflect.Indirect(reflect.ValueOf(data))) +} + +func marshal(buf []byte, order binary.ByteOrder, data reflect.Value) []byte { + switch data.Kind() { + case reflect.Int8: + buf = append(buf, byte(int8(data.Int()))) + case reflect.Int16: + buf = AppendUint16(buf, order, uint16(int16(data.Int()))) + case reflect.Int32: + buf = AppendUint32(buf, order, uint32(int32(data.Int()))) + case reflect.Int64: + buf = AppendUint64(buf, order, uint64(data.Int())) + + case reflect.Uint8: + buf = append(buf, byte(data.Uint())) + case reflect.Uint16: + buf = AppendUint16(buf, order, uint16(data.Uint())) + case reflect.Uint32: + buf = AppendUint32(buf, order, uint32(data.Uint())) + case reflect.Uint64: + buf = AppendUint64(buf, order, data.Uint()) + + case reflect.Array, reflect.Slice: + for i, l := 0, data.Len(); i < l; i++ { + buf = marshal(buf, order, data.Index(i)) + } + + case reflect.Struct: + for i, l := 0, data.NumField(); i < l; i++ { + buf = marshal(buf, order, data.Field(i)) + } + + default: + panic("invalid type: " + data.Type().String()) + } + return buf +} + +// Unmarshal unpacks buf into data. +// +// data must be a slice or a pointer and buf must have a length of exactly +// Size(data). data must only contain fixed-length signed and unsigned ints, +// arrays, slices, structs and compositions of said types. +func Unmarshal(buf []byte, order binary.ByteOrder, data any) { + value := reflect.ValueOf(data) + switch value.Kind() { + case reflect.Ptr: + value = value.Elem() + case reflect.Slice: + default: + panic("invalid type: " + value.Type().String()) + } + buf = unmarshal(buf, order, value) + if len(buf) != 0 { + panic(fmt.Sprintf("buffer too long by %d bytes", len(buf))) + } +} + +func unmarshal(buf []byte, order binary.ByteOrder, data reflect.Value) []byte { + switch data.Kind() { + case reflect.Int8: + data.SetInt(int64(int8(buf[0]))) + buf = buf[1:] + case reflect.Int16: + data.SetInt(int64(int16(order.Uint16(buf)))) + buf = buf[2:] + case reflect.Int32: + data.SetInt(int64(int32(order.Uint32(buf)))) + buf = buf[4:] + case reflect.Int64: + data.SetInt(int64(order.Uint64(buf))) + buf = buf[8:] + + case reflect.Uint8: + data.SetUint(uint64(buf[0])) + buf = buf[1:] + case reflect.Uint16: + data.SetUint(uint64(order.Uint16(buf))) + buf = buf[2:] + case reflect.Uint32: + data.SetUint(uint64(order.Uint32(buf))) + buf = buf[4:] + case reflect.Uint64: + data.SetUint(order.Uint64(buf)) + buf = buf[8:] + + case reflect.Array, reflect.Slice: + for i, l := 0, data.Len(); i < l; i++ { + buf = unmarshal(buf, order, data.Index(i)) + } + + case reflect.Struct: + for i, l := 0, data.NumField(); i < l; i++ { + if field := data.Field(i); field.CanSet() { + buf = unmarshal(buf, order, field) + } else { + buf = buf[sizeof(field):] + } + } + + default: + panic("invalid type: " + data.Type().String()) + } + return buf +} + +// Size calculates the buffer sized needed by Marshal or Unmarshal. +// +// Size only support the types supported by Marshal. +func Size(v any) uintptr { + return sizeof(reflect.Indirect(reflect.ValueOf(v))) +} + +func sizeof(data reflect.Value) uintptr { + switch data.Kind() { + case reflect.Int8, reflect.Uint8: + return 1 + case reflect.Int16, reflect.Uint16: + return 2 + case reflect.Int32, reflect.Uint32: + return 4 + case reflect.Int64, reflect.Uint64: + return 8 + + case reflect.Array, reflect.Slice: + var size uintptr + for i, l := 0, data.Len(); i < l; i++ { + size += sizeof(data.Index(i)) + } + return size + + case reflect.Struct: + var size uintptr + for i, l := 0, data.NumField(); i < l; i++ { + size += sizeof(data.Field(i)) + } + return size + + default: + panic("invalid type: " + data.Type().String()) + } +} + +// ReadUint16 reads a uint16 from r. +func ReadUint16(r io.Reader, order binary.ByteOrder) (uint16, error) { + buf := make([]byte, 2) + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return order.Uint16(buf), nil +} + +// ReadUint32 reads a uint32 from r. +func ReadUint32(r io.Reader, order binary.ByteOrder) (uint32, error) { + buf := make([]byte, 4) + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return order.Uint32(buf), nil +} + +// ReadUint64 reads a uint64 from r. +func ReadUint64(r io.Reader, order binary.ByteOrder) (uint64, error) { + buf := make([]byte, 8) + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return order.Uint64(buf), nil +} + +// WriteUint16 writes a uint16 to w. +func WriteUint16(w io.Writer, order binary.ByteOrder, num uint16) error { + buf := make([]byte, 2) + order.PutUint16(buf, num) + _, err := w.Write(buf) + return err +} + +// WriteUint32 writes a uint32 to w. +func WriteUint32(w io.Writer, order binary.ByteOrder, num uint32) error { + buf := make([]byte, 4) + order.PutUint32(buf, num) + _, err := w.Write(buf) + return err +} + +// WriteUint64 writes a uint64 to w. +func WriteUint64(w io.Writer, order binary.ByteOrder, num uint64) error { + buf := make([]byte, 8) + order.PutUint64(buf, num) + _, err := w.Write(buf) + return err +} + +// AlignUp rounds a length up to an alignment. align must be a power of 2. +func AlignUp(length int, align uint) int { + return (length + int(align) - 1) & ^(int(align) - 1) +} + +// AlignDown rounds a length down to an alignment. align must be a power of 2. +func AlignDown(length int, align uint) int { + return length & ^(int(align) - 1) +} diff --git a/pkg/binary/binary_state_autogen.go b/pkg/binary/binary_state_autogen.go new file mode 100644 index 0000000..4661a59 --- /dev/null +++ b/pkg/binary/binary_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package binary diff --git a/pkg/bits/bits.go b/pkg/bits/bits.go new file mode 100644 index 0000000..d16448c --- /dev/null +++ b/pkg/bits/bits.go @@ -0,0 +1,26 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bits includes all bit related types and operations. +package bits + +// AlignUp rounds a length up to an alignment. align must be a power of 2. +func AlignUp(length int, align uint) int { + return (length + int(align) - 1) & ^(int(align) - 1) +} + +// AlignDown rounds a length down to an alignment. align must be a power of 2. +func AlignDown(length int, align uint) int { + return length & ^(int(align) - 1) +} diff --git a/pkg/bits/bits32.go b/pkg/bits/bits32.go new file mode 100644 index 0000000..28134a9 --- /dev/null +++ b/pkg/bits/bits32.go @@ -0,0 +1,33 @@ +package bits + +// IsOn returns true if *all* bits set in 'bits' are set in 'mask'. +func IsOn32(mask, bits uint32) bool { + return mask&bits == bits +} + +// IsAnyOn returns true if *any* bit set in 'bits' is set in 'mask'. +func IsAnyOn32(mask, bits uint32) bool { + return mask&bits != 0 +} + +// Mask returns a T with all of the given bits set. +func Mask32(is ...int) uint32 { + ret := uint32(0) + for _, i := range is { + ret |= MaskOf32(i) + } + return ret +} + +// MaskOf is like Mask, but sets only a single bit (more efficiently). +func MaskOf32(i int) uint32 { + return uint32(1) << uint32(i) +} + +// IsPowerOfTwo returns true if v is power of 2. +func IsPowerOfTwo32(v uint32) bool { + if v == 0 { + return false + } + return v&(v-1) == 0 +} diff --git a/pkg/bits/bits64.go b/pkg/bits/bits64.go new file mode 100644 index 0000000..73117b1 --- /dev/null +++ b/pkg/bits/bits64.go @@ -0,0 +1,33 @@ +package bits + +// IsOn returns true if *all* bits set in 'bits' are set in 'mask'. +func IsOn64(mask, bits uint64) bool { + return mask&bits == bits +} + +// IsAnyOn returns true if *any* bit set in 'bits' is set in 'mask'. +func IsAnyOn64(mask, bits uint64) bool { + return mask&bits != 0 +} + +// Mask returns a T with all of the given bits set. +func Mask64(is ...int) uint64 { + ret := uint64(0) + for _, i := range is { + ret |= MaskOf64(i) + } + return ret +} + +// MaskOf is like Mask, but sets only a single bit (more efficiently). +func MaskOf64(i int) uint64 { + return uint64(1) << uint64(i) +} + +// IsPowerOfTwo returns true if v is power of 2. +func IsPowerOfTwo64(v uint64) bool { + if v == 0 { + return false + } + return v&(v-1) == 0 +} diff --git a/pkg/bits/bits_state_autogen.go b/pkg/bits/bits_state_autogen.go new file mode 100644 index 0000000..436c111 --- /dev/null +++ b/pkg/bits/bits_state_autogen.go @@ -0,0 +1,8 @@ +// automatically generated by stateify. + +//go:build (amd64 || arm64) && !amd64 && !arm64 +// +build amd64 arm64 +// +build !amd64 +// +build !arm64 + +package bits diff --git a/pkg/bits/uint64_arch.go b/pkg/bits/uint64_arch.go new file mode 100644 index 0000000..fc56341 --- /dev/null +++ b/pkg/bits/uint64_arch.go @@ -0,0 +1,37 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package bits + +// TrailingZeros64 returns the number of bits before the least significant 1 +// bit in x; in other words, it returns the index of the least significant 1 +// bit in x. If x is 0, TrailingZeros64 returns 64. +func TrailingZeros64(x uint64) int + +// MostSignificantOne64 returns the index of the most significant 1 bit in +// x. If x is 0, MostSignificantOne64 returns 64. +func MostSignificantOne64(x uint64) int + +// ForEachSetBit64 calls f once for each set bit in x, with argument i equal to +// the set bit's index. +func ForEachSetBit64(x uint64, f func(i int)) { + for x != 0 { + i := TrailingZeros64(x) + f(i) + x &^= MaskOf64(i) + } +} diff --git a/pkg/bits/uint64_arch_amd64_asm.s b/pkg/bits/uint64_arch_amd64_asm.s new file mode 100644 index 0000000..2931b5d --- /dev/null +++ b/pkg/bits/uint64_arch_amd64_asm.s @@ -0,0 +1,32 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +TEXT ·TrailingZeros64(SB),$0-16 + BSFQ x+0(FP), AX + JNZ end + MOVQ $64, AX +end: + MOVQ AX, ret+8(FP) + RET + +TEXT ·MostSignificantOne64(SB),$0-16 + BSRQ x+0(FP), AX + JNZ end + MOVQ $64, AX +end: + MOVQ AX, ret+8(FP) + RET diff --git a/pkg/bits/uint64_arch_arm64_asm.s b/pkg/bits/uint64_arch_arm64_asm.s new file mode 100644 index 0000000..eb8d4d2 --- /dev/null +++ b/pkg/bits/uint64_arch_arm64_asm.s @@ -0,0 +1,34 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +TEXT ·TrailingZeros64(SB),$0-16 + MOVD x+0(FP), R0 + RBIT R0, R0 + CLZ R0, R0 // return 64 if x == 0 + MOVD R0, ret+8(FP) + RET + +TEXT ·MostSignificantOne64(SB),$0-16 + MOVD x+0(FP), R0 + CLZ R0, R0 // return 64 if x == 0 + MOVD $63, R1 + SUBS R0, R1, R0 // ret = 63 - CLZ + BPL end + MOVD $64, R0 // x == 0 +end: + MOVD R0, ret+8(FP) + RET diff --git a/pkg/bits/uint64_arch_generic.go b/pkg/bits/uint64_arch_generic.go new file mode 100644 index 0000000..83b23a3 --- /dev/null +++ b/pkg/bits/uint64_arch_generic.go @@ -0,0 +1,56 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !amd64 && !arm64 +// +build !amd64,!arm64 + +package bits + +// TrailingZeros64 returns the number of bits before the least significant 1 +// bit in x; in other words, it returns the index of the least significant 1 +// bit in x. If x is 0, TrailingZeros64 returns 64. +func TrailingZeros64(x uint64) int { + if x == 0 { + return 64 + } + i := 0 + for ; x&1 == 0; i++ { + x >>= 1 + } + return i +} + +// MostSignificantOne64 returns the index of the most significant 1 bit in +// x. If x is 0, MostSignificantOne64 returns 64. +func MostSignificantOne64(x uint64) int { + if x == 0 { + return 64 + } + i := 63 + for ; x&(1<<63) == 0; i-- { + x <<= 1 + } + return i +} + +// ForEachSetBit64 calls f once for each set bit in x, with argument i equal to +// the set bit's index. +func ForEachSetBit64(x uint64, f func(i int)) { + for i := 0; x != 0; i++ { + if x&1 != 0 { + f(i) + } + x >>= 1 + } +} diff --git a/pkg/buffer/buffer.go b/pkg/buffer/buffer.go new file mode 100644 index 0000000..4957860 --- /dev/null +++ b/pkg/buffer/buffer.go @@ -0,0 +1,660 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package buffer provides the implementation of a non-contiguous buffer that +// is reference counted, pooled, and copy-on-write. It allows O(1) append, +// and prepend operations. +package buffer + +import ( + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +// Buffer is a non-linear buffer. +// +// +stateify savable +type Buffer struct { + data ViewList `state:".([]byte)"` + size int64 +} + +func (b *Buffer) removeView(v *View) { + b.data.Remove(v) + v.Release() +} + +// MakeWithData creates a new Buffer initialized with given data. This function +// should be used with caution to avoid unnecessary []byte allocations. When in +// doubt use NewWithView to maximize chunk reuse. +func MakeWithData(b []byte) Buffer { + buf := Buffer{} + if len(b) == 0 { + return buf + } + v := NewViewWithData(b) + buf.Append(v) + return buf +} + +// MakeWithView creates a new Buffer initialized with given view. This function +// takes ownership of v. +func MakeWithView(v *View) Buffer { + if v == nil { + return Buffer{} + } + b := Buffer{ + size: int64(v.Size()), + } + if b.size == 0 { + v.Release() + return b + } + b.data.PushBack(v) + return b +} + +// Release frees all resources held by b. +func (b *Buffer) Release() { + for v := b.data.Front(); v != nil; v = b.data.Front() { + b.removeView(v) + } + b.size = 0 +} + +// TrimFront removes the first count bytes from the buffer. +func (b *Buffer) TrimFront(count int64) { + if count >= b.size { + b.advanceRead(b.size) + } else { + b.advanceRead(count) + } +} + +// ReadAt implements io.ReaderAt.ReadAt. +func (b *Buffer) ReadAt(p []byte, offset int64) (int, error) { + var ( + skipped int64 + done int64 + ) + for v := b.data.Front(); v != nil && done < int64(len(p)); v = v.Next() { + needToSkip := int(offset - skipped) + if sz := v.Size(); sz <= needToSkip { + skipped += int64(sz) + continue + } + + // Actually read data. + n := copy(p[done:], v.AsSlice()[needToSkip:]) + skipped += int64(needToSkip) + done += int64(n) + } + if int(done) < len(p) || offset+done == b.size { + return int(done), io.EOF + } + return int(done), nil +} + +// advanceRead advances the Buffer's read index. +// +// Precondition: there must be sufficient bytes in the buffer. +func (b *Buffer) advanceRead(count int64) { + for v := b.data.Front(); v != nil && count > 0; { + sz := int64(v.Size()) + if sz > count { + // There is still data for reading. + v.TrimFront(int(count)) + b.size -= count + count = 0 + return + } + + // Consume the whole view. + oldView := v + v = v.Next() // Iterate. + b.removeView(oldView) + + // Update counts. + count -= sz + b.size -= sz + } + if count > 0 { + panic(fmt.Sprintf("advanceRead still has %d bytes remaining", count)) + } +} + +// Truncate truncates the Buffer to the given length. +// +// This will not grow the Buffer, only shrink it. If a length is passed that is +// greater than the current size of the Buffer, then nothing will happen. +// +// Precondition: length must be >= 0. +func (b *Buffer) Truncate(length int64) { + if length < 0 { + panic("negative length provided") + } + if length >= b.size { + return // Nothing to do. + } + for v := b.data.Back(); v != nil && b.size > length; v = b.data.Back() { + sz := int64(v.Size()) + if after := b.size - sz; after < length { + // Truncate the buffer locally. + left := (length - after) + v.write = v.read + int(left) + b.size = length + break + } + + // Drop the buffer completely; see above. + b.removeView(v) + b.size -= sz + } +} + +// GrowTo grows the given Buffer to the number of bytes, which will be appended. +// If zero is true, all these bytes will be zero. If zero is false, then this is +// the caller's responsibility. +// +// Precondition: length must be >= 0. +func (b *Buffer) GrowTo(length int64, zero bool) { + if length < 0 { + panic("negative length provided") + } + for b.size < length { + v := b.data.Back() + + // Is there some space in the last buffer? + if v.Full() { + v = NewView(int(length - b.size)) + b.data.PushBack(v) + } + + // Write up to length bytes. + sz := v.AvailableSize() + if int64(sz) > length-b.size { + sz = int(length - b.size) + } + + // Zero the written section. + if zero { + clear(v.chunk.data[v.write : v.write+sz]) + } + + // Advance the index. + v.Grow(sz) + b.size += int64(sz) + } +} + +// Prepend prepends the given data. Prepend takes ownership of src. +func (b *Buffer) Prepend(src *View) error { + if src == nil { + return nil + } + if src.Size() == 0 { + src.Release() + return nil + } + // If the first buffer does not have room just prepend the view. + v := b.data.Front() + if v == nil || v.read == 0 { + b.prependOwned(src) + return nil + } + + // If there's room at the front and we won't incur a copy by writing to this + // view, fill in the extra room first. + if !v.sharesChunk() { + avail := v.read + vStart := 0 + srcStart := src.Size() - avail + if avail > src.Size() { + vStart = avail - src.Size() + srcStart = 0 + } + // Save the write index and restore it after. + old := v.write + v.read = vStart + n, err := v.WriteAt(src.AsSlice()[srcStart:], 0) + if err != nil { + return fmt.Errorf("could not write to view during append: %w", err) + } + b.size += int64(n) + v.write = old + src.write = srcStart + + // If there's no more to be written, then we're done. + if src.Size() == 0 { + src.Release() + return nil + } + } + + // Otherwise, just prepend the view. + b.prependOwned(src) + return nil +} + +// Append appends the given data. Append takes ownership of src. +func (b *Buffer) Append(src *View) error { + if src == nil { + return nil + } + if src.Size() == 0 { + src.Release() + return nil + } + // If the last buffer is full, just append the view. + v := b.data.Back() + if v.Full() { + b.appendOwned(src) + return nil + } + + // If a write won't incur a copy, then fill the back of the existing last + // chunk. + if !v.sharesChunk() { + writeSz := src.Size() + if src.Size() > v.AvailableSize() { + writeSz = v.AvailableSize() + } + done, err := v.Write(src.AsSlice()[:writeSz]) + if err != nil { + return fmt.Errorf("could not write to view during append: %w", err) + } + src.TrimFront(done) + b.size += int64(done) + if src.Size() == 0 { + src.Release() + return nil + } + } + + // If there is still data left just append the src. + b.appendOwned(src) + return nil +} + +func (b *Buffer) appendOwned(v *View) { + b.data.PushBack(v) + b.size += int64(v.Size()) +} + +func (b *Buffer) prependOwned(v *View) { + b.data.PushFront(v) + b.size += int64(v.Size()) +} + +// PullUp makes the specified range contiguous and returns the backing memory. +func (b *Buffer) PullUp(offset, length int) (View, bool) { + if length == 0 { + return View{}, true + } + tgt := Range{begin: offset, end: offset + length} + if tgt.Intersect(Range{end: int(b.size)}).Len() != length { + return View{}, false + } + + curr := Range{} + v := b.data.Front() + for ; v != nil; v = v.Next() { + origLen := v.Size() + curr.end = curr.begin + origLen + + if x := curr.Intersect(tgt); x.Len() == tgt.Len() { + // buf covers the whole requested target range. + sub := x.Offset(-curr.begin) + if v.sharesChunk() { + old := v.chunk + v.chunk = v.chunk.Clone() + old.DecRef() + } + new := View{ + read: v.read + sub.begin, + write: v.read + sub.end, + chunk: v.chunk, + } + return new, true + } else if x.Len() > 0 { + // buf is pointing at the starting buffer we want to merge. + break + } + + curr.begin += origLen + } + + // Calculate the total merged length. + totLen := 0 + for n := v; n != nil; n = n.Next() { + totLen += n.Size() + if curr.begin+totLen >= tgt.end { + break + } + } + + // Merge the buffers. + merged := NewViewSize(totLen) + off := 0 + for n := v; n != nil && off < totLen; { + merged.WriteAt(n.AsSlice(), off) + off += n.Size() + + // Remove buffers except for the first one, which will be reused. + if n == v { + n = n.Next() + } else { + old := n + n = n.Next() + b.removeView(old) + } + } + // Make data the first buffer. + b.data.InsertBefore(v, merged) + b.removeView(v) + + r := tgt.Offset(-curr.begin) + pulled := View{ + read: r.begin, + write: r.end, + chunk: merged.chunk, + } + return pulled, true +} + +// Flatten returns a flattened copy of this data. +// +// This method should not be used in any performance-sensitive paths. It may +// allocate a fresh byte slice sufficiently large to contain all the data in +// the buffer. This is principally for debugging. +// +// N.B. Tee data still belongs to this Buffer, as if there is a single buffer +// present, then it will be returned directly. This should be used for +// temporary use only, and a reference to the given slice should not be held. +func (b *Buffer) Flatten() []byte { + if v := b.data.Front(); v == nil { + return nil // No data at all. + } + data := make([]byte, 0, b.size) // Need to flatten. + for v := b.data.Front(); v != nil; v = v.Next() { + // Copy to the allocated slice. + data = append(data, v.AsSlice()...) + } + return data +} + +// Size indicates the total amount of data available in this Buffer. +func (b *Buffer) Size() int64 { + return b.size +} + +// AsViewList returns the ViewList backing b. Users may not save or modify the +// ViewList returned. +func (b *Buffer) AsViewList() ViewList { + return b.data +} + +// Clone creates a copy-on-write clone of b. The underlying chunks are shared +// until they are written to. +func (b *Buffer) Clone() Buffer { + other := Buffer{ + size: b.size, + } + for v := b.data.Front(); v != nil; v = v.Next() { + newView := v.Clone() + other.data.PushBack(newView) + } + return other +} + +// DeepClone creates a deep clone of b, copying data such that no bytes are +// shared with any other Buffers. +func (b *Buffer) DeepClone() Buffer { + newBuf := Buffer{} + buf := b.Clone() + reader := buf.AsBufferReader() + newBuf.WriteFromReader(&reader, b.size) + return newBuf +} + +// Apply applies the given function across all valid data. +func (b *Buffer) Apply(fn func(*View)) { + for v := b.data.Front(); v != nil; v = v.Next() { + d := v.Clone() + fn(d) + d.Release() + } +} + +// SubApply applies fn to a given range of data in b. Any part of the range +// outside of b is ignored. +func (b *Buffer) SubApply(offset, length int, fn func(*View)) { + for v := b.data.Front(); length > 0 && v != nil; v = v.Next() { + if offset >= v.Size() { + offset -= v.Size() + continue + } + d := v.Clone() + if offset > 0 { + d.TrimFront(offset) + offset = 0 + } + if length < d.Size() { + d.write = d.read + length + } + fn(d) + length -= d.Size() + d.Release() + } +} + +// Checksum calculates a checksum over the buffer's payload starting at offset. +func (b *Buffer) Checksum(offset int) uint16 { + if offset >= int(b.size) { + return 0 + } + var v *View + for v = b.data.Front(); v != nil && offset >= v.Size(); v = v.Next() { + offset -= v.Size() + } + + var cs checksum.Checksumer + cs.Add(v.AsSlice()[offset:]) + for v = v.Next(); v != nil; v = v.Next() { + cs.Add(v.AsSlice()) + } + return cs.Checksum() +} + +// Merge merges the provided Buffer with this one. +// +// The other Buffer will be appended to v, and other will be empty after this +// operation completes. +func (b *Buffer) Merge(other *Buffer) { + b.data.PushBackList(&other.data) + other.data = ViewList{} + + // Adjust sizes. + b.size += other.size + other.size = 0 +} + +// WriteFromReader writes to the buffer from an io.Reader. A maximum read size +// of MaxChunkSize is enforced to prevent allocating views from the heap. +func (b *Buffer) WriteFromReader(r io.Reader, count int64) (int64, error) { + return b.WriteFromReaderAndLimitedReader(r, count, nil) +} + +// WriteFromReaderAndLimitedReader is the same as WriteFromReader, but +// optimized to avoid allocations if a LimitedReader is passed in. +// +// This function clobbers the values of lr. +func (b *Buffer) WriteFromReaderAndLimitedReader(r io.Reader, count int64, lr *io.LimitedReader) (int64, error) { + if lr == nil { + lr = &io.LimitedReader{} + } + + var done int64 + for done < count { + vsize := count - done + if vsize > MaxChunkSize { + vsize = MaxChunkSize + } + v := NewView(int(vsize)) + lr.R = r + lr.N = vsize + n, err := io.Copy(v, lr) + b.Append(v) + done += n + if err == io.EOF { + break + } + if err != nil { + return done, err + } + } + return done, nil +} + +// ReadToWriter reads from the buffer into an io.Writer. +// +// N.B. This does not consume the bytes read. TrimFront should +// be called appropriately after this call in order to do so. +func (b *Buffer) ReadToWriter(w io.Writer, count int64) (int64, error) { + bytesLeft := int(count) + for v := b.data.Front(); v != nil && bytesLeft > 0; v = v.Next() { + view := v.Clone() + if view.Size() > bytesLeft { + view.CapLength(bytesLeft) + } + n, err := io.Copy(w, view) + bytesLeft -= int(n) + view.Release() + if err != nil { + return count - int64(bytesLeft), err + } + } + return count - int64(bytesLeft), nil +} + +// read implements the io.Reader interface. This method is used by BufferReader +// to consume its underlying buffer. To perform io operations on buffers +// directly, use ReadToWriter or WriteToReader. +func (b *Buffer) read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if b.Size() == 0 { + return 0, io.EOF + } + done := 0 + v := b.data.Front() + for v != nil && done < len(p) { + n, err := v.Read(p[done:]) + done += n + next := v.Next() + if v.Size() == 0 { + b.removeView(v) + } + b.size -= int64(n) + if err != nil && err != io.EOF { + return done, err + } + v = next + } + return done, nil +} + +// readByte implements the io.ByteReader interface. This method is used by +// BufferReader to consume its underlying buffer. To perform io operations on +// buffers directly, use ReadToWriter or WriteToReader. +func (b *Buffer) readByte() (byte, error) { + if b.Size() == 0 { + return 0, io.EOF + } + v := b.data.Front() + bt := v.AsSlice()[0] + b.TrimFront(1) + return bt, nil +} + +// AsBufferReader returns the Buffer as a BufferReader capable of io methods. +// The new BufferReader takes ownership of b. +func (b *Buffer) AsBufferReader() BufferReader { + return BufferReader{b} +} + +// BufferReader implements io methods on Buffer. Users must call Close() +// when finished with the buffer to free the underlying memory. +type BufferReader struct { + b *Buffer +} + +// Read implements the io.Reader interface. +func (br *BufferReader) Read(p []byte) (int, error) { + return br.b.read(p) +} + +// ReadByte implements the io.ByteReader interface. +func (br *BufferReader) ReadByte() (byte, error) { + return br.b.readByte() +} + +// Close implements the io.Closer interface. +func (br *BufferReader) Close() { + br.b.Release() +} + +// Len returns the number of bytes in the unread portion of the buffer. +func (br *BufferReader) Len() int { + return int(br.b.Size()) +} + +// Range specifies a range of buffer. +type Range struct { + begin int + end int +} + +// Intersect returns the intersection of x and y. +func (x Range) Intersect(y Range) Range { + if x.begin < y.begin { + x.begin = y.begin + } + if x.end > y.end { + x.end = y.end + } + if x.begin >= x.end { + return Range{} + } + return x +} + +// Offset returns x offset by off. +func (x Range) Offset(off int) Range { + x.begin += off + x.end += off + return x +} + +// Len returns the length of x. +func (x Range) Len() int { + l := x.end - x.begin + if l < 0 { + l = 0 + } + return l +} diff --git a/pkg/buffer/buffer_state.go b/pkg/buffer/buffer_state.go new file mode 100644 index 0000000..d57dfa0 --- /dev/null +++ b/pkg/buffer/buffer_state.go @@ -0,0 +1,29 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffer + +import ( + "context" +) + +// saveData is invoked by stateify. +func (b *Buffer) saveData() []byte { + return b.Flatten() +} + +// loadData is invoked by stateify. +func (b *Buffer) loadData(_ context.Context, data []byte) { + *b = MakeWithData(data) +} diff --git a/pkg/buffer/buffer_state_autogen.go b/pkg/buffer/buffer_state_autogen.go new file mode 100644 index 0000000..1cf2bd0 --- /dev/null +++ b/pkg/buffer/buffer_state_autogen.go @@ -0,0 +1,187 @@ +// automatically generated by stateify. + +package buffer + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (b *Buffer) StateTypeName() string { + return "pkg/buffer.Buffer" +} + +func (b *Buffer) StateFields() []string { + return []string{ + "data", + "size", + } +} + +func (b *Buffer) beforeSave() {} + +// +checklocksignore +func (b *Buffer) StateSave(stateSinkObject state.Sink) { + b.beforeSave() + var dataValue []byte + dataValue = b.saveData() + stateSinkObject.SaveValue(0, dataValue) + stateSinkObject.Save(1, &b.size) +} + +func (b *Buffer) afterLoad(context.Context) {} + +// +checklocksignore +func (b *Buffer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(1, &b.size) + stateSourceObject.LoadValue(0, new([]byte), func(y any) { b.loadData(ctx, y.([]byte)) }) +} + +func (c *chunk) StateTypeName() string { + return "pkg/buffer.chunk" +} + +func (c *chunk) StateFields() []string { + return []string{ + "chunkRefs", + "data", + } +} + +func (c *chunk) beforeSave() {} + +// +checklocksignore +func (c *chunk) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.chunkRefs) + stateSinkObject.Save(1, &c.data) +} + +func (c *chunk) afterLoad(context.Context) {} + +// +checklocksignore +func (c *chunk) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.chunkRefs) + stateSourceObject.Load(1, &c.data) +} + +func (r *chunkRefs) StateTypeName() string { + return "pkg/buffer.chunkRefs" +} + +func (r *chunkRefs) StateFields() []string { + return []string{ + "refCount", + } +} + +func (r *chunkRefs) beforeSave() {} + +// +checklocksignore +func (r *chunkRefs) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.refCount) +} + +// +checklocksignore +func (r *chunkRefs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.refCount) + stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) }) +} + +func (v *View) StateTypeName() string { + return "pkg/buffer.View" +} + +func (v *View) StateFields() []string { + return []string{ + "read", + "write", + "chunk", + } +} + +func (v *View) beforeSave() {} + +// +checklocksignore +func (v *View) StateSave(stateSinkObject state.Sink) { + v.beforeSave() + stateSinkObject.Save(0, &v.read) + stateSinkObject.Save(1, &v.write) + stateSinkObject.Save(2, &v.chunk) +} + +func (v *View) afterLoad(context.Context) {} + +// +checklocksignore +func (v *View) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &v.read) + stateSourceObject.Load(1, &v.write) + stateSourceObject.Load(2, &v.chunk) +} + +func (l *ViewList) StateTypeName() string { + return "pkg/buffer.ViewList" +} + +func (l *ViewList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *ViewList) beforeSave() {} + +// +checklocksignore +func (l *ViewList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *ViewList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *ViewList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *ViewEntry) StateTypeName() string { + return "pkg/buffer.ViewEntry" +} + +func (e *ViewEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *ViewEntry) beforeSave() {} + +// +checklocksignore +func (e *ViewEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *ViewEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ViewEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*Buffer)(nil)) + state.Register((*chunk)(nil)) + state.Register((*chunkRefs)(nil)) + state.Register((*View)(nil)) + state.Register((*ViewList)(nil)) + state.Register((*ViewEntry)(nil)) +} diff --git a/pkg/buffer/buffer_unsafe_state_autogen.go b/pkg/buffer/buffer_unsafe_state_autogen.go new file mode 100644 index 0000000..5a5c407 --- /dev/null +++ b/pkg/buffer/buffer_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package buffer diff --git a/pkg/buffer/chunk.go b/pkg/buffer/chunk.go new file mode 100644 index 0000000..db1d1fe --- /dev/null +++ b/pkg/buffer/chunk.go @@ -0,0 +1,113 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffer + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/bits" + "github.com/sagernet/gvisor/pkg/sync" +) + +const ( + // This is log2(baseChunkSize). This number is used to calculate which pool + // to use for a payload size by right shifting the payload size by this + // number and passing the result to MostSignificantOne64. + baseChunkSizeLog2 = 6 + + // This is the size of the buffers in the first pool. Each subsequent pool + // creates payloads 2^(pool index) times larger than the first pool's + // payloads. + baseChunkSize = 1 << baseChunkSizeLog2 // 64 + + // MaxChunkSize is largest payload size that we pool. Payloads larger than + // this will be allocated from the heap and garbage collected as normal. + MaxChunkSize = baseChunkSize << (numPools - 1) // 64k + + // The number of chunk pools we have for use. + numPools = 11 +) + +// chunkPools is a collection of pools for payloads of different sizes. The +// size of the payloads doubles in each successive pool. +var chunkPools [numPools]sync.Pool + +func init() { + for i := 0; i < numPools; i++ { + chunkSize := baseChunkSize * (1 << i) + chunkPools[i].New = func() any { + return &chunk{ + data: make([]byte, chunkSize), + } + } + } +} + +// Precondition: 0 <= size <= maxChunkSize +func getChunkPool(size int) *sync.Pool { + idx := 0 + if size > baseChunkSize { + idx = bits.MostSignificantOne64(uint64(size) >> baseChunkSizeLog2) + if size > 1<<(idx+baseChunkSizeLog2) { + idx++ + } + } + if idx >= numPools { + panic(fmt.Sprintf("pool for chunk size %d does not exist", size)) + } + return &chunkPools[idx] +} + +// Chunk represents a slice of pooled memory. +// +// +stateify savable +type chunk struct { + chunkRefs + data []byte +} + +func newChunk(size int) *chunk { + var c *chunk + if size > MaxChunkSize { + c = &chunk{ + data: make([]byte, size), + } + } else { + pool := getChunkPool(size) + c = pool.Get().(*chunk) + clear(c.data) + } + c.InitRefs() + return c +} + +func (c *chunk) destroy() { + if len(c.data) > MaxChunkSize { + c.data = nil + return + } + pool := getChunkPool(len(c.data)) + pool.Put(c) +} + +func (c *chunk) DecRef() { + c.chunkRefs.DecRef(c.destroy) +} + +func (c *chunk) Clone() *chunk { + cpy := newChunk(len(c.data)) + copy(cpy.data, c.data) + return cpy +} diff --git a/pkg/buffer/chunk_refs.go b/pkg/buffer/chunk_refs.go new file mode 100644 index 0000000..30d9ce8 --- /dev/null +++ b/pkg/buffer/chunk_refs.go @@ -0,0 +1,141 @@ +package buffer + +import ( + "context" + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/refs" +) + +// enableLogging indicates whether reference-related events should be logged (with +// stack traces). This is false by default and should only be set to true for +// debugging purposes, as it can generate an extremely large amount of output +// and drastically degrade performance. +const chunkenableLogging = false + +// obj is used to customize logging. Note that we use a pointer to T so that +// we do not copy the entire object when passed as a format parameter. +var chunkobj *chunk + +// Refs implements refs.RefCounter. It keeps a reference count using atomic +// operations and calls the destructor when the count reaches zero. +// +// NOTE: Do not introduce additional fields to the Refs struct. It is used by +// many filesystem objects, and we want to keep it as small as possible (i.e., +// the same size as using an int64 directly) to avoid taking up extra cache +// space. In general, this template should not be extended at the cost of +// performance. If it does not offer enough flexibility for a particular object +// (example: b/187877947), we should implement the RefCounter/CheckedObject +// interfaces manually. +// +// +stateify savable +type chunkRefs struct { + // refCount is composed of two fields: + // + // [32-bit speculative references]:[32-bit real references] + // + // Speculative references are used for TryIncRef, to avoid a CompareAndSwap + // loop. See IncRef, DecRef and TryIncRef for details of how these fields are + // used. + refCount atomicbitops.Int64 +} + +// InitRefs initializes r with one reference and, if enabled, activates leak +// checking. +func (r *chunkRefs) InitRefs() { + r.refCount.RacyStore(1) + refs.Register(r) +} + +// RefType implements refs.CheckedObject.RefType. +func (r *chunkRefs) RefType() string { + return fmt.Sprintf("%T", chunkobj)[1:] +} + +// LeakMessage implements refs.CheckedObject.LeakMessage. +func (r *chunkRefs) LeakMessage() string { + return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs()) +} + +// LogRefs implements refs.CheckedObject.LogRefs. +func (r *chunkRefs) LogRefs() bool { + return chunkenableLogging +} + +// ReadRefs returns the current number of references. The returned count is +// inherently racy and is unsafe to use without external synchronization. +func (r *chunkRefs) ReadRefs() int64 { + return r.refCount.Load() +} + +// IncRef implements refs.RefCounter.IncRef. +// +//go:nosplit +func (r *chunkRefs) IncRef() { + v := r.refCount.Add(1) + if chunkenableLogging { + refs.LogIncRef(r, v) + } + if v <= 1 { + panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType())) + } +} + +// TryIncRef implements refs.TryRefCounter.TryIncRef. +// +// To do this safely without a loop, a speculative reference is first acquired +// on the object. This allows multiple concurrent TryIncRef calls to distinguish +// other TryIncRef calls from genuine references held. +// +//go:nosplit +func (r *chunkRefs) TryIncRef() bool { + const speculativeRef = 1 << 32 + if v := r.refCount.Add(speculativeRef); int32(v) == 0 { + + r.refCount.Add(-speculativeRef) + return false + } + + v := r.refCount.Add(-speculativeRef + 1) + if chunkenableLogging { + refs.LogTryIncRef(r, v) + } + return true +} + +// DecRef implements refs.RefCounter.DecRef. +// +// Note that speculative references are counted here. Since they were added +// prior to real references reaching zero, they will successfully convert to +// real references. In other words, we see speculative references only in the +// following case: +// +// A: TryIncRef [speculative increase => sees non-negative references] +// B: DecRef [real decrease] +// A: TryIncRef [transform speculative to real] +// +//go:nosplit +func (r *chunkRefs) DecRef(destroy func()) { + v := r.refCount.Add(-1) + if chunkenableLogging { + refs.LogDecRef(r, v) + } + switch { + case v < 0: + panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType())) + + case v == 0: + refs.Unregister(r) + + if destroy != nil { + destroy() + } + } +} + +func (r *chunkRefs) afterLoad(context.Context) { + if r.ReadRefs() > 0 { + refs.Register(r) + } +} diff --git a/pkg/buffer/view.go b/pkg/buffer/view.go new file mode 100644 index 0000000..d913468 --- /dev/null +++ b/pkg/buffer/view.go @@ -0,0 +1,366 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffer + +import ( + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// ReadSize is the default amount that a View's size is increased by when an +// io.Reader has more data than a View can hold during calls to ReadFrom. +const ReadSize = 512 + +var viewPool = sync.Pool{ + New: func() any { + return &View{} + }, +} + +// View is a window into a shared chunk. Views are held by Buffers in +// viewLists to represent contiguous memory. +// +// A View must be created with NewView, NewViewWithData, or Clone. Owners are +// responsible for maintaining ownership over their views. When Views need to be +// shared or copied, the owner should create a new View with Clone. Clone must +// only ever be called on a owned View, not a borrowed one. +// +// Users are responsible for calling Release when finished with their View so +// that its resources can be returned to the pool. +// +// Users must not write directly to slices returned by AsSlice. Instead, they +// must use Write/WriteAt/CopyIn to modify the underlying View. This preserves +// the safety guarantees of copy-on-write. +// +// +stateify savable +type View struct { + ViewEntry `state:"nosave"` + read int + write int + chunk *chunk +} + +// NewView creates a new view with capacity at least as big as cap. It is +// analogous to make([]byte, 0, cap). +func NewView(cap int) *View { + c := newChunk(cap) + v := viewPool.Get().(*View) + *v = View{chunk: c} + return v +} + +// NewViewSize creates a new view with capacity at least as big as size and +// length that is exactly size. It is analogous to make([]byte, size). +func NewViewSize(size int) *View { + v := NewView(size) + v.Grow(size) + return v +} + +// NewViewWithData creates a new view and initializes it with data. This +// function should be used with caution to avoid unnecessary []byte allocations. +// When in doubt use NewWithView to maximize chunk reuse in production +// environments. +func NewViewWithData(data []byte) *View { + c := newChunk(len(data)) + v := viewPool.Get().(*View) + *v = View{chunk: c} + v.Write(data) + return v +} + +// Clone creates a shallow clone of v where the underlying chunk is shared. +// +// The caller must own the View to call Clone. It is not safe to call Clone +// on a borrowed or shared View because it can race with other View methods. +func (v *View) Clone() *View { + if v == nil { + panic("cannot clone a nil view") + } + v.chunk.IncRef() + newV := viewPool.Get().(*View) + newV.chunk = v.chunk + newV.read = v.read + newV.write = v.write + return newV +} + +// Release releases the chunk held by v and returns v to the pool. +func (v *View) Release() { + if v == nil { + panic("cannot release a nil view") + } + v.chunk.DecRef() + *v = View{} + viewPool.Put(v) +} + +// Reset sets the view's read and write indices back to zero. +func (v *View) Reset() { + if v == nil { + panic("cannot reset a nil view") + } + v.read = 0 + v.write = 0 +} + +func (v *View) sharesChunk() bool { + return v.chunk.refCount.Load() > 1 +} + +// Full indicates the chunk is full. +// +// This indicates there is no capacity left to write. +func (v *View) Full() bool { + return v == nil || v.write == len(v.chunk.data) +} + +// Capacity returns the total size of this view's chunk. +func (v *View) Capacity() int { + if v == nil { + return 0 + } + return len(v.chunk.data) +} + +// Size returns the size of data written to the view. +func (v *View) Size() int { + if v == nil { + return 0 + } + return v.write - v.read +} + +// TrimFront advances the read index by the given amount. +func (v *View) TrimFront(n int) { + if v.read+n > v.write { + panic("cannot trim past the end of a view") + } + v.read += n +} + +// AsSlice returns a slice of the data written to this view. +func (v *View) AsSlice() []byte { + if v.Size() == 0 { + return nil + } + return v.chunk.data[v.read:v.write] +} + +// ToSlice returns an owned copy of the data in this view. +func (v *View) ToSlice() []byte { + if v.Size() == 0 { + return nil + } + s := make([]byte, v.Size()) + copy(s, v.AsSlice()) + return s +} + +// AvailableSize returns the number of bytes available for writing. +func (v *View) AvailableSize() int { + if v == nil { + return 0 + } + return len(v.chunk.data) - v.write +} + +// Read reads v's data into p. +// +// Implements the io.Reader interface. +func (v *View) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if v.Size() == 0 { + return 0, io.EOF + } + n := copy(p, v.AsSlice()) + v.TrimFront(n) + return n, nil +} + +// ReadByte implements the io.ByteReader interface. +func (v *View) ReadByte() (byte, error) { + if v.Size() == 0 { + return 0, io.EOF + } + b := v.AsSlice()[0] + v.read++ + return b, nil +} + +// WriteTo writes data to w until the view is empty or an error occurs. The +// return value n is the number of bytes written. +// +// WriteTo implements the io.WriterTo interface. +func (v *View) WriteTo(w io.Writer) (n int64, err error) { + if v.Size() > 0 { + sz := v.Size() + m, e := w.Write(v.AsSlice()) + v.TrimFront(m) + n = int64(m) + if e != nil { + return n, e + } + if m != sz { + return n, io.ErrShortWrite + } + } + return n, nil +} + +// ReadAt reads data to the p starting at offset. +// +// Implements the io.ReaderAt interface. +func (v *View) ReadAt(p []byte, off int) (int, error) { + if off < 0 || off > v.Size() { + return 0, fmt.Errorf("ReadAt(): offset out of bounds: want 0 < off < %d, got off=%d", v.Size(), off) + } + n := copy(p, v.AsSlice()[off:]) + return n, nil +} + +// Write writes data to the view's chunk starting at the v.write index. If the +// view's chunk has a reference count greater than 1, the chunk is copied first +// and then written to. +// +// Implements the io.Writer interface. +func (v *View) Write(p []byte) (int, error) { + if v == nil { + panic("cannot write to a nil view") + } + if v.AvailableSize() < len(p) { + v.growCap(len(p) - v.AvailableSize()) + } else if v.sharesChunk() { + defer v.chunk.DecRef() + v.chunk = v.chunk.Clone() + } + n := copy(v.chunk.data[v.write:], p) + v.write += n + if n < len(p) { + return n, io.ErrShortWrite + } + return n, nil +} + +// ReadFrom reads data from r until EOF and appends it to the buffer, growing +// the buffer as needed. The return value n is the number of bytes read. Any +// error except io.EOF encountered during the read is also returned. +// +// ReadFrom implements the io.ReaderFrom interface. +func (v *View) ReadFrom(r io.Reader) (n int64, err error) { + if v == nil { + panic("cannot write to a nil view") + } + if v.sharesChunk() { + defer v.chunk.DecRef() + v.chunk = v.chunk.Clone() + } + for { + // Check for EOF to avoid an unnnecesary allocation. + if _, e := r.Read(nil); e == io.EOF { + return n, nil + } + if v.AvailableSize() == 0 { + v.growCap(ReadSize) + } + m, e := r.Read(v.availableSlice()) + v.write += m + n += int64(m) + + if e == io.EOF { + return n, nil + } + if e != nil { + return n, e + } + } +} + +// WriteAt writes data to the views's chunk starting at start. If the +// view's chunk has a reference count greater than 1, the chunk is copied first +// and then written to. +// +// Implements the io.WriterAt interface. +func (v *View) WriteAt(p []byte, off int) (int, error) { + if v == nil { + panic("cannot write to a nil view") + } + if off < 0 || off > v.Size() { + return 0, fmt.Errorf("write offset out of bounds: want 0 < off < %d, got off=%d", v.Size(), off) + } + if v.sharesChunk() { + defer v.chunk.DecRef() + v.chunk = v.chunk.Clone() + } + n := copy(v.AsSlice()[off:], p) + if n < len(p) { + return n, io.ErrShortWrite + } + return n, nil +} + +// Grow increases the size of the view. If the new size is greater than the +// view's current capacity, Grow will reallocate the view with an increased +// capacity. +func (v *View) Grow(n int) { + if v == nil { + panic("cannot grow a nil view") + } + if v.write+n > v.Capacity() { + v.growCap(n) + } + v.write += n +} + +// growCap increases the capacity of the view by at least n. +func (v *View) growCap(n int) { + if v == nil { + panic("cannot grow a nil view") + } + defer v.chunk.DecRef() + old := v.AsSlice() + v.chunk = newChunk(v.Capacity() + n) + copy(v.chunk.data, old) + v.read = 0 + v.write = len(old) +} + +// CapLength caps the length of the view's read slice to n. If n > v.Size(), +// the function is a no-op. +func (v *View) CapLength(n int) { + if v == nil { + panic("cannot resize a nil view") + } + if n < 0 { + panic("n must be >= 0") + } + if n > v.Size() { + n = v.Size() + } + v.write = v.read + n +} + +func (v *View) availableSlice() []byte { + if v.sharesChunk() { + defer v.chunk.DecRef() + c := v.chunk.Clone() + v.chunk = c + } + return v.chunk.data[v.write:] +} diff --git a/pkg/buffer/view_list.go b/pkg/buffer/view_list.go new file mode 100644 index 0000000..db855df --- /dev/null +++ b/pkg/buffer/view_list.go @@ -0,0 +1,239 @@ +package buffer + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type ViewElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (ViewElementMapper) linkerFor(elem *View) *View { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type ViewList struct { + head *View + tail *View +} + +// Reset resets list l to the empty state. +func (l *ViewList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *ViewList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *ViewList) Front() *View { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *ViewList) Back() *View { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *ViewList) Len() (count int) { + for e := l.Front(); e != nil; e = (ViewElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *ViewList) PushFront(e *View) { + linker := ViewElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + ViewElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *ViewList) PushFrontList(m *ViewList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + ViewElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + ViewElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *ViewList) PushBack(e *View) { + linker := ViewElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + ViewElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *ViewList) PushBackList(m *ViewList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + ViewElementMapper{}.linkerFor(l.tail).SetNext(m.head) + ViewElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *ViewList) InsertAfter(b, e *View) { + bLinker := ViewElementMapper{}.linkerFor(b) + eLinker := ViewElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + ViewElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *ViewList) InsertBefore(a, e *View) { + aLinker := ViewElementMapper{}.linkerFor(a) + eLinker := ViewElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + ViewElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *ViewList) Remove(e *View) { + linker := ViewElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + ViewElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + ViewElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type ViewEntry struct { + next *View + prev *View +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *ViewEntry) Next() *View { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *ViewEntry) Prev() *View { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *ViewEntry) SetNext(elem *View) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *ViewEntry) SetPrev(elem *View) { + e.prev = elem +} diff --git a/pkg/buffer/view_unsafe.go b/pkg/buffer/view_unsafe.go new file mode 100644 index 0000000..cef7e7e --- /dev/null +++ b/pkg/buffer/view_unsafe.go @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffer + +import ( + "reflect" + "unsafe" +) + +// BasePtr returns a pointer to the view's chunk. +func (v *View) BasePtr() *byte { + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&v.chunk.data)) + return (*byte)(unsafe.Pointer(hdr.Data)) +} diff --git a/pkg/cleanup/cleanup.go b/pkg/cleanup/cleanup.go new file mode 100644 index 0000000..f963ed3 --- /dev/null +++ b/pkg/cleanup/cleanup.go @@ -0,0 +1,61 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cleanup provides utilities to clean "stuff" on defers. +package cleanup + +// Cleanup allows defers to be aborted when cleanup needs to happen +// conditionally. Usage: +// +// cu := cleanup.Make(func() { f.Close() }) +// defer cu.Clean() // failure before release is called will close the file. +// ... +// cu.Add(func() { f2.Close() }) // Adds another cleanup function +// ... +// cu.Release() // on success, aborts closing the file. +// return f +type Cleanup struct { + cleaners []func() +} + +// Make creates a new Cleanup object. +func Make(f func()) Cleanup { + return Cleanup{cleaners: []func(){f}} +} + +// Add adds a new function to be called on Clean(). +func (c *Cleanup) Add(f func()) { + c.cleaners = append(c.cleaners, f) +} + +// Clean calls all cleanup functions in reverse order. +func (c *Cleanup) Clean() { + clean(c.cleaners) + c.cleaners = nil +} + +// Release releases the cleanup from its duties, i.e. cleanup functions are not +// called after this point. Returns a function that calls all registered +// functions in case the caller has use for them. +func (c *Cleanup) Release() func() { + old := c.cleaners + c.cleaners = nil + return func() { clean(old) } +} + +func clean(cleaners []func()) { + for i := len(cleaners) - 1; i >= 0; i-- { + cleaners[i]() + } +} diff --git a/pkg/cleanup/cleanup_state_autogen.go b/pkg/cleanup/cleanup_state_autogen.go new file mode 100644 index 0000000..8373268 --- /dev/null +++ b/pkg/cleanup/cleanup_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package cleanup diff --git a/pkg/compressio/compressio.go b/pkg/compressio/compressio.go new file mode 100644 index 0000000..6f10475 --- /dev/null +++ b/pkg/compressio/compressio.go @@ -0,0 +1,788 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compressio provides parallel compression and decompression, as well +// as optional SHA-256 hashing. It also provides another storage variant +// (nocompressio) that does not compress data but tracks its integrity. +// +// The stream format is defined as follows. +// +// /------------------------------------------------------\ +// | chunk size (4-bytes) | +// +------------------------------------------------------+ +// | (optional) hash (32-bytes) | +// +------------------------------------------------------+ +// | compressed data size (4-bytes) | +// +------------------------------------------------------+ +// | compressed data | +// +------------------------------------------------------+ +// | (optional) hash (32-bytes) | +// +------------------------------------------------------+ +// | compressed data size (4-bytes) | +// +------------------------------------------------------+ +// | ...... | +// \------------------------------------------------------/ +// +// where each subsequent hash is calculated from the following items in order +// +// compressed data +// compressed data size +// previous hash +// +// so the stream integrity cannot be compromised by switching and mixing +// compressed chunks. +package compressio + +import ( + "bytes" + "compress/flate" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "errors" + "hash" + "io" + "runtime" + + "github.com/sagernet/gvisor/pkg/sync" +) + +var bufPool = sync.Pool{ + New: func() any { + return bytes.NewBuffer(nil) + }, +} + +var chunkPool = sync.Pool{ + New: func() any { + return new(chunk) + }, +} + +// chunk is a unit of work. +type chunk struct { + // compressed is compressed data. + // + // This will always be returned to the bufPool directly when work has + // finished (in schedule) and therefore must be allocated. + compressed *bytes.Buffer + + // uncompressed is the uncompressed data. + // + // This is not returned to the bufPool automatically, since it may + // correspond to a inline slice (provided directly to Read or Write). + uncompressed *bytes.Buffer + + // The current hash object. Only used in compress mode. + h hash.Hash + + // The hash from previous chunks. Only used in uncompress mode. + lastSum []byte + + // The expected hash after current chunk. Only used in uncompress mode. + sum []byte +} + +// newChunk allocates a new chunk object (or pulls one from the pool). Buffers +// will be allocated if nil is provided for compressed or uncompressed. +func newChunk(lastSum []byte, sum []byte, compressed *bytes.Buffer, uncompressed *bytes.Buffer) *chunk { + c := chunkPool.Get().(*chunk) + c.lastSum = lastSum + c.sum = sum + if compressed != nil { + c.compressed = compressed + } else { + c.compressed = bufPool.Get().(*bytes.Buffer) + } + if uncompressed != nil { + c.uncompressed = uncompressed + } else { + c.uncompressed = bufPool.Get().(*bytes.Buffer) + } + return c +} + +// result is the result of some work; it includes the original chunk. +type result struct { + *chunk + err error +} + +// worker is a compression/decompression worker. +// +// The associated worker goroutine reads in uncompressed buffers from input and +// writes compressed buffers to its output. Alternatively, the worker reads +// compressed buffers from input and writes uncompressed buffers to its output. +// +// The goroutine will exit when input is closed, and the goroutine will close +// output. +type worker struct { + hashPool *hashPool + input chan *chunk + output chan result + + // scratch is a temporary buffer used for marshalling. This is declared + // unfront here to avoid reallocation. + scratch [4]byte +} + +// work is the main work routine; see worker. +func (w *worker) work(compress bool, level int) { + defer close(w.output) + + var h hash.Hash + + for c := range w.input { + if h == nil && w.hashPool != nil { + h = w.hashPool.getHash() + } + if compress { + mw := io.Writer(c.compressed) + if h != nil { + mw = io.MultiWriter(mw, h) + } + + // Encode this slice. + fw, err := flate.NewWriter(mw, level) + if err != nil { + w.output <- result{c, err} + continue + } + + // Encode the input. + if _, err := io.CopyN(fw, c.uncompressed, int64(c.uncompressed.Len())); err != nil { + w.output <- result{c, err} + continue + } + if err := fw.Close(); err != nil { + w.output <- result{c, err} + continue + } + + // Write the hash, if enabled. + if h != nil { + binary.BigEndian.PutUint32(w.scratch[:], uint32(c.compressed.Len())) + h.Write(w.scratch[:4]) + c.h = h + h = nil + } + } else { + // Check the hash of the compressed contents. + if h != nil { + h.Write(c.compressed.Bytes()) + binary.BigEndian.PutUint32(w.scratch[:], uint32(c.compressed.Len())) + h.Write(w.scratch[:4]) + io.CopyN(h, bytes.NewReader(c.lastSum), int64(len(c.lastSum))) + + sum := h.Sum(nil) + h.Reset() + if !hmac.Equal(c.sum, sum) { + w.output <- result{c, ErrHashMismatch} + continue + } + } + + // Decode this slice. + fr := flate.NewReader(c.compressed) + + // Decode the input. + if _, err := io.Copy(c.uncompressed, fr); err != nil { + w.output <- result{c, err} + continue + } + } + + // Send the output. + w.output <- result{c, nil} + } +} + +type hashPool struct { + // mu protects the hash list. + mu sync.Mutex + + // key is the key used to create hash objects. + key []byte + + // hashes is the hash object free list. Note that this cannot be + // globally shared across readers or writers, as it is key-specific. + hashes []hash.Hash +} + +// getHash gets a hash object for the pool. It should only be called when the +// pool key is non-nil. +func (p *hashPool) getHash() hash.Hash { + p.mu.Lock() + defer p.mu.Unlock() + + if len(p.hashes) == 0 { + return hmac.New(sha256.New, p.key) + } + + h := p.hashes[len(p.hashes)-1] + p.hashes = p.hashes[:len(p.hashes)-1] + return h +} + +func (p *hashPool) putHash(h hash.Hash) { + h.Reset() + + p.mu.Lock() + defer p.mu.Unlock() + + p.hashes = append(p.hashes, h) +} + +// pool is common functionality for reader/writers. +type pool struct { + // workers are the compression/decompression workers. + workers []worker + + // chunkSize is the chunk size. This is the first four bytes in the + // stream and is shared across both the reader and writer. + chunkSize uint32 + + // mu protects below; it is generally the responsibility of users to + // acquire this mutex before calling any methods on the pool. + mu sync.Mutex + + // nextInput is the next worker for input (scheduling). + nextInput int + + // nextOutput is the next worker for output (result). + nextOutput int + + // buf is the current active buffer; the exact semantics of this buffer + // depending on whether this is a reader or a writer. + buf *bytes.Buffer + + // lasSum records the hash of the last chunk processed. + lastSum []byte + + // hashPool is the hash object pool. It cannot be embedded into pool + // itself as worker refers to it and that would stop pool from being + // GCed. + hashPool *hashPool +} + +// init initializes the worker pool. +// +// This should only be called once. +func (p *pool) init(key []byte, workers int, compress bool, level int) { + if key != nil { + p.hashPool = &hashPool{key: key} + } + p.workers = make([]worker, workers) + for i := 0; i < len(p.workers); i++ { + p.workers[i] = worker{ + hashPool: p.hashPool, + input: make(chan *chunk, 1), + output: make(chan result, 1), + } + go p.workers[i].work(compress, level) // S/R-SAFE: In save path only. + } + runtime.SetFinalizer(p, (*pool).stop) +} + +// stop stops all workers. +func (p *pool) stop() { + for i := 0; i < len(p.workers); i++ { + close(p.workers[i].input) + } + // Wait for all workers to finish since p.schedule(c=nil) may have returned + // early if any worker emitted an error. + if len(p.workers) != 0 { + for p.nextOutput < p.nextInput { + handleResult(<-p.workers[(p.nextOutput+1)%len(p.workers)].output, func(*chunk) error { + return nil + }) + p.nextOutput++ + } + } + p.workers = nil + p.hashPool = nil +} + +// handleResult calls the callback. +func handleResult(r result, callback func(*chunk) error) error { + defer func() { + r.chunk.compressed.Reset() + bufPool.Put(r.chunk.compressed) + chunkPool.Put(r.chunk) + }() + if r.err != nil { + return r.err + } + return callback(r.chunk) +} + +// schedule schedules the given buffers. +// +// If c is non-nil, then it will return as soon as the chunk is scheduled. If c +// is nil, then it will return only when no more work is left to do. +// +// If no callback function is provided, then the output channel will be +// ignored. You must be sure that the input is schedulable in this case. +func (p *pool) schedule(c *chunk, callback func(*chunk) error) error { + for { + var ( + inputChan chan *chunk + outputChan chan result + ) + if c != nil && len(p.workers) != 0 { + inputChan = p.workers[(p.nextInput+1)%len(p.workers)].input + } + if callback != nil && p.nextOutput != p.nextInput && len(p.workers) != 0 { + outputChan = p.workers[(p.nextOutput+1)%len(p.workers)].output + } + if inputChan == nil && outputChan == nil { + return nil + } + + select { + case inputChan <- c: + p.nextInput++ + return nil + case r := <-outputChan: + p.nextOutput++ + if err := handleResult(r, callback); err != nil { + return err + } + } + } +} + +// Reader is a compressed reader. +type Reader struct { + pool + + // in is the source. + in io.ReadCloser + + // scratch is a temporary buffer used for marshalling. This is declared + // unfront here to avoid reallocation. + scratch [4]byte +} + +var _ io.Reader = (*Reader)(nil) + +// NewReader returns a new compressed reader. If key is non-nil, the data stream +// is assumed to contain expected hash values, which will be compared against +// hash values computed from the compressed bytes. See package comments for +// details. +func NewReader(in io.ReadCloser, key []byte) (*Reader, error) { + r := &Reader{ + in: in, + } + + // Use double buffering for read. + r.init(key, 2*runtime.GOMAXPROCS(0), false, 0) + + if _, err := io.ReadFull(in, r.scratch[:4]); err != nil { + return nil, err + } + r.chunkSize = binary.BigEndian.Uint32(r.scratch[:4]) + + if r.hashPool != nil { + h := r.hashPool.getHash() + binary.BigEndian.PutUint32(r.scratch[:], r.chunkSize) + h.Write(r.scratch[:4]) + r.lastSum = h.Sum(nil) + r.hashPool.putHash(h) + sum := make([]byte, len(r.lastSum)) + if _, err := io.ReadFull(r.in, sum); err != nil { + return nil, err + } + if !hmac.Equal(r.lastSum, sum) { + return nil, ErrHashMismatch + } + } + + return r, nil +} + +// errNewBuffer is returned when a new buffer is completed. +var errNewBuffer = errors.New("buffer ready") + +// ErrHashMismatch is returned if the hash does not match. +var ErrHashMismatch = errors.New("hash mismatch") + +// Read implements io.Reader.Read. +func (r *Reader) Read(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + + // Total bytes completed; this is declared up front because it must be + // adjustable by the callback below. + done := 0 + + // Total bytes pending in the asynchronous workers for buffers. This is + // used to process the proper regions of the input as inline buffers. + var ( + pendingPre = r.nextInput - r.nextOutput + pendingInline = 0 + ) + + // Define our callback for completed work. + callback := func(c *chunk) error { + // Check for an inline buffer. + if pendingPre == 0 && pendingInline > 0 { + pendingInline-- + done += c.uncompressed.Len() + return nil + } + + // Copy the resulting buffer to our intermediate one, and + // return errNewBuffer to ensure that we aren't called a second + // time. This error code is handled specially below. + // + // c.buf will be freed and return to the pool when it is done. + if pendingPre > 0 { + pendingPre-- + } + r.buf = c.uncompressed + return errNewBuffer + } + + for done < len(p) { + // Do we have buffered data available? + if r.buf != nil { + n, err := r.buf.Read(p[done:]) + done += n + if err == io.EOF { + // This is the uncompressed buffer, it can be + // returned to the pool at this point. + r.buf.Reset() + bufPool.Put(r.buf) + r.buf = nil + } else if err != nil { + // Should never happen. + defer r.stop() + return done, err + } + continue + } + + // Read the length of the next chunk and reset the + // reader. The length is used to limit the reader. + // + // See writer.flush. + if _, err := io.ReadFull(r.in, r.scratch[:4]); err != nil { + // This is generally okay as long as there + // are still buffers outstanding. We actually + // just wait for completion of those buffers here + // and continue our loop. + if err := r.schedule(nil, callback); err == nil { + // We've actually finished all buffers; this is + // the normal EOF exit path. + defer r.stop() + return done, io.EOF + } else if err == errNewBuffer { + // A new buffer is now available. + continue + } else { + // Some other error occurred; we cannot + // process any further. + defer r.stop() + return done, err + } + } + l := binary.BigEndian.Uint32(r.scratch[:4]) + + // Read this chunk and schedule decompression. + compressed := bufPool.Get().(*bytes.Buffer) + if _, err := io.CopyN(compressed, r.in, int64(l)); err != nil { + // Some other error occurred; see above. + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return done, err + } + + var sum []byte + if r.hashPool != nil { + sum = make([]byte, len(r.lastSum)) + if _, err := io.ReadFull(r.in, sum); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return done, err + } + } + + // Are we doing inline decoding? + // + // Note that we need to check the length here against + // bytes.MinRead, since the bytes library will choose to grow + // the slice if the available capacity is not at least + // bytes.MinRead. This limits inline decoding to chunkSizes + // that are at least bytes.MinRead (which is not unreasonable). + var c *chunk + start := done + ((pendingPre + pendingInline) * int(r.chunkSize)) + if len(p) >= start+int(r.chunkSize) && len(p) >= start+bytes.MinRead { + c = newChunk(r.lastSum, sum, compressed, bytes.NewBuffer(p[start:start])) + pendingInline++ + } else { + c = newChunk(r.lastSum, sum, compressed, nil) + } + r.lastSum = sum + if err := r.schedule(c, callback); err == errNewBuffer { + // A new buffer was completed while we were reading. + // That's great, but we need to force schedule the + // current buffer so that it does not get lost. + // + // It is safe to pass nil as an output function here, + // because we know that we just freed up a slot above. + r.schedule(c, nil) + } else if err != nil { + // Some other error occurred; see above. + defer r.stop() + return done, err + } + } + + // Make sure that everything has been decoded successfully, otherwise + // parts of p may not actually have completed. + for pendingInline > 0 { + if err := r.schedule(nil, func(c *chunk) error { + if err := callback(c); err != nil { + return err + } + // The nil case means that an inline buffer has + // completed. The callback will have already removed + // the inline buffer from the map, so we just return an + // error to check the top of the loop again. + return errNewBuffer + }); err != errNewBuffer { + // Some other error occurred; see above. + return done, err + } + } + + // Need to return done here, since it may have been adjusted by the + // callback to compensation for partial reads on some inline buffer. + return done, nil +} + +// Close implements io.Closer.Close. +func (r *Reader) Close() error { + return r.in.Close() +} + +// Writer is a compressed writer. +type Writer struct { + pool + + // out is the underlying writer. + out io.Writer + + // closed indicates whether the file has been closed. + closed bool + + // scratch is a temporary buffer used for marshalling. This is declared + // unfront here to avoid reallocation. + scratch [4]byte +} + +var _ io.Writer = (*Writer)(nil) + +// NewWriter returns a new compressed writer. If key is non-nil, hash values are +// generated and written out for compressed bytes. See package comments for +// details. +// +// The recommended chunkSize is on the order of 1M. Extra memory may be +// buffered (in the form of read-ahead, or buffered writes), and is limited to +// O(chunkSize * [1+GOMAXPROCS]). +func NewWriter(out io.Writer, key []byte, chunkSize uint32, level int) (*Writer, error) { + w := &Writer{ + pool: pool{ + chunkSize: chunkSize, + buf: bufPool.Get().(*bytes.Buffer), + }, + out: out, + } + w.init(key, 1+runtime.GOMAXPROCS(0), true, level) + + binary.BigEndian.PutUint32(w.scratch[:], chunkSize) + if _, err := w.out.Write(w.scratch[:4]); err != nil { + return nil, err + } + + if w.hashPool != nil { + h := w.hashPool.getHash() + binary.BigEndian.PutUint32(w.scratch[:], chunkSize) + h.Write(w.scratch[:4]) + w.lastSum = h.Sum(nil) + w.hashPool.putHash(h) + if _, err := io.CopyN(w.out, bytes.NewReader(w.lastSum), int64(len(w.lastSum))); err != nil { + return nil, err + } + } + + return w, nil +} + +// flush writes a single buffer. +func (w *Writer) flush(c *chunk) error { + // Prefix each chunk with a length; this allows the reader to safely + // limit reads while buffering. + l := uint32(c.compressed.Len()) + + binary.BigEndian.PutUint32(w.scratch[:], l) + if _, err := w.out.Write(w.scratch[:4]); err != nil { + return err + } + + // Write out to the stream. + if _, err := io.CopyN(w.out, c.compressed, int64(c.compressed.Len())); err != nil { + return err + } + + if w.hashPool != nil { + io.CopyN(c.h, bytes.NewReader(w.lastSum), int64(len(w.lastSum))) + sum := c.h.Sum(nil) + w.hashPool.putHash(c.h) + c.h = nil + if _, err := io.CopyN(w.out, bytes.NewReader(sum), int64(len(sum))); err != nil { + return err + } + w.lastSum = sum + } + + return nil +} + +// Write implements io.Writer.Write. +func (w *Writer) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + // Did we close already? + if w.closed { + return 0, io.ErrUnexpectedEOF + } + + // See above; we need to track in the same way. + var ( + pendingPre = w.nextInput - w.nextOutput + pendingInline = 0 + ) + callback := func(c *chunk) error { + if pendingPre > 0 { + pendingPre-- + err := w.flush(c) + c.uncompressed.Reset() + bufPool.Put(c.uncompressed) + return err + } + if pendingInline > 0 { + pendingInline-- + return w.flush(c) + } + panic("both pendingPre and pendingInline exhausted") + } + + for done := 0; done < len(p); { + // Construct an inline buffer if we're doing an inline + // encoding; see above regarding the bytes.MinRead constraint. + inline := false + if w.buf.Len() == 0 && len(p) >= done+int(w.chunkSize) && len(p) >= done+bytes.MinRead { + bufPool.Put(w.buf) // Return to the pool; never scheduled. + w.buf = bytes.NewBuffer(p[done : done+int(w.chunkSize)]) + done += int(w.chunkSize) + pendingInline++ + inline = true + } + + // Do we need to flush w.buf? Note that this case should be hit + // immediately following the inline case above. + left := int(w.chunkSize) - w.buf.Len() + if left == 0 { + if err := w.schedule(newChunk(nil, nil, nil, w.buf), callback); err != nil { + return done, err + } + if !inline { + pendingPre++ + } + // Reset the buffer, since this has now been scheduled + // for compression. Note that this may be trampled + // immediately by the bufPool.Put(w.buf) above if the + // next buffer happens to be inline, but that's okay. + w.buf = bufPool.Get().(*bytes.Buffer) + continue + } + + // Read from p into w.buf. + toWrite := len(p) - done + if toWrite > left { + toWrite = left + } + n, err := w.buf.Write(p[done : done+toWrite]) + done += n + if err != nil { + return done, err + } + } + + // Make sure that everything has been flushed, we can't return until + // all the contents from p have been used. + for pendingInline > 0 { + if err := w.schedule(nil, func(c *chunk) error { + if err := callback(c); err != nil { + return err + } + // The flush was successful, return errNewBuffer here + // to break from the loop and check the condition + // again. + return errNewBuffer + }); err != errNewBuffer { + return len(p), err + } + } + + return len(p), nil +} + +// Close implements io.Closer.Close. +func (w *Writer) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + + // Did we already close? After the call to Close, we always mark as + // closed, regardless of whether the flush is successful. + if w.closed { + return io.ErrUnexpectedEOF + } + w.closed = true + defer w.stop() + + // Schedule any remaining partial buffer; we pass w.flush directly here + // because the final buffer is guaranteed to not be an inline buffer. + if w.buf.Len() > 0 { + if err := w.schedule(newChunk(nil, nil, nil, w.buf), w.flush); err != nil { + return err + } + } + + // Flush all scheduled buffers; see above. + if err := w.schedule(nil, w.flush); err != nil { + return err + } + + // Close the underlying writer (if necessary). + if closer, ok := w.out.(io.Closer); ok { + return closer.Close() + } + return nil +} diff --git a/pkg/compressio/compressio_state_autogen.go b/pkg/compressio/compressio_state_autogen.go new file mode 100644 index 0000000..c47e0dd --- /dev/null +++ b/pkg/compressio/compressio_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package compressio diff --git a/pkg/compressio/nocompressio.go b/pkg/compressio/nocompressio.go new file mode 100644 index 0000000..d257b8a --- /dev/null +++ b/pkg/compressio/nocompressio.go @@ -0,0 +1,352 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compressio + +import ( + "bufio" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "hash" + "io" +) + +// nocompressio provides data storage that does not use data compression but +// offers optional data integrity via SHA-256 hashing. +// +// When using data integrity option, the stream format is defined as follows: +// +// /------------------------------------------------------\ +// | data size (4-bytes) | +// +------------------------------------------------------+ +// | data | +// +------------------------------------------------------+ +// | (optional) hash (32-bytes) | +// +------------------------------------------------------+ +// | data size (4-bytes) | +// +------------------------------------------------------+ +// | ...... | +// \------------------------------------------------------/ +// +// where each hash is calculated from the following items in order +// +// data +// data size +// previous hash + +// SimpleReader is a reader for uncompressed image containing hashes. +type SimpleReader struct { + // source is the underlying stream. + source io.ReadCloser + + // bin is a bufio reader for the underlying stream. + bin *bufio.Reader + + // h is the hash object. + h hash.Hash + + // current data chunk size + chunkSize uint32 + + // current chunk position + done uint32 + + // prevHash is the previous hash value. + prevHash [sha256.Size]byte + + // scratch is a scratch buffer used for reading chunk size and hash values. + scratch [sha256.Size]byte +} + +var _ io.Reader = (*SimpleReader)(nil) + +const ( + defaultBufSize = 256 * 1024 +) + +// NewSimpleReader returns a new (uncompressed) reader. If key is non-nil, the +// data stream is assumed to contain expected hash values. See package comments +// for details. +func NewSimpleReader(in io.ReadCloser, key []byte) *SimpleReader { + bin := bufio.NewReaderSize(in, defaultBufSize) + r := &SimpleReader{ + source: in, + bin: bin, + } + if key != nil { + r.h = hmac.New(sha256.New, key) + } + return r +} + +// Read implements io.Reader.Read. +func (r *SimpleReader) Read(p []byte) (int, error) { + if r.h == nil || len(p) == 0 { + // Since there is no key, this image doesn't use the data integrity stream + // format mentioned in package comments. We can just use the bufio reader. + return r.bin.Read(p) + } + + // need next chunk? + if r.done >= r.chunkSize { + if _, err := io.ReadFull(r.bin, r.scratch[:4]); err != nil { + return 0, err + } + + r.chunkSize = binary.BigEndian.Uint32(r.scratch[:4]) + r.done = 0 + r.h.Reset() + + if r.chunkSize == 0 { + // this must not happen + return 0, io.ErrNoProgress + } + } + + toRead := uint32(len(p)) + // can't read more than what's left + if toRead > r.chunkSize-r.done { + toRead = r.chunkSize - r.done + } + + n, err := r.bin.Read(p[:toRead]) + if err != nil { + if err == io.EOF { + // this only can happen if storage or data size is corrupted, + // but we have no other means to detect it earlier as we store + // hash after the data block. + return n, ErrHashMismatch + } + return n, err + } + + // Add data to hash. + _, _ = r.h.Write(p[:n]) + r.done += uint32(n) + // Is current chunk done? + if r.done >= r.chunkSize { + // Add data size to hash. + binary.BigEndian.PutUint32(r.scratch[:4], r.chunkSize) + r.h.Write(r.scratch[:4]) + + // Add previous hash to hash. + r.h.Write(r.prevHash[:]) + + // Compute the hash into prevHash, now that we don't need the old value. + // Pass a 32-byte capacity slice (with 0 length) to avoid allocation. + r.h.Sum(r.prevHash[0:0:sha256.Size]) + + // Read the hash value from the stream. + if _, err := io.ReadFull(r.bin, r.scratch[:]); err != nil { + if err == io.EOF { + return n, io.ErrUnexpectedEOF + } + return n, err + } + + if !hmac.Equal(r.scratch[:sha256.Size], r.prevHash[:sha256.Size]) { + return n, ErrHashMismatch + } + + r.done = 0 + r.chunkSize = 0 + } + + return n, nil +} + +// Close implements io.Closer.Close. +func (r *SimpleReader) Close() error { + return r.source.Close() +} + +// SimpleWriter is a writer that does not compress. +type SimpleWriter struct { + // base is the underlying writer. + base io.Writer + + // bufOut is a buffered writer. If nil, SimpleWriter does buffering manually. + bufOut *bufio.Writer + + // h is the hash object which will be used to checksum each chunk. + h hash.Hash + + // chunkSize is the data chunk size. chunkSize is immutable. + chunkSize int + + // done is the current chunk position. + done int + + // prevHash is the previous hash value. + prevHash [sha256.Size]byte + + // buf is used to buffer the output. + buf []byte + + // closed indicates whether the file has been closed. + closed bool +} + +var ( + _ io.Writer = (*SimpleWriter)(nil) + _ io.Closer = (*SimpleWriter)(nil) +) + +// NewSimpleWriter returns a new non-compressing writer. If key is non-nil, +// hash values are generated and written out for compressed bytes. See package +// comments for details. chunkSize is the buffer size used for buffering. Large +// writes are not buffered and written out directly as a single chunk. +func NewSimpleWriter(out io.Writer, key []byte, chunkSize uint32) *SimpleWriter { + if key == nil { + // Since there is no key, this image doesn't use the data integrity stream + // format mentioned in package comments. We can just use a bufio writer. + return &SimpleWriter{ + base: out, + bufOut: bufio.NewWriterSize(out, defaultBufSize), + } + } + + return &SimpleWriter{ + base: out, + h: hmac.New(sha256.New, key), + chunkSize: int(chunkSize), + // Allocate space for the data size header and the hash. + buf: make([]byte, 4+chunkSize+sha256.Size), + } +} + +// Write implements io.Writer.Write. +func (w *SimpleWriter) Write(p []byte) (int, error) { + // Did we close already? + if w.closed { + return 0, io.ErrUnexpectedEOF + } + + if w.bufOut != nil { + return w.bufOut.Write(p) + } + + total := 0 + for len(p) > 0 { + if len(p) > w.chunkSize && w.done == 0 { + // If the payload is larger than the chunk size and we are not in the + // middle of writing another chunk, we can just write it out as one chunk. + n, err := w.directWrite(p) + return total + n, err + } + + // Copy to buffer. + n := copy(w.buf[4+w.done:4+w.chunkSize], p) + + // Update state. + w.done += n + p = p[n:] + total += n + + // Flush if necessary. + if w.done >= w.chunkSize { + if err := w.flush(); err != nil { + return total, err + } + } + } + return total, nil +} + +// Precondition: w.done == 0. +func (w *SimpleWriter) directWrite(p []byte) (int, error) { + // Write the data size. + binary.BigEndian.PutUint32(w.buf[:4], uint32(len(p))) + if _, err := w.base.Write(w.buf[:4]); err != nil { + return 0, err + } + + // Write the data. + n, err := w.base.Write(p) + if err != nil { + return n, err + } + + // Write the hash. Compute it as per package comments. + w.h.Reset() + _, _ = w.h.Write(p) + _, _ = w.h.Write(w.buf[:4]) + _, _ = w.h.Write(w.prevHash[:]) + // Compute the hash into prevHash, now that we don't need the old value. + // Pass a 32-byte capacity slice (with 0 length) to avoid allocation. + w.h.Sum(w.prevHash[0:0:sha256.Size]) + _, err = w.base.Write(w.prevHash[:sha256.Size]) + return n, err +} + +func (w *SimpleWriter) flush() error { + if w.done <= 0 { + return nil + } + + // Add the data size header at the beginning of the buffer. + binary.BigEndian.PutUint32(w.buf[:4], uint32(w.done)) + + // Compute the hash by writing the data followed by data size. + w.h.Reset() + _, _ = w.h.Write(w.buf[4 : 4+w.done]) + _, _ = w.h.Write(w.buf[:4]) + _, _ = w.h.Write(w.prevHash[:]) + + // Compute the hash into prevHash, now that we don't need the old value. + // Pass a 32-byte capacity slice (with 0 length) to avoid allocation. + w.h.Sum(w.prevHash[0:0:sha256.Size]) + // Write it after the data section in the buffer. + copy(w.buf[4+w.done:4+w.done+sha256.Size], w.prevHash[:sha256.Size]) + + // Write out to the stream. + _, err := w.base.Write(w.buf[:4+w.done+sha256.Size]) + + // Reset state. + w.done = 0 + return err +} + +// Close implements io.Closer.Close. +func (w *SimpleWriter) Close() error { + // Did we already close? After the call to Close, we always mark as + // closed, regardless of whether the flush is successful. + if w.closed { + return io.ErrUnexpectedEOF + } + w.closed = true + + // Flush buffers. + if w.bufOut != nil { + if err := w.bufOut.Flush(); err != nil { + return err + } + } else { + if err := w.flush(); err != nil { + return err + } + } + + // Close the underlying writer (if necessary). + if closer, ok := w.base.(io.Closer); ok { + return closer.Close() + } + + w.bufOut = nil + w.base = nil + w.buf = nil + + return nil +} diff --git a/pkg/context/context.go b/pkg/context/context.go new file mode 100644 index 0000000..dc0e1ef --- /dev/null +++ b/pkg/context/context.go @@ -0,0 +1,238 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package context defines an internal context type. +// +// The given Context conforms to the standard Go context, but mandates +// additional methods that are specific to the kernel internals. Note however, +// that the Context described by this package carries additional constraints +// regarding concurrent access and retaining beyond the scope of a call. +// +// See the Context type for complete details. +package context + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// Blocker represents an object with control flow hooks. +// +// These may be used to perform blocking operations, sleep or otherwise +// wait, since there may be asynchronous events that require processing. +type Blocker interface { + // Interrupt interrupts any Block operations. + Interrupt() + + // Interrupted notes whether this context is Interrupted. + Interrupted() bool + + // Killed returns true if this context is interrupted by a fatal signal. + Killed() bool + + // BlockOn blocks until one of the previously registered events occurs, + // or some external interrupt (cancellation). + // + // The return value should indicate whether the wake-up occurred as a + // result of the requested event (versus an external interrupt). + BlockOn(waiter.Waitable, waiter.EventMask) bool + + // Block blocks until an event is received from C, or some external + // interrupt. It returns nil if an event is received from C and an err if t + // is interrupted. + Block(C <-chan struct{}) error + + // BlockWithTimeoutOn blocks until either the conditions of Block are + // satisfied, or the timeout is hit. Note that deadlines are not supported + // since the notion of "with respect to what clock" is not resolved. + // + // The return value is per BlockOn. + BlockWithTimeoutOn(waiter.Waitable, waiter.EventMask, time.Duration) (time.Duration, bool) + + // UninterruptibleSleepStart indicates the beginning of an uninterruptible + // sleep state (equivalent to Linux's TASK_UNINTERRUPTIBLE). If deactivate + // is true and the Context represents a Task, the Task's AddressSpace is + // deactivated. + UninterruptibleSleepStart(deactivate bool) + + // UninterruptibleSleepFinish indicates the end of an uninterruptible sleep + // state that was begun by a previous call to UninterruptibleSleepStart. If + // activate is true and the Context represents a Task, the Task's + // AddressSpace is activated. Normally activate is the same value as the + // deactivate parameter passed to UninterruptibleSleepStart. + UninterruptibleSleepFinish(activate bool) +} + +// NoTask is an implementation of Blocker that does not block. +type NoTask struct { + cancel chan struct{} +} + +// Interrupt implements Blocker.Interrupt. +func (nt *NoTask) Interrupt() { + select { + case nt.cancel <- struct{}{}: + default: + } +} + +// Interrupted implements Blocker.Interrupted. +func (nt *NoTask) Interrupted() bool { + return nt.cancel != nil && len(nt.cancel) > 0 +} + +// Killed implements Blocker.Killed. +func (nt *NoTask) Killed() bool { + return false +} + +// Block implements Blocker.Block. +func (nt *NoTask) Block(C <-chan struct{}) error { + if nt.cancel == nil { + nt.cancel = make(chan struct{}, 1) + } + select { + case <-nt.cancel: + return errors.New("interrupted system call") // Interrupted. + case <-C: + return nil + } +} + +// BlockOn implements Blocker.BlockOn. +func (nt *NoTask) BlockOn(w waiter.Waitable, mask waiter.EventMask) bool { + if nt.cancel == nil { + nt.cancel = make(chan struct{}, 1) + } + e, ch := waiter.NewChannelEntry(mask) + w.EventRegister(&e) + defer w.EventUnregister(&e) + select { + case <-nt.cancel: + return false // Interrupted. + case _, ok := <-ch: + return ok + } +} + +// BlockWithTimeoutOn implements Blocker.BlockWithTimeoutOn. +func (nt *NoTask) BlockWithTimeoutOn(w waiter.Waitable, mask waiter.EventMask, duration time.Duration) (time.Duration, bool) { + if nt.cancel == nil { + nt.cancel = make(chan struct{}, 1) + } + e, ch := waiter.NewChannelEntry(mask) + w.EventRegister(&e) + defer w.EventUnregister(&e) + start := time.Now() // In system time. + t := time.AfterFunc(duration, func() { ch <- struct{}{} }) + select { + case <-nt.cancel: + return time.Since(start), false // Interrupted. + case _, ok := <-ch: + if ok && t.Stop() { + // Timer never fired. + return time.Since(start), ok + } + // Timer fired, remain is zero. + return time.Duration(0), ok + } +} + +// UninterruptibleSleepStart implmenents Blocker.UninterruptedSleepStart. +func (*NoTask) UninterruptibleSleepStart(bool) {} + +// UninterruptibleSleepFinish implmenents Blocker.UninterruptibleSleepFinish. +func (*NoTask) UninterruptibleSleepFinish(bool) {} + +// Context represents a thread of execution (hereafter "goroutine" to reflect +// Go idiosyncrasy). It carries state associated with the goroutine across API +// boundaries. +// +// While Context exists for essentially the same reasons as Go's standard +// context.Context, the standard type represents the state of an operation +// rather than that of a goroutine. This is a critical distinction: +// +// - Unlike context.Context, which "may be passed to functions running in +// different goroutines", it is *not safe* to use the same Context in multiple +// concurrent goroutines. +// +// - It is *not safe* to retain a Context passed to a function beyond the scope +// of that function call. +// +// In both cases, values extracted from the Context should be used instead. +type Context interface { + context.Context + log.Logger + Blocker +} + +// logContext implements basic logging. +type logContext struct { + NoTask + log.Logger + context.Context +} + +// bgContext is the context returned by context.Background. +var ( + bgContext Context + bgOnce sync.Once +) + +// Background returns an empty context using the default logger. +// Generally, one should use the Task as their context when available, or avoid +// having to use a context in places where a Task is unavailable. +// +// Using a Background context for tests is fine, as long as no values are +// needed from the context in the tested code paths. +// +// The global log.SetTarget() must be called before context.Background() +func Background() Context { + bgOnce.Do(func() { + bgContext = &logContext{ + Context: context.Background(), + Logger: log.Log(), + } + }) + return bgContext +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +func WithValue(parent Context, key, val any) Context { + return &withValue{ + Context: parent, + key: key, + val: val, + } +} + +type withValue struct { + Context + key any + val any +} + +// Value implements Context.Value. +func (ctx *withValue) Value(key any) any { + if key == ctx.key { + return ctx.val + } + return ctx.Context.Value(key) +} diff --git a/pkg/context/context_state_autogen.go b/pkg/context/context_state_autogen.go new file mode 100644 index 0000000..fdc3c9f --- /dev/null +++ b/pkg/context/context_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package context diff --git a/pkg/coretag/coretag.go b/pkg/coretag/coretag.go new file mode 100644 index 0000000..36f4a2e --- /dev/null +++ b/pkg/coretag/coretag.go @@ -0,0 +1,99 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package coretag implements core tagging. +package coretag + +import ( + "fmt" + "os" + "strconv" + + "github.com/sagernet/gvisor/pkg/abi/linux" + "golang.org/x/sys/unix" +) + +// Enable core tagging. If this returns with no error, all threads in the +// current thread group will be run in a core tagged thread. Only available on +// linux kernel >= 5.14. +func Enable() error { + // Set core tag on current thread group. + // prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, pid=0, + // PR_SCHED_CORE_SCOPE_THREAD_GROUP, cookie=nullptr) + // pid=0 means current pid. + // cookie=nullptr is required for PR_SCHED_CORE_CREATE. + if _, _, errno := unix.Syscall6(unix.SYS_PRCTL, unix.PR_SCHED_CORE, + unix.PR_SCHED_CORE_CREATE, 0 /*pid*/, linux.PR_SCHED_CORE_SCOPE_THREAD_GROUP, 0, 0); errno != 0 { + return fmt.Errorf("failed to core tag sentry: %w", errno) + } + return nil +} + +// GetAllCoreTags returns the core tag of all the threads in the thread group. +// PID 0 means the current pid. +func GetAllCoreTags(pid int) ([]uint64, error) { + // prctl(PR_SCHED_CORE_GET, PR_SCHED_CORE_SCOPE_THREAD_GROUP, ...) is not supported + // in linux. So instead we get all threads from /proc//task and get all the + // core tags individually. + tagSet := make(map[uint64]struct{}) + // Get current pid core tag. + tag, err := getCoreTag(pid) + if err != nil { + return nil, err + } + tagSet[tag] = struct{}{} + + // Get core tags of tids. + tids, err := getTids(pid) + if err != nil { + return nil, err + } + for tid := range tids { + tag, err := getCoreTag(tid) + if err != nil { + return nil, err + } + tagSet[tag] = struct{}{} + } + + // Return set of tags as a slice. + tags := make([]uint64, 0, len(tagSet)) + for t := range tagSet { + tags = append(tags, t) + } + return tags, nil +} + +// getTids returns set of tids as reported by /proc//task. +// PID 0 means the current PID. +func getTids(pid int) (map[int]struct{}, error) { + tids := make(map[int]struct{}) + path := "/proc/self/task" + if pid != 0 { + path = fmt.Sprintf("/proc/%d/task", pid) + } + files, err := os.ReadDir(path) + if err != nil { + return nil, err + } + for _, file := range files { + tid, err := strconv.Atoi(file.Name()) + if err != nil { + return nil, err + } + tids[tid] = struct{}{} + } + + return tids, nil +} diff --git a/pkg/coretag/coretag_state_autogen.go b/pkg/coretag/coretag_state_autogen.go new file mode 100644 index 0000000..c68d1db --- /dev/null +++ b/pkg/coretag/coretag_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package coretag diff --git a/pkg/coretag/coretag_unsafe.go b/pkg/coretag/coretag_unsafe.go new file mode 100644 index 0000000..996982f --- /dev/null +++ b/pkg/coretag/coretag_unsafe.go @@ -0,0 +1,34 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package coretag + +import ( + "fmt" + "unsafe" + + "github.com/sagernet/gvisor/pkg/abi/linux" + "golang.org/x/sys/unix" +) + +// getCoreTag returns the core tag of the tid. Only available on linux kernel >= 5.14. +func getCoreTag(tid int) (uint64, error) { + var cookie uint64 + if _, _, errno := unix.Syscall6(unix.SYS_PRCTL, unix.PR_SCHED_CORE, + unix.PR_SCHED_CORE_GET, uintptr(tid), linux.PR_SCHED_CORE_SCOPE_THREAD, + uintptr(unsafe.Pointer(&cookie)), 0); errno != 0 { + return 0, fmt.Errorf("prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, %d, PR_SCHED_CORE_SCOPE_THREAD) (errno=%d)", tid, errno) + } + return cookie, nil +} diff --git a/pkg/coretag/coretag_unsafe_state_autogen.go b/pkg/coretag/coretag_unsafe_state_autogen.go new file mode 100644 index 0000000..c68d1db --- /dev/null +++ b/pkg/coretag/coretag_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package coretag diff --git a/pkg/cpuid/cpuid.go b/pkg/cpuid/cpuid.go new file mode 100644 index 0000000..129d3b9 --- /dev/null +++ b/pkg/cpuid/cpuid.go @@ -0,0 +1,264 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cpuid provides basic functionality for creating and adjusting CPU +// feature sets. +// +// Each architecture should define its own FeatureSet type, that must be +// savable, along with an allFeatures map, appropriate arch hooks and a +// HostFeatureSet function. This file contains common functionality to all +// architectures, which is essentially string munging and some errors. +// +// Individual architectures may export methods on FeatureSet that are relevant, +// e.g. FeatureSet.Vendor(). Common to all architectures, FeatureSets include +// HasFeature, which provides a trivial mechanism to test for the presence of +// specific hardware features. The hardware features are also defined on a +// per-architecture basis. +package cpuid + +import ( + "encoding/binary" + "fmt" + "os" + "runtime" + "strings" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/sync" +) + +// contextID is the package for anyContext.Context.Value keys. +type contextID int + +const ( + // CtxFeatureSet is the FeatureSet for the context. + CtxFeatureSet contextID = iota + + // hardware capability bit vector. + _AT_HWCAP = 16 + // hardware capability bit vector 2. + _AT_HWCAP2 = 26 +) + +// anyContext represents context.Context. +type anyContext interface { + Value(key any) any +} + +// FromContext returns the FeatureSet from the context, if available. +func FromContext(ctx anyContext) FeatureSet { + v := ctx.Value(CtxFeatureSet) + if v == nil { + return FeatureSet{} // Panics if used. + } + return v.(FeatureSet) +} + +// Feature is a unique identifier for a particular cpu feature. We just use an +// int as a feature number on x86 and arm64. +// +// On x86, features are numbered according to "blocks". Each block is 32 bits, and +// feature bits from the same source (cpuid leaf/level) are in the same block. +// +// On arm64, features are numbered according to the ELF HWCAP definition, from +// arch/arm64/include/uapi/asm/hwcap.h. +type Feature int + +// allFeatureInfo is the value for allFeatures. +type allFeatureInfo struct { + // displayName is the short display name for the feature. + displayName string + + // shouldAppear indicates whether the feature normally appears in + // cpuinfo. This affects FlagString only. + shouldAppear bool +} + +// String implements fmt.Stringer.String. +func (f Feature) String() string { + info, ok := allFeatures[f] + if ok { + return info.displayName + } + return fmt.Sprintf("[0x%x?]", int(f)) // No given name. +} + +// reverseMap is a map from displayName to Feature. +var reverseMap = func() map[string]Feature { + m := make(map[string]Feature) + for feature, info := range allFeatures { + if info.displayName != "" { + // Sanity check that the name is unique. + if old, ok := m[info.displayName]; ok { + panic(fmt.Sprintf("feature %v has conflicting values (0x%x vs 0x%x)", info.displayName, old, feature)) + } + m[info.displayName] = feature + } + } + return m +}() + +// FeatureFromString returns the Feature associated with the given feature +// string plus a bool to indicate if it could find the feature. +func FeatureFromString(s string) (Feature, bool) { + feature, ok := reverseMap[s] + return feature, ok +} + +// AllFeatures returns the full set of all possible features. +func AllFeatures() (features []Feature) { + archFlagOrder(func(f Feature) { + features = append(features, f) + }) + return +} + +// Subtract returns the features present in fs that are not present in other. +// If all features in fs are present in other, Subtract returns nil. +// +// This does not check for any kinds of incompatibility. +func (fs FeatureSet) Subtract(other FeatureSet) (left map[Feature]struct{}) { + for feature := range allFeatures { + thisHas := fs.HasFeature(feature) + otherHas := other.HasFeature(feature) + if thisHas && !otherHas { + if left == nil { + left = make(map[Feature]struct{}) + } + left[feature] = struct{}{} + } + } + return +} + +// FlagString prints out supported CPU flags. +func (fs FeatureSet) FlagString() string { + var s []string + archFlagOrder(func(feature Feature) { + if !fs.HasFeature(feature) { + return + } + info := allFeatures[feature] + if !info.shouldAppear { + return + } + s = append(s, info.displayName) + }) + return strings.Join(s, " ") +} + +// ErrIncompatible is returned for incompatible feature sets. +type ErrIncompatible struct { + reason string +} + +// Error implements error.Error. +func (e *ErrIncompatible) Error() string { + return fmt.Sprintf("incompatible FeatureSet: %v", e.reason) +} + +// CheckHostCompatible returns nil if fs is a subset of the host feature set. +func (fs FeatureSet) CheckHostCompatible() error { + hfs := HostFeatureSet() + + // Check that hfs is a superset of fs. + if diff := fs.Subtract(hfs); len(diff) > 0 { + return &ErrIncompatible{ + reason: fmt.Sprintf("missing features: %v", diff), + } + } + + // Make arch-specific checks. + return fs.archCheckHostCompatible(hfs) +} + +// +stateify savable +type hwCap struct { + // hwCap1 stores HWCAP bits exposed through the elf auxiliary vector. + hwCap1 uint64 + // hwCap2 stores HWCAP2 bits exposed through the elf auxiliary vector. + hwCap2 uint64 +} + +// The auxiliary vector of a process on the Linux system can be read +// from /proc/self/auxv, and tags and values are stored as 8-bytes +// decimal key-value pairs on the 64-bit system. +// +// $ od -t d8 /proc/self/auxv +// +// 0000000 33 140734615224320 +// 0000020 16 3219913727 +// 0000040 6 4096 +// 0000060 17 100 +// 0000100 3 94665627353152 +// 0000120 4 56 +// 0000140 5 9 +// 0000160 7 140425502162944 +// 0000200 8 0 +// 0000220 9 94665627365760 +// 0000240 11 1000 +// 0000260 12 1000 +// 0000300 13 1000 +// 0000320 14 1000 +// 0000340 23 0 +// 0000360 25 140734614619513 +// 0000400 26 0 +// 0000420 31 140734614626284 +// 0000440 15 140734614619529 +// 0000460 0 0 +func readHWCap(auxvFilepath string) (hwCap, error) { + c := hwCap{} + if runtime.GOOS != "linux" { + // Don't try to read Linux-specific /proc files. + return c, fmt.Errorf("readHwCap only supported on linux, not %s", runtime.GOOS) + } + + auxv, err := os.ReadFile(auxvFilepath) + if err != nil { + return c, fmt.Errorf("failed to read file %s: %w", auxvFilepath, err) + } + + l := len(auxv) / 16 + for i := 0; i < l; i++ { + tag := binary.LittleEndian.Uint64(auxv[i*16:]) + val := binary.LittleEndian.Uint64(auxv[i*16+8:]) + if tag == _AT_HWCAP { + c.hwCap1 = val + } else if tag == _AT_HWCAP2 { + c.hwCap2 = val + } + + if (c.hwCap1 != 0) && (c.hwCap2 != 0) { + break + } + } + return c, nil +} + +func initHWCap() { + c, err := readHWCap("/proc/self/auxv") + if err != nil { + log.Warningf("cpuid HWCap not initialized: %w", err) + } else { + hostFeatureSet.hwCap = c + } +} + +var initOnce sync.Once + +// Initialize initializes the global data structures used by this package. +// Must be called prior to using anything else in this package. +func Initialize() { + initOnce.Do(archInitialize) +} diff --git a/pkg/cpuid/cpuid_amd64.go b/pkg/cpuid/cpuid_amd64.go new file mode 100644 index 0000000..0f7e1d0 --- /dev/null +++ b/pkg/cpuid/cpuid_amd64.go @@ -0,0 +1,496 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package cpuid + +import ( + "context" + "fmt" + "io" +) + +// FeatureSet defines features in terms of CPUID leaves and bits. +// The kernel also exposes the presence of features to userspace through +// a set of flags(HWCAP/HWCAP2) bits, exposed in the auxiliary vector, which +// are necessary to read for some features (e.g. FSGSBASE). +// +// Common references: +// +// Intel: +// - Intel SDM Volume 2, Chapter 3.2 "CPUID" (more up-to-date) +// - Intel Application Note 485 (more detailed) +// +// AMD: +// - AMD64 APM Volume 3, Appendix 3 "Obtaining Processor Information ..." +// +// +stateify savable +type FeatureSet struct { + // Function is the underlying CPUID Function. + // + // This is exported to allow direct calls of the underlying CPUID + // function, where required. + Function `state:".(Static)"` + // hwCap stores HWCAP1/2 exposed from the elf auxiliary vector. + hwCap hwCap +} + +// saveFunction saves the function as a static query. +func (fs *FeatureSet) saveFunction() Static { + if s, ok := fs.Function.(Static); ok { + return s + } + return fs.ToStatic() +} + +// loadFunction saves the function as a static query. +func (fs *FeatureSet) loadFunction(_ context.Context, s Static) { + fs.Function = s +} + +// Helper to convert 3 regs into 12-byte vendor ID. +// +//go:nosplit +func vendorIDFromRegs(bx, cx, dx uint32) (r [12]byte) { + for i := uint(0); i < 4; i++ { + b := byte(bx >> (i * 8)) + r[i] = b + } + + for i := uint(0); i < 4; i++ { + b := byte(dx >> (i * 8)) + r[4+i] = b + } + + for i := uint(0); i < 4; i++ { + b := byte(cx >> (i * 8)) + r[8+i] = b + } + + return r +} + +// Helper to merge a 12-byte vendor ID back to registers. +// +// Used by static_amd64.go. +func regsFromVendorID(r [12]byte) (bx, cx, dx uint32) { + bx |= uint32(r[0]) + bx |= uint32(r[1]) << 8 + bx |= uint32(r[2]) << 16 + bx |= uint32(r[3]) << 24 + cx |= uint32(r[4]) + cx |= uint32(r[5]) << 8 + cx |= uint32(r[6]) << 16 + cx |= uint32(r[7]) << 24 + dx |= uint32(r[8]) + dx |= uint32(r[9]) << 8 + dx |= uint32(r[10]) << 16 + dx |= uint32(r[10]) << 24 + return +} + +// VendorID is the 12-char string returned in ebx:edx:ecx for eax=0. +// +//go:nosplit +func (fs FeatureSet) VendorID() [12]byte { + _, bx, cx, dx := fs.query(vendorID) + return vendorIDFromRegs(bx, cx, dx) +} + +// Helper to deconstruct signature dword. +// +//go:nosplit +func signatureSplit(v uint32) (ef, em, pt, f, m, sid uint8) { + sid = uint8(v & 0xf) + m = uint8(v>>4) & 0xf + f = uint8(v>>8) & 0xf + pt = uint8(v>>12) & 0x3 + em = uint8(v>>16) & 0xf + ef = uint8(v >> 20) + return +} + +// ExtendedFamily is part of the processor signature. +// +//go:nosplit +func (fs FeatureSet) ExtendedFamily() uint8 { + ax, _, _, _ := fs.query(featureInfo) + ef, _, _, _, _, _ := signatureSplit(ax) + return ef +} + +// ExtendedModel is part of the processor signature. +// +//go:nosplit +func (fs FeatureSet) ExtendedModel() uint8 { + ax, _, _, _ := fs.query(featureInfo) + _, em, _, _, _, _ := signatureSplit(ax) + return em +} + +// ProcessorType is part of the processor signature. +// +//go:nosplit +func (fs FeatureSet) ProcessorType() uint8 { + ax, _, _, _ := fs.query(featureInfo) + _, _, pt, _, _, _ := signatureSplit(ax) + return pt +} + +// Family is part of the processor signature. +// +//go:nosplit +func (fs FeatureSet) Family() uint8 { + ax, _, _, _ := fs.query(featureInfo) + _, _, _, f, _, _ := signatureSplit(ax) + return f +} + +// Model is part of the processor signature. +// +//go:nosplit +func (fs FeatureSet) Model() uint8 { + ax, _, _, _ := fs.query(featureInfo) + _, _, _, _, m, _ := signatureSplit(ax) + return m +} + +// SteppingID is part of the processor signature. +// +//go:nosplit +func (fs FeatureSet) SteppingID() uint8 { + ax, _, _, _ := fs.query(featureInfo) + _, _, _, _, _, sid := signatureSplit(ax) + return sid +} + +// VirtualAddressBits returns the number of bits available for virtual +// addresses. +// +//go:nosplit +func (fs FeatureSet) VirtualAddressBits() uint32 { + ax, _, _, _ := fs.query(addressSizes) + return (ax >> 8) & 0xff +} + +// PhysicalAddressBits returns the number of bits available for physical +// addresses. +// +//go:nosplit +func (fs FeatureSet) PhysicalAddressBits() uint32 { + ax, _, _, _ := fs.query(addressSizes) + return ax & 0xff +} + +// CacheType describes the type of a cache, as returned in eax[4:0] for eax=4. +type CacheType uint8 + +const ( + // cacheNull indicates that there are no more entries. + cacheNull CacheType = iota + + // CacheData is a data cache. + CacheData + + // CacheInstruction is an instruction cache. + CacheInstruction + + // CacheUnified is a unified instruction and data cache. + CacheUnified +) + +// Cache describes the parameters of a single cache on the system. +// +// This is returned by the Caches method on FeatureSet. +type Cache struct { + // Level is the hierarchical level of this cache (L1, L2, etc). + Level uint32 + + // Type is the type of cache. + Type CacheType + + // FullyAssociative indicates that entries may be placed in any block. + FullyAssociative bool + + // Partitions is the number of physical partitions in the cache. + Partitions uint32 + + // Ways is the number of ways of associativity in the cache. + Ways uint32 + + // Sets is the number of sets in the cache. + Sets uint32 + + // InvalidateHierarchical indicates that WBINVD/INVD from threads + // sharing this cache acts upon lower level caches for threads sharing + // this cache. + InvalidateHierarchical bool + + // Inclusive indicates that this cache is inclusive of lower cache + // levels. + Inclusive bool + + // DirectMapped indicates that this cache is directly mapped from + // address, rather than using a hash function. + DirectMapped bool +} + +// Caches describes the caches on the CPU. +// +// Only supported on Intel; requires allocation. +func (fs FeatureSet) Caches() (caches []Cache) { + if !fs.Intel() { + return + } + // Check against the cache line, which should be consistent. + cacheLine := fs.CacheLine() + for i := uint32(0); ; i++ { + out := fs.Query(In{ + Eax: uint32(intelDeterministicCacheParams), + Ecx: i, + }) + t := CacheType(out.Eax & 0xf) + if t == cacheNull { + break + } + lineSize := (out.Ebx & 0xfff) + 1 + if lineSize != cacheLine { + panic(fmt.Sprintf("Mismatched cache line size: %d vs %d", lineSize, cacheLine)) + } + caches = append(caches, Cache{ + Type: t, + Level: (out.Eax >> 5) & 0x7, + FullyAssociative: ((out.Eax >> 9) & 1) == 1, + Partitions: ((out.Ebx >> 12) & 0x3ff) + 1, + Ways: ((out.Ebx >> 22) & 0x3ff) + 1, + Sets: out.Ecx + 1, + InvalidateHierarchical: (out.Edx & 1) == 0, + Inclusive: ((out.Edx >> 1) & 1) == 1, + DirectMapped: ((out.Edx >> 2) & 1) == 0, + }) + } + return +} + +// CacheLine is the size of a cache line in bytes. +// +// All caches use the same line size. This is not enforced in the CPUID +// encoding, but is true on all known x86 processors. +// +//go:nosplit +func (fs FeatureSet) CacheLine() uint32 { + _, bx, _, _ := fs.query(featureInfo) + return 8 * (bx >> 8) & 0xff +} + +// HasFeature tests whether or not a feature is in the given feature set. +// +// This function is safe to call from a nosplit context, as long as the +// FeatureSet does not have any masked features. +// +//go:nosplit +func (fs FeatureSet) HasFeature(feature Feature) bool { + return feature.check(fs) +} + +// WriteCPUInfoTo is to generate a section of one cpu in /proc/cpuinfo. This is +// a minimal /proc/cpuinfo, it is missing some fields like "microcode" that are +// not always printed in Linux. Several fields are simply made up. +func (fs FeatureSet) WriteCPUInfoTo(cpu, numCPU uint, w io.Writer) { + // Avoid many redundant calls here, since this can occasionally appear + // in the hot path. Read all basic information up front, see above. + ax, _, _, _ := fs.query(featureInfo) + ef, em, _, f, m, _ := signatureSplit(ax) + vendor := fs.VendorID() + fmt.Fprintf(w, "processor\t: %d\n", cpu) + fmt.Fprintf(w, "vendor_id\t: %s\n", string(vendor[:])) + fmt.Fprintf(w, "cpu family\t: %d\n", ((ef<<4)&0xff)|f) + fmt.Fprintf(w, "model\t\t: %d\n", ((em<<4)&0xff)|m) + fmt.Fprintf(w, "model name\t: %s\n", "unknown") // Unknown for now. + fmt.Fprintf(w, "stepping\t: %s\n", "unknown") // Unknown for now. + fmt.Fprintf(w, "cpu MHz\t\t: %.3f\n", cpuFreqMHz) + // Pretend the CPU has 8192 KB of cache. Note that real /proc/cpuinfo exposes total L3 cache + // size on Intel and per-core L2 cache size on AMD (as of Linux 6.1.0), so the value of this + // field is not really important in practice. Any value that is chosen here will be wrong + // by an order of magnitude on a significant chunk of x86 machines. + // 8192 KB is selected because it is a reasonable size that will be effectively usable on + // lightly loaded machines - most machines have 1-4MB of L3 cache per core. + fmt.Fprintf(w, "cache size\t: 8192 KB\n") + fmt.Fprintf(w, "physical id\t: 0\n") // Pretend all CPUs are in the same socket. + fmt.Fprintf(w, "siblings\t: %d\n", numCPU) + fmt.Fprintf(w, "core id\t\t: %d\n", cpu) + fmt.Fprintf(w, "cpu cores\t: %d\n", numCPU) // Pretend each CPU is a distinct core (rather than a hyperthread). + fmt.Fprintf(w, "apicid\t\t: %d\n", cpu) + fmt.Fprintf(w, "initial apicid\t: %d\n", cpu) + fmt.Fprintf(w, "fpu\t\t: yes\n") + fmt.Fprintf(w, "fpu_exception\t: yes\n") + fmt.Fprintf(w, "cpuid level\t: %d\n", uint32(xSaveInfo)) // Same as ax in vendorID. + fmt.Fprintf(w, "wp\t\t: yes\n") + fmt.Fprintf(w, "flags\t\t: %s\n", fs.FlagString()) + fmt.Fprintf(w, "bogomips\t: %.02f\n", cpuFreqMHz) // It's bogus anyway. + fmt.Fprintf(w, "clflush size\t: %d\n", fs.CacheLine()) + fmt.Fprintf(w, "cache_alignment\t: %d\n", fs.CacheLine()) + fmt.Fprintf(w, "address sizes\t: %d bits physical, %d bits virtual\n", 46, 48) + fmt.Fprintf(w, "power management:\n") // This is always here, but can be blank. + fmt.Fprintf(w, "\n") // The /proc/cpuinfo file ends with an extra newline. +} + +var ( + authenticAMD = [12]byte{'A', 'u', 't', 'h', 'e', 'n', 't', 'i', 'c', 'A', 'M', 'D'} + genuineIntel = [12]byte{'G', 'e', 'n', 'u', 'i', 'n', 'e', 'I', 'n', 't', 'e', 'l'} +) + +// AMD returns true if fs describes an AMD CPU. +// +//go:nosplit +func (fs FeatureSet) AMD() bool { + return fs.VendorID() == authenticAMD +} + +// Intel returns true if fs describes an Intel CPU. +// +//go:nosplit +func (fs FeatureSet) Intel() bool { + return fs.VendorID() == genuineIntel +} + +// Leaf 0 of xsaveinfo function returns the size for currently +// enabled xsave features in ebx, the maximum size if all valid +// features are saved with xsave in ecx, and valid XCR0 bits in +// edx:eax. +// +// If xSaveInfo isn't supported, cpuid will not fault but will +// return bogus values. +var ( + xsaveSize = native(In{Eax: uint32(xSaveInfo)}).Ebx + maxXsaveSize = native(In{Eax: uint32(xSaveInfo)}).Ecx + amxTileCfgSize = native(In{Eax: uint32(xSaveInfo), Ecx: 17}).Eax + amxTileDataSize = native(In{Eax: uint32(xSaveInfo), Ecx: 18}).Eax +) + +const ( + // XCR0AMXMask are the bits that enable xsave to operate on AMX TILECFG + // and TILEDATA. + // + // Note: TILECFG and TILEDATA are always either both enabled or both + // disabled. + // + // See Intel® 64 and IA-32 Architectures Software Developer’s Manual Vol.1 + // section 13.3 for details. + XCR0AMXMask = uint64((1 << 17) | (1 << 18)) +) + +// ExtendedStateSize returns the number of bytes needed to save the "extended +// state" for the enabled features and the boundary it must be aligned to. +// Extended state includes floating point registers, and other cpu state that's +// not associated with the normal task context. +// +// Note: the return value matches the size of signal FP state frames. +// Look at check_xstate_in_sigframe() in the kernel sources for more details. +// +//go:nosplit +func (fs FeatureSet) ExtendedStateSize() (size, align uint) { + if fs.UseXsave() { + return uint(xsaveSize), 64 + } + + // If we don't support xsave, we fall back to fxsave, which requires + // 512 bytes aligned to 16 bytes. + return 512, 16 +} + +// AMXExtendedStateSize returns the number of bytes within the "extended state" +// area that is used for AMX. +func (fs FeatureSet) AMXExtendedStateSize() uint { + if fs.UseXsave() { + xcr0 := xgetbv(0) + if (xcr0 & XCR0AMXMask) != 0 { + return uint(amxTileCfgSize + amxTileDataSize) + } + } + return 0 +} + +// ValidXCR0Mask returns the valid bits in control register XCR0. +// +// Always exclude AMX bits, because we do not support it. +// TODO(gvisor.dev/issues/9896): Implement AMX Support. +// +//go:nosplit +func (fs FeatureSet) ValidXCR0Mask() uint64 { + if !fs.HasFeature(X86FeatureXSAVE) { + return 0 + } + ax, _, _, dx := fs.query(xSaveInfo) + return (uint64(dx)<<32 | uint64(ax)) &^ XCR0AMXMask +} + +// UseXsave returns the choice of fp state saving instruction. +// +//go:nosplit +func (fs FeatureSet) UseXsave() bool { + return fs.HasFeature(X86FeatureXSAVE) && fs.HasFeature(X86FeatureOSXSAVE) +} + +// UseXsaveopt returns true if 'fs' supports the "xsaveopt" instruction. +// +//go:nosplit +func (fs FeatureSet) UseXsaveopt() bool { + return fs.UseXsave() && fs.HasFeature(X86FeatureXSAVEOPT) +} + +// UseXsavec returns true if 'fs' supports the "xsavec" instruction. +// +//go:nosplit +func (fs FeatureSet) UseXsavec() bool { + return fs.UseXsaveopt() && fs.HasFeature(X86FeatureXSAVEC) +} + +// UseFSGSBASE returns true if 'fs' supports the (RD|WR)(FS|GS)BASE instructions. +func (fs FeatureSet) UseFSGSBASE() bool { + HWCAP2_FSGSBASE := uint64(1) << 1 + return fs.HasFeature(X86FeatureFSGSBase) && ((fs.hwCap.hwCap2 & HWCAP2_FSGSBASE) != 0) +} + +// archCheckHostCompatible checks for compatibility. +func (fs FeatureSet) archCheckHostCompatible(hfs FeatureSet) error { + // The size of a cache line must match, as it is critical to correctly + // utilizing CLFLUSH. Other cache properties are allowed to change, as + // they are not important to correctness. + fsCache := fs.CacheLine() + hostCache := hfs.CacheLine() + if fsCache != hostCache { + return &ErrIncompatible{ + reason: fmt.Sprintf("CPU cache line size %d incompatible with host cache line size %d", fsCache, hostCache), + } + } + + return nil +} + +// AllowedHWCap1 returns the HWCAP1 bits that the guest is allowed to depend +// on. +func (fs FeatureSet) AllowedHWCap1() uint64 { + // HWCAPS are not supported on amd64. + return 0 +} + +// AllowedHWCap2 returns the HWCAP2 bits that the guest is allowed to depend +// on. +func (fs FeatureSet) AllowedHWCap2() uint64 { + // HWCAPS are not supported on amd64. + return 0 +} diff --git a/pkg/cpuid/cpuid_amd64_state_autogen.go b/pkg/cpuid/cpuid_amd64_state_autogen.go new file mode 100644 index 0000000..a487bbc --- /dev/null +++ b/pkg/cpuid/cpuid_amd64_state_autogen.go @@ -0,0 +1,110 @@ +// automatically generated by stateify. + +//go:build amd64 && amd64 && amd64 && amd64 && amd64 +// +build amd64,amd64,amd64,amd64,amd64 + +package cpuid + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (fs *FeatureSet) StateTypeName() string { + return "pkg/cpuid.FeatureSet" +} + +func (fs *FeatureSet) StateFields() []string { + return []string{ + "Function", + "hwCap", + } +} + +func (fs *FeatureSet) beforeSave() {} + +// +checklocksignore +func (fs *FeatureSet) StateSave(stateSinkObject state.Sink) { + fs.beforeSave() + var FunctionValue Static + FunctionValue = fs.saveFunction() + stateSinkObject.SaveValue(0, FunctionValue) + stateSinkObject.Save(1, &fs.hwCap) +} + +func (fs *FeatureSet) afterLoad(context.Context) {} + +// +checklocksignore +func (fs *FeatureSet) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(1, &fs.hwCap) + stateSourceObject.LoadValue(0, new(Static), func(y any) { fs.loadFunction(ctx, y.(Static)) }) +} + +func (i *In) StateTypeName() string { + return "pkg/cpuid.In" +} + +func (i *In) StateFields() []string { + return []string{ + "Eax", + "Ecx", + } +} + +func (i *In) beforeSave() {} + +// +checklocksignore +func (i *In) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Eax) + stateSinkObject.Save(1, &i.Ecx) +} + +func (i *In) afterLoad(context.Context) {} + +// +checklocksignore +func (i *In) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Eax) + stateSourceObject.Load(1, &i.Ecx) +} + +func (o *Out) StateTypeName() string { + return "pkg/cpuid.Out" +} + +func (o *Out) StateFields() []string { + return []string{ + "Eax", + "Ebx", + "Ecx", + "Edx", + } +} + +func (o *Out) beforeSave() {} + +// +checklocksignore +func (o *Out) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.Eax) + stateSinkObject.Save(1, &o.Ebx) + stateSinkObject.Save(2, &o.Ecx) + stateSinkObject.Save(3, &o.Edx) +} + +func (o *Out) afterLoad(context.Context) {} + +// +checklocksignore +func (o *Out) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.Eax) + stateSourceObject.Load(1, &o.Ebx) + stateSourceObject.Load(2, &o.Ecx) + stateSourceObject.Load(3, &o.Edx) +} + +func init() { + state.Register((*FeatureSet)(nil)) + state.Register((*In)(nil)) + state.Register((*Out)(nil)) +} diff --git a/pkg/cpuid/cpuid_arm64.go b/pkg/cpuid/cpuid_arm64.go new file mode 100644 index 0000000..3072a37 --- /dev/null +++ b/pkg/cpuid/cpuid_arm64.go @@ -0,0 +1,154 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package cpuid + +import ( + "fmt" + "io" +) + +// FeatureSet for ARM64 is defined as a static set of bits. +// +// ARM64 doesn't have a CPUID equivalent, which means it has no architected +// discovery mechanism for hardware features available to userspace code at +// EL0. The kernel exposes the presence of these features to userspace through +// a set of flags(HWCAP/HWCAP2) bits, exposed in the auxiliary vector. See +// Documentation/arm64/elf_hwcaps.rst for more info. +// +// Currently, only the HWCAP bits are supported. +// +// +stateify savable +type FeatureSet struct { + hwCap hwCap + cpuFreqMHz float64 + cpuImplHex uint64 + cpuArchDec uint64 + cpuVarHex uint64 + cpuPartHex uint64 + cpuRevDec uint64 +} + +// CPUImplementer is part of the processor signature. +func (fs FeatureSet) CPUImplementer() uint8 { + return uint8(fs.cpuImplHex) +} + +// CPUArchitecture is part of the processor signature. +func (fs FeatureSet) CPUArchitecture() uint8 { + return uint8(fs.cpuArchDec) +} + +// CPUVariant is part of the processor signature. +func (fs FeatureSet) CPUVariant() uint8 { + return uint8(fs.cpuVarHex) +} + +// CPUPartnum is part of the processor signature. +func (fs FeatureSet) CPUPartnum() uint16 { + return uint16(fs.cpuPartHex) +} + +// CPURevision is part of the processor signature. +func (fs FeatureSet) CPURevision() uint8 { + return uint8(fs.cpuRevDec) +} + +// ExtendedStateSize returns the number of bytes needed to save the "extended +// state" for this processor and the boundary it must be aligned to. Extended +// state includes floating point(NEON) registers, and other cpu state that's not +// associated with the normal task context. +func (fs FeatureSet) ExtendedStateSize() (size, align uint) { + // ARMv8 provide 32x128bits NEON registers. + // + // Ref arch/arm64/include/uapi/asm/ptrace.h + // struct user_fpsimd_state { + // __uint128_t vregs[32]; + // __u32 fpsr; + // __u32 fpcr; + // __u32 __reserved[2]; + // }; + return 528, 16 +} + +// HasFeature checks for the presence of a feature. +func (fs FeatureSet) HasFeature(feature Feature) bool { + return fs.hwCap.hwCap1&(1<= uint32(extendedFeatures) { + _, _, cx, dx := fs.query(extendedFeatures) + if f.block() == 5 { + return (cx & f.bit()) != 0 + } + // Ignore features duplicated from block 1 on AMD. + // These bits are reserved on Intel. + return ((dx &^ block6DuplicateMask) & f.bit()) != 0 + } + return false + case 7: + _, _, _, dx := fs.query(extendedFeatureInfo) + return (dx & f.bit()) != 0 + default: + return false + } +} + +// Block 0 constants are all of the "basic" feature bits returned by a cpuid in +// ecx with eax=1. +const ( + X86FeatureSSE3 Feature = iota + X86FeaturePCLMULDQ + X86FeatureDTES64 + X86FeatureMONITOR + X86FeatureDSCPL + X86FeatureVMX + X86FeatureSMX + X86FeatureEST + X86FeatureTM2 + X86FeatureSSSE3 // Not a typo, "supplemental" SSE3. + X86FeatureCNXTID + X86FeatureSDBG + X86FeatureFMA + X86FeatureCX16 + X86FeatureXTPR + X86FeaturePDCM + _ // ecx bit 16 is reserved. + X86FeaturePCID + X86FeatureDCA + X86FeatureSSE4_1 + X86FeatureSSE4_2 + X86FeatureX2APIC + X86FeatureMOVBE + X86FeaturePOPCNT + X86FeatureTSCD + X86FeatureAES + X86FeatureXSAVE + X86FeatureOSXSAVE + X86FeatureAVX + X86FeatureF16C + X86FeatureRDRAND + X86FeatureHypervisor +) + +// Block 1 constants are all of the "basic" feature bits returned by a cpuid in +// edx with eax=1. +const ( + X86FeatureFPU Feature = 32 + iota + X86FeatureVME + X86FeatureDE + X86FeaturePSE + X86FeatureTSC + X86FeatureMSR + X86FeaturePAE + X86FeatureMCE + X86FeatureCX8 + X86FeatureAPIC + _ // edx bit 10 is reserved. + X86FeatureSEP + X86FeatureMTRR + X86FeaturePGE + X86FeatureMCA + X86FeatureCMOV + X86FeaturePAT + X86FeaturePSE36 + X86FeaturePSN + X86FeatureCLFSH + _ // edx bit 20 is reserved. + X86FeatureDS + X86FeatureACPI + X86FeatureMMX + X86FeatureFXSR + X86FeatureSSE + X86FeatureSSE2 + X86FeatureSS + X86FeatureHTT + X86FeatureTM + X86FeatureIA64 + X86FeaturePBE +) + +// Block 2 bits are the "structured extended" features returned in ebx for +// eax=7, ecx=0. +const ( + X86FeatureFSGSBase Feature = 2*32 + iota + X86FeatureTSC_ADJUST + _ // ebx bit 2 is reserved. + X86FeatureBMI1 + X86FeatureHLE + X86FeatureAVX2 + X86FeatureFDP_EXCPTN_ONLY + X86FeatureSMEP + X86FeatureBMI2 + X86FeatureERMS + X86FeatureINVPCID + X86FeatureRTM + X86FeatureCQM + X86FeatureFPCSDS + X86FeatureMPX + X86FeatureRDT + X86FeatureAVX512F + X86FeatureAVX512DQ + X86FeatureRDSEED + X86FeatureADX + X86FeatureSMAP + X86FeatureAVX512IFMA + X86FeaturePCOMMIT + X86FeatureCLFLUSHOPT + X86FeatureCLWB + X86FeatureIPT // Intel processor trace. + X86FeatureAVX512PF + X86FeatureAVX512ER + X86FeatureAVX512CD + X86FeatureSHA + X86FeatureAVX512BW + X86FeatureAVX512VL +) + +// Block 3 bits are the "extended" features returned in ecx for eax=7, ecx=0. +const ( + X86FeaturePREFETCHWT1 Feature = 3*32 + iota + X86FeatureAVX512VBMI + X86FeatureUMIP + X86FeaturePKU + X86FeatureOSPKE + X86FeatureWAITPKG + X86FeatureAVX512_VBMI2 + X86FeatureCET_SS + X86FeatureGFNI + X86FeatureVAES + X86FeatureVPCLMULQDQ + X86FeatureAVX512_VNNI + X86FeatureAVX512_BITALG + X86FeatureTME + X86FeatureAVX512_VPOPCNTDQ + _ // ecx bit 15 is reserved + X86FeatureLA57 + // ecx bits 17-21 are reserved + _ + _ + _ + _ + _ + X86FeatureRDPID + // ecx bits 23-24 are reserved + _ + _ + X86FeatureCLDEMOTE + _ // ecx bit 26 is reserved + X86FeatureMOVDIRI + X86FeatureMOVDIR64B +) + +// Block 4 constants are for xsave capabilities in CPUID.(EAX=0DH,ECX=01H):EAX. +// The CPUID leaf is available only if 'X86FeatureXSAVE' is present. +const ( + X86FeatureXSAVEOPT Feature = 4*32 + iota + X86FeatureXSAVEC + X86FeatureXGETBV1 + X86FeatureXSAVES + // EAX[31:4] are reserved. +) + +// Block 5 constants are the extended feature bits in +// CPUID.(EAX=0x80000001):ECX. +const ( + X86FeatureLAHF64 Feature = 5*32 + iota + X86FeatureCMP_LEGACY + X86FeatureSVM + X86FeatureEXTAPIC + X86FeatureCR8_LEGACY + X86FeatureLZCNT + X86FeatureSSE4A + X86FeatureMISALIGNSSE + X86FeaturePREFETCHW + X86FeatureOSVW + X86FeatureIBS + X86FeatureXOP + X86FeatureSKINIT + X86FeatureWDT + _ // ecx bit 14 is reserved. + X86FeatureLWP + X86FeatureFMA4 + X86FeatureTCE + _ // ecx bit 18 is reserved. + _ // ecx bit 19 is reserved. + _ // ecx bit 20 is reserved. + X86FeatureTBM + X86FeatureTOPOLOGY + X86FeaturePERFCTR_CORE + X86FeaturePERFCTR_NB + _ // ecx bit 25 is reserved. + X86FeatureBPEXT + X86FeaturePERFCTR_TSC + X86FeaturePERFCTR_LLC + X86FeatureMWAITX + X86FeatureADMSKEXTN + _ // ecx bit 31 is reserved. +) + +// Block 6 constants are the extended feature bits in +// CPUID.(EAX=0x80000001):EDX. +// +// These are sparse, and so the bit positions are assigned manually. +const ( + // On AMD, EDX[24:23] | EDX[17:12] | EDX[9:0] are duplicate features + // also defined in block 1 (in identical bit positions). Those features + // are not listed here. + block6DuplicateMask = 0x183f3ff + + X86FeatureSYSCALL Feature = 6*32 + 11 + X86FeatureNX Feature = 6*32 + 20 + X86FeatureMMXEXT Feature = 6*32 + 22 + X86FeatureFXSR_OPT Feature = 6*32 + 25 + X86FeatureGBPAGES Feature = 6*32 + 26 + X86FeatureRDTSCP Feature = 6*32 + 27 + X86FeatureLM Feature = 6*32 + 29 + X86Feature3DNOWEXT Feature = 6*32 + 30 + X86Feature3DNOW Feature = 6*32 + 31 +) + +// Block 7 constants are the extended features bits in +// CPUID.(EAX=07H,ECX=0):EDX. +const ( + _ Feature = 7*32 + iota // edx bit 0 is reserved. + _ // edx bit 1 is reserved. + X86FeatureAVX512_4VNNIW + X86FeatureAVX512_4FMAPS + X86FeatureFSRM + _ // edx bit 5 is not used in Linux. + _ // edx bit 6 is reserved. + _ // edx bit 7 is reserved. + X86FeatureAVX512_VP2INTERSECT + X86FeatureSRBDS_CTRL + X86FeatureMD_CLEAR + X86FeatureRTM_ALWAYS_ABORT + _ // edx bit 12 is reserved. + X86FeatureTSX_FORCE_ABORT + X86FeatureSERIALIZE + X86FeatureHYBRID_CPU + X86FeatureTSXLDTRK + _ // edx bit 17 is reserved. + X86FeaturePCONFIG + X86FeatureARCH_LBR + X86FeatureIBT + _ // edx bit 21 is reserved. + X86FeatureAMX_BF16 + X86FeatureAVX512_FP16 + X86FeatureAMX_TILE + X86FeatureAMX_INT8 + X86FeatureSPEC_CTRL + X86FeatureINTEL_STIBP + X86FeatureFLUSH_L1D + X86FeatureARCH_CAPABILITIES + X86FeatureCORE_CAPABILITIES + X86FeatureSPEC_CTRL_SSBD +) + +// These are the extended floating point state features. They are used to +// enumerate floating point features in XCR0, XSTATE_BV, etc. +const ( + XSAVEFeatureX87 = 1 << 0 + XSAVEFeatureSSE = 1 << 1 + XSAVEFeatureAVX = 1 << 2 + XSAVEFeatureBNDREGS = 1 << 3 + XSAVEFeatureBNDCSR = 1 << 4 + XSAVEFeatureAVX512op = 1 << 5 + XSAVEFeatureAVX512zmm0 = 1 << 6 + XSAVEFeatureAVX512zmm16 = 1 << 7 + XSAVEFeaturePKRU = 1 << 9 +) + +// allFeatures is the set of allFeatures. +// +// These match names used in arch/x86/kernel/cpu/capflags.c. +var allFeatures = map[Feature]allFeatureInfo{ + // Block 0. + X86FeatureSSE3: {"pni", true}, + X86FeaturePCLMULDQ: {"pclmulqdq", true}, + X86FeatureDTES64: {"dtes64", true}, + X86FeatureMONITOR: {"monitor", true}, + X86FeatureDSCPL: {"ds_cpl", true}, + X86FeatureVMX: {"vmx", true}, + X86FeatureSMX: {"smx", true}, + X86FeatureEST: {"est", true}, + X86FeatureTM2: {"tm2", true}, + X86FeatureSSSE3: {"ssse3", true}, + X86FeatureCNXTID: {"cid", true}, + X86FeatureSDBG: {"sdbg", true}, + X86FeatureFMA: {"fma", true}, + X86FeatureCX16: {"cx16", true}, + X86FeatureXTPR: {"xtpr", true}, + X86FeaturePDCM: {"pdcm", true}, + X86FeaturePCID: {"pcid", true}, + X86FeatureDCA: {"dca", true}, + X86FeatureSSE4_1: {"sse4_1", true}, + X86FeatureSSE4_2: {"sse4_2", true}, + X86FeatureX2APIC: {"x2apic", true}, + X86FeatureMOVBE: {"movbe", true}, + X86FeaturePOPCNT: {"popcnt", true}, + X86FeatureTSCD: {"tsc_deadline_timer", true}, + X86FeatureAES: {"aes", true}, + X86FeatureXSAVE: {"xsave", true}, + X86FeatureAVX: {"avx", true}, + X86FeatureF16C: {"f16c", true}, + X86FeatureRDRAND: {"rdrand", true}, + X86FeatureHypervisor: {"hypervisor", true}, + X86FeatureOSXSAVE: {"osxsave", false}, + + // Block 1. + X86FeatureFPU: {"fpu", true}, + X86FeatureVME: {"vme", true}, + X86FeatureDE: {"de", true}, + X86FeaturePSE: {"pse", true}, + X86FeatureTSC: {"tsc", true}, + X86FeatureMSR: {"msr", true}, + X86FeaturePAE: {"pae", true}, + X86FeatureMCE: {"mce", true}, + X86FeatureCX8: {"cx8", true}, + X86FeatureAPIC: {"apic", true}, + X86FeatureSEP: {"sep", true}, + X86FeatureMTRR: {"mtrr", true}, + X86FeaturePGE: {"pge", true}, + X86FeatureMCA: {"mca", true}, + X86FeatureCMOV: {"cmov", true}, + X86FeaturePAT: {"pat", true}, + X86FeaturePSE36: {"pse36", true}, + X86FeaturePSN: {"pn", true}, + X86FeatureCLFSH: {"clflush", true}, + X86FeatureDS: {"dts", true}, + X86FeatureACPI: {"acpi", true}, + X86FeatureMMX: {"mmx", true}, + X86FeatureFXSR: {"fxsr", true}, + X86FeatureSSE: {"sse", true}, + X86FeatureSSE2: {"sse2", true}, + X86FeatureSS: {"ss", true}, + X86FeatureHTT: {"ht", true}, + X86FeatureTM: {"tm", true}, + X86FeatureIA64: {"ia64", true}, + X86FeaturePBE: {"pbe", true}, + + // Block 2. + X86FeatureFSGSBase: {"fsgsbase", true}, + X86FeatureTSC_ADJUST: {"tsc_adjust", true}, + X86FeatureBMI1: {"bmi1", true}, + X86FeatureHLE: {"hle", true}, + X86FeatureAVX2: {"avx2", true}, + X86FeatureSMEP: {"smep", true}, + X86FeatureBMI2: {"bmi2", true}, + X86FeatureERMS: {"erms", true}, + X86FeatureINVPCID: {"invpcid", true}, + X86FeatureRTM: {"rtm", true}, + X86FeatureCQM: {"cqm", true}, + X86FeatureMPX: {"mpx", true}, + X86FeatureRDT: {"rdt_a", true}, + X86FeatureAVX512F: {"avx512f", true}, + X86FeatureAVX512DQ: {"avx512dq", true}, + X86FeatureRDSEED: {"rdseed", true}, + X86FeatureADX: {"adx", true}, + X86FeatureSMAP: {"smap", true}, + X86FeatureCLWB: {"clwb", true}, + X86FeatureAVX512PF: {"avx512pf", true}, + X86FeatureAVX512ER: {"avx512er", true}, + X86FeatureAVX512CD: {"avx512cd", true}, + X86FeatureSHA: {"sha_ni", true}, + X86FeatureAVX512BW: {"avx512bw", true}, + X86FeatureAVX512VL: {"avx512vl", true}, + X86FeatureFDP_EXCPTN_ONLY: {"fdp_excptn_only", false}, + X86FeatureFPCSDS: {"fpcsds", false}, + X86FeatureIPT: {"ipt", false}, + X86FeatureCLFLUSHOPT: {"clfushopt", false}, + + // Block 3. + X86FeatureAVX512VBMI: {"avx512vbmi", true}, + X86FeatureUMIP: {"umip", true}, + X86FeaturePKU: {"pku", true}, + X86FeatureOSPKE: {"ospke", true}, + X86FeatureWAITPKG: {"waitpkg", true}, + X86FeatureAVX512_VBMI2: {"avx512_vbmi2", true}, + X86FeatureGFNI: {"gfni", true}, + X86FeatureCET_SS: {"cet_ss", false}, + X86FeatureVAES: {"vaes", true}, + X86FeatureVPCLMULQDQ: {"vpclmulqdq", true}, + X86FeatureAVX512_VNNI: {"avx512_vnni", true}, + X86FeatureAVX512_BITALG: {"avx512_bitalg", true}, + X86FeatureTME: {"tme", true}, + X86FeatureAVX512_VPOPCNTDQ: {"avx512_vpopcntdq", true}, + X86FeatureLA57: {"la57", true}, + X86FeatureRDPID: {"rdpid", true}, + X86FeatureCLDEMOTE: {"cldemote", true}, + X86FeatureMOVDIRI: {"movdiri", true}, + X86FeatureMOVDIR64B: {"movdir64b", true}, + X86FeaturePREFETCHWT1: {"prefetchwt1", false}, + + // Block 4. + X86FeatureXSAVEOPT: {"xsaveopt", true}, + X86FeatureXSAVEC: {"xsavec", true}, + X86FeatureXGETBV1: {"xgetbv1", true}, + X86FeatureXSAVES: {"xsaves", true}, + + // Block 5. + X86FeatureLAHF64: {"lahf_lm", true}, // LAHF/SAHF in long mode. + X86FeatureCMP_LEGACY: {"cmp_legacy", true}, + X86FeatureSVM: {"svm", true}, + X86FeatureEXTAPIC: {"extapic", true}, + X86FeatureCR8_LEGACY: {"cr8_legacy", true}, + X86FeatureLZCNT: {"abm", true}, // Advanced bit manipulation. + X86FeatureSSE4A: {"sse4a", true}, + X86FeatureMISALIGNSSE: {"misalignsse", true}, + X86FeaturePREFETCHW: {"3dnowprefetch", true}, + X86FeatureOSVW: {"osvw", true}, + X86FeatureIBS: {"ibs", true}, + X86FeatureXOP: {"xop", true}, + X86FeatureSKINIT: {"skinit", true}, + X86FeatureWDT: {"wdt", true}, + X86FeatureLWP: {"lwp", true}, + X86FeatureFMA4: {"fma4", true}, + X86FeatureTCE: {"tce", true}, + X86FeatureTBM: {"tbm", true}, + X86FeatureTOPOLOGY: {"topoext", true}, + X86FeaturePERFCTR_CORE: {"perfctr_core", true}, + X86FeaturePERFCTR_NB: {"perfctr_nb", true}, + X86FeatureBPEXT: {"bpext", true}, + X86FeaturePERFCTR_TSC: {"ptsc", true}, + X86FeaturePERFCTR_LLC: {"perfctr_llc", true}, + X86FeatureMWAITX: {"mwaitx", true}, + X86FeatureADMSKEXTN: {"ad_mask_extn", false}, + + // Block 6. + X86FeatureSYSCALL: {"syscall", true}, + X86FeatureNX: {"nx", true}, + X86FeatureMMXEXT: {"mmxext", true}, + X86FeatureFXSR_OPT: {"fxsr_opt", true}, + X86FeatureGBPAGES: {"pdpe1gb", true}, + X86FeatureRDTSCP: {"rdtscp", true}, + X86FeatureLM: {"lm", true}, + X86Feature3DNOWEXT: {"3dnowext", true}, + X86Feature3DNOW: {"3dnow", true}, + + // Block 7. + X86FeatureAVX512_4VNNIW: {"avx512_4vnniw", true}, + X86FeatureAVX512_4FMAPS: {"avx512_4fmaps", true}, + X86FeatureFSRM: {"fsrm", true}, + X86FeatureAVX512_VP2INTERSECT: {"avx512_vp2intersect", true}, + X86FeatureSRBDS_CTRL: {"srbds_ctrl", false}, + X86FeatureMD_CLEAR: {"md_clear", true}, + X86FeatureRTM_ALWAYS_ABORT: {"rtm_always_abort", false}, + X86FeatureTSX_FORCE_ABORT: {"tsx_force_abort", false}, + X86FeatureSERIALIZE: {"serialize", true}, + X86FeatureHYBRID_CPU: {"hybrid_cpu", false}, + X86FeatureTSXLDTRK: {"tsxldtrk", true}, + X86FeaturePCONFIG: {"pconfig", true}, + X86FeatureARCH_LBR: {"arch_lbr", true}, + X86FeatureIBT: {"ibt", true}, + X86FeatureAMX_BF16: {"amx_bf16", true}, + X86FeatureAVX512_FP16: {"avx512_fp16", true}, + X86FeatureAMX_TILE: {"amx_tile", true}, + X86FeatureAMX_INT8: {"amx_int8", true}, + X86FeatureSPEC_CTRL: {"spec_ctrl", false}, + X86FeatureINTEL_STIBP: {"intel_stibp", false}, + X86FeatureFLUSH_L1D: {"flush_l1d", true}, + X86FeatureARCH_CAPABILITIES: {"arch_capabilities", true}, + X86FeatureCORE_CAPABILITIES: {"core_capabilities", false}, + X86FeatureSPEC_CTRL_SSBD: {"spec_ctrl_ssbd", false}, +} + +// linuxBlockOrder defines the order in which linux organizes the feature +// blocks. Linux also tracks feature bits in 32-bit blocks, but in an order +// which doesn't match well here, so for the /proc/cpuinfo generation we simply +// re-map the blocks to Linux's ordering and then go through the bits in each +// block. +var linuxBlockOrder = []block{1, 6, 0, 5, 2, 4, 3, 7} + +func archFlagOrder(fn func(Feature)) { + for _, b := range linuxBlockOrder { + for i := 0; i < blockSize; i++ { + f := featureID(b, i) + if _, ok := allFeatures[f]; ok { + fn(f) + } + } + } +} diff --git a/pkg/cpuid/features_arm64.go b/pkg/cpuid/features_arm64.go new file mode 100644 index 0000000..bd39296 --- /dev/null +++ b/pkg/cpuid/features_arm64.go @@ -0,0 +1,147 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package cpuid + +const ( + // ARM64FeatureFP indicates support for single and double precision + // float point types. + ARM64FeatureFP Feature = iota + + // ARM64FeatureASIMD indicates support for Advanced SIMD with single + // and double precision float point arithmetic. + ARM64FeatureASIMD + + // ARM64FeatureEVTSTRM indicates support for the generic timer + // configured to generate events at a frequency of approximately + // 100KHz. + ARM64FeatureEVTSTRM + + // ARM64FeatureAES indicates support for AES instructions + // (AESE/AESD/AESMC/AESIMC). + ARM64FeatureAES + + // ARM64FeaturePMULL indicates support for AES instructions + // (PMULL/PMULL2). + ARM64FeaturePMULL + + // ARM64FeatureSHA1 indicates support for SHA1 instructions + // (SHA1C/SHA1P/SHA1M etc). + ARM64FeatureSHA1 + + // ARM64FeatureSHA2 indicates support for SHA2 instructions + // (SHA256H/SHA256H2/SHA256SU0 etc). + ARM64FeatureSHA2 + + // ARM64FeatureCRC32 indicates support for CRC32 instructions + // (CRC32B/CRC32H/CRC32W etc). + ARM64FeatureCRC32 + + // ARM64FeatureATOMICS indicates support for atomic instructions + // (LDADD/LDCLR/LDEOR/LDSET etc). + ARM64FeatureATOMICS + + // ARM64FeatureFPHP indicates support for half precision float point + // arithmetic. + ARM64FeatureFPHP + + // ARM64FeatureASIMDHP indicates support for ASIMD with half precision + // float point arithmetic. + ARM64FeatureASIMDHP + + // ARM64FeatureCPUID indicates support for EL0 access to certain ID + // registers is available. + ARM64FeatureCPUID + + // ARM64FeatureASIMDRDM indicates support for SQRDMLAH and SQRDMLSH + // instructions. + ARM64FeatureASIMDRDM + + // ARM64FeatureJSCVT indicates support for the FJCVTZS instruction. + ARM64FeatureJSCVT + + // ARM64FeatureFCMA indicates support for the FCMLA and FCADD + // instructions. + ARM64FeatureFCMA + + // ARM64FeatureLRCPC indicates support for the LDAPRB/LDAPRH/LDAPR + // instructions. + ARM64FeatureLRCPC + + // ARM64FeatureDCPOP indicates support for DC instruction (DC CVAP). + ARM64FeatureDCPOP + + // ARM64FeatureSHA3 indicates support for SHA3 instructions + // (EOR3/RAX1/XAR/BCAX). + ARM64FeatureSHA3 + + // ARM64FeatureSM3 indicates support for SM3 instructions + // (SM3SS1/SM3TT1A/SM3TT1B). + ARM64FeatureSM3 + + // ARM64FeatureSM4 indicates support for SM4 instructions + // (SM4E/SM4EKEY). + ARM64FeatureSM4 + + // ARM64FeatureASIMDDP indicates support for dot product instructions + // (UDOT/SDOT). + ARM64FeatureASIMDDP + + // ARM64FeatureSHA512 indicates support for SHA2 instructions + // (SHA512H/SHA512H2/SHA512SU0). + ARM64FeatureSHA512 + + // ARM64FeatureSVE indicates support for Scalable Vector Extension. + ARM64FeatureSVE + + // ARM64FeatureASIMDFHM indicates support for FMLAL and FMLSL + // instructions. + ARM64FeatureASIMDFHM +) + +var allFeatures = map[Feature]allFeatureInfo{ + ARM64FeatureFP: {"fp", true}, + ARM64FeatureASIMD: {"asimd", true}, + ARM64FeatureEVTSTRM: {"evtstrm", true}, + ARM64FeatureAES: {"aes", true}, + ARM64FeaturePMULL: {"pmull", true}, + ARM64FeatureSHA1: {"sha1", true}, + ARM64FeatureSHA2: {"sha2", true}, + ARM64FeatureCRC32: {"crc32", true}, + ARM64FeatureATOMICS: {"atomics", true}, + ARM64FeatureFPHP: {"fphp", true}, + ARM64FeatureASIMDHP: {"asimdhp", true}, + ARM64FeatureCPUID: {"cpuid", true}, + ARM64FeatureASIMDRDM: {"asimdrdm", true}, + ARM64FeatureJSCVT: {"jscvt", true}, + ARM64FeatureFCMA: {"fcma", true}, + ARM64FeatureLRCPC: {"lrcpc", true}, + ARM64FeatureDCPOP: {"dcpop", true}, + ARM64FeatureSHA3: {"sha3", true}, + ARM64FeatureSM3: {"sm3", true}, + ARM64FeatureSM4: {"sm4", true}, + ARM64FeatureASIMDDP: {"asimddp", true}, + ARM64FeatureSHA512: {"sha512", true}, + ARM64FeatureSVE: {"sve", true}, + ARM64FeatureASIMDFHM: {"asimdfhm", true}, +} + +func archFlagOrder(fn func(Feature)) { + for i := 0; i < len(allFeatures); i++ { + fn(Feature(i)) + } +} diff --git a/pkg/cpuid/hwcap_amd64.go b/pkg/cpuid/hwcap_amd64.go new file mode 100644 index 0000000..f75a0a6 --- /dev/null +++ b/pkg/cpuid/hwcap_amd64.go @@ -0,0 +1,24 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package cpuid + +// See arch/x86/include/uapi/asm/hwcap2.h +const ( + HWCAP2_RING3MWAIT = 1 << 0 + HWCAP2_FSGSBASE = 1 << 1 +) diff --git a/pkg/cpuid/hwcap_arm64.go b/pkg/cpuid/hwcap_arm64.go new file mode 100644 index 0000000..8b85bf9 --- /dev/null +++ b/pkg/cpuid/hwcap_arm64.go @@ -0,0 +1,79 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package cpuid + +// See arch/arm64/include/uapi/asm/hwcap.h +const ( + // HWCAP flags for AT_HWCAP. + HWCAP_FP = 1 << 0 + HWCAP_ASIMD = 1 << 1 + HWCAP_EVTSTRM = 1 << 2 + HWCAP_AES = 1 << 3 + HWCAP_PMULL = 1 << 4 + HWCAP_SHA1 = 1 << 5 + HWCAP_SHA2 = 1 << 6 + HWCAP_CRC32 = 1 << 7 + HWCAP_ATOMICS = 1 << 8 + HWCAP_FPHP = 1 << 9 + HWCAP_ASIMDHP = 1 << 10 + HWCAP_CPUID = 1 << 11 + HWCAP_ASIMDRDM = 1 << 12 + HWCAP_JSCVT = 1 << 13 + HWCAP_FCMA = 1 << 14 + HWCAP_LRCPC = 1 << 15 + HWCAP_DCPOP = 1 << 16 + HWCAP_SHA3 = 1 << 17 + HWCAP_SM3 = 1 << 18 + HWCAP_SM4 = 1 << 19 + HWCAP_ASIMDDP = 1 << 20 + HWCAP_SHA512 = 1 << 21 + HWCAP_SVE = 1 << 22 + HWCAP_ASIMDFHM = 1 << 23 + HWCAP_DIT = 1 << 24 + HWCAP_USCAT = 1 << 25 + HWCAP_ILRCPC = 1 << 26 + HWCAP_FLAGM = 1 << 27 + HWCAP_SSBS = 1 << 28 + HWCAP_SB = 1 << 29 + HWCAP_PACA = 1 << 30 + HWCAP_PACG = 1 << 31 + + // HWCAP2 flags for AT_HWCAP2. + HWCAP2_DCPODP = 1 << 0 + HWCAP2_SVE2 = 1 << 1 + HWCAP2_SVEAES = 1 << 2 + HWCAP2_SVEPMULL = 1 << 3 + HWCAP2_SVEBITPERM = 1 << 4 + HWCAP2_SVESHA3 = 1 << 5 + HWCAP2_SVESM4 = 1 << 6 + HWCAP2_FLAGM2 = 1 << 7 + HWCAP2_FRINT = 1 << 8 + HWCAP2_SVEI8MM = 1 << 9 + HWCAP2_SVEF32MM = 1 << 10 + HWCAP2_SVEF64MM = 1 << 11 + HWCAP2_SVEBF16 = 1 << 12 + HWCAP2_I8MM = 1 << 13 + HWCAP2_BF16 = 1 << 14 + HWCAP2_DGH = 1 << 15 + HWCAP2_RNG = 1 << 16 + HWCAP2_BTI = 1 << 17 + HWCAP2_MTE = 1 << 18 + HWCAP2_ECV = 1 << 19 + HWCAP2_AFP = 1 << 20 + HWCAP2_RPRES = 1 << 21 +) diff --git a/pkg/cpuid/native_amd64.go b/pkg/cpuid/native_amd64.go new file mode 100644 index 0000000..fb2bc5c --- /dev/null +++ b/pkg/cpuid/native_amd64.go @@ -0,0 +1,250 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package cpuid + +import ( + "bufio" + "bytes" + "os" + "strconv" + + "github.com/sagernet/gvisor/pkg/log" +) + +// cpuididFunction is a useful type wrapper. The format is eax | (ecx << 32). +type cpuidFunction uint64 + +func (f cpuidFunction) eax() uint32 { + return uint32(f) +} + +func (f cpuidFunction) ecx() uint32 { + return uint32(f >> 32) +} + +// The constants below are the lower or "standard" cpuid functions, ordered as +// defined by the hardware. Note that these may not be included in the standard +// set of functions that we are allowed to execute, which are filtered in the +// Native.Query function defined below. +const ( + vendorID cpuidFunction = 0x0 // Returns vendor ID and largest standard function. + featureInfo cpuidFunction = 0x1 // Returns basic feature bits and processor signature. + intelCacheDescriptors cpuidFunction = 0x2 // Returns list of cache descriptors. Intel only. + intelSerialNumber cpuidFunction = 0x3 // Returns processor serial number (obsolete on new hardware). Intel only. + intelDeterministicCacheParams cpuidFunction = 0x4 // Returns deterministic cache information. Intel only. + monitorMwaitParams cpuidFunction = 0x5 // Returns information about monitor/mwait instructions. + powerParams cpuidFunction = 0x6 // Returns information about power management and thermal sensors. + extendedFeatureInfo cpuidFunction = 0x7 // Returns extended feature bits. + _ // Function 0x8 is reserved. + intelDCAParams cpuidFunction = 0x9 // Returns direct cache access information. Intel only. + intelPMCInfo cpuidFunction = 0xa // Returns information about performance monitoring features. Intel only. + intelX2APICInfo cpuidFunction = 0xb // Returns core/logical processor topology. Intel only. + _ // Function 0xc is reserved. + xSaveInfo cpuidFunction = 0xd // Returns information about extended state management. + xSaveInfoSub cpuidFunction = 0xd | (0x1 << 32) // Returns information about extended state management (Sub-leaf). +) + +const xSaveInfoNumLeaves = 64 // Maximum number of xSaveInfo leaves. + +// The "extended" functions. +const ( + extendedStart cpuidFunction = 0x80000000 + extendedFunctionInfo cpuidFunction = extendedStart + 0 // Returns highest available extended function in eax. + extendedFeatures = extendedStart + 1 // Returns some extended feature bits in edx and ecx. + processorBrandString2 = extendedStart + 2 // Processor Name String Identifier. + processorBrandString3 = extendedStart + 3 // Processor Name String Identifier. + processorBrandString4 = extendedStart + 4 // Processor Name String Identifier. + l1CacheAndTLBInfo = extendedStart + 5 // Returns L2 cache information. + l2CacheInfo = extendedStart + 6 // Returns L2 cache information. + addressSizes = extendedStart + 8 // Physical and virtual address sizes. +) + +var allowedBasicFunctions = [...]bool{ + vendorID: true, + featureInfo: true, + extendedFeatureInfo: true, + intelCacheDescriptors: true, + intelDeterministicCacheParams: true, + xSaveInfo: true, +} + +var allowedExtendedFunctions = [...]bool{ + extendedFunctionInfo - extendedStart: true, + extendedFeatures - extendedStart: true, + addressSizes - extendedStart: true, + processorBrandString2 - extendedStart: true, + processorBrandString3 - extendedStart: true, + processorBrandString4 - extendedStart: true, + l1CacheAndTLBInfo - extendedStart: true, + l2CacheInfo - extendedStart: true, +} + +// Function executes a CPUID function. +// +// This is typically the native function or a Static definition. +type Function interface { + Query(In) Out +} + +// Native is a native Function. +// +// This implements Function. +type Native struct{} + +// In is input to the Query function. +// +// +stateify savable +type In struct { + Eax uint32 + Ecx uint32 +} + +// normalize drops irrelevant Ecx values. +func (i *In) normalize() { + switch cpuidFunction(i.Eax) { + case vendorID, featureInfo, intelCacheDescriptors, extendedFunctionInfo, extendedFeatures: + i.Ecx = 0 // Ignore. + case processorBrandString2, processorBrandString3, processorBrandString4, l1CacheAndTLBInfo, l2CacheInfo: + i.Ecx = 0 // Ignore. + case intelDeterministicCacheParams, extendedFeatureInfo: + // Preserve i.Ecx. + } +} + +// Out is output from the Query function. +// +// +stateify savable +type Out struct { + Eax uint32 + Ebx uint32 + Ecx uint32 + Edx uint32 +} + +// native is the native Query function. +func native(In) Out + +// Query executes CPUID natively. +// +// This implements Function. +// +//go:nosplit +func (*Native) Query(in In) Out { + if int(in.Eax) < len(allowedBasicFunctions) && allowedBasicFunctions[in.Eax] { + return native(in) + } else if in.Eax >= uint32(extendedStart) { + if l := int(in.Eax - uint32(extendedStart)); l < len(allowedExtendedFunctions) && allowedExtendedFunctions[l] { + return native(in) + } + } + return Out{} // All zeros. +} + +// query is a internal wrapper. +// +//go:nosplit +func (fs FeatureSet) query(fn cpuidFunction) (uint32, uint32, uint32, uint32) { + out := fs.Query(In{Eax: fn.eax(), Ecx: fn.ecx()}) + return out.Eax, out.Ebx, out.Ecx, out.Edx +} + +// Intersect returns the intersection of features between self and allowedFeatures. +func (fs FeatureSet) Intersect(allowedFeatures map[Feature]struct{}) (FeatureSet, error) { + hs := fs.ToStatic() + + // only keep features inside allowedFeatures. + for f := range allFeatures { + if fs.HasFeature(f) { + if _, ok := allowedFeatures[f]; !ok { + log.Infof("Removing CPU feature %v as it is not allowed.", f) + hs.Remove(f) + } + } + } + + return hs.ToFeatureSet(), nil +} + +var hostFeatureSet FeatureSet + +// HostFeatureSet returns a host CPUID. +// +//go:nosplit +func HostFeatureSet() FeatureSet { + return hostFeatureSet +} + +// cpuFreqMHz is the native CPU frequency. +var cpuFreqMHz float64 + +// Reads max cpu frequency from host /proc/cpuinfo. Must run before syscall +// filter installation. This value is used to create the fake /proc/cpuinfo +// from a FeatureSet. +func readMaxCPUFreq() { + cpuinfoFile, err := os.Open("/proc/cpuinfo") + if err != nil { + // Leave it as 0... the VDSO bails out in the same way. + log.Warningf("Could not open /proc/cpuinfo: %v", err) + return + } + defer cpuinfoFile.Close() + + // We get the value straight from host /proc/cpuinfo. On machines with + // frequency scaling enabled, this will only get the current value + // which will likely be inaccurate. This is fine on machines with + // frequency scaling disabled. + s := bufio.NewScanner(cpuinfoFile) + for s.Scan() { + line := s.Bytes() + if bytes.Contains(line, []byte("cpu MHz")) { + splitMHz := bytes.Split(line, []byte(":")) + if len(splitMHz) < 2 { + log.Warningf("Could not parse /proc/cpuinfo: malformed cpu MHz line: %q", line) + return + } + + var err error + splitMHzStr := string(bytes.TrimSpace(splitMHz[1])) + f64MHz, err := strconv.ParseFloat(splitMHzStr, 64) + if err != nil { + log.Warningf("Could not parse cpu MHz value %q: %v", splitMHzStr, err) + return + } + cpuFreqMHz = f64MHz + return + } + } + if err := s.Err(); err != nil { + log.Warningf("Could not read /proc/cpuinfo: %v", err) + return + } + log.Warningf("Could not parse /proc/cpuinfo, it is empty or does not contain cpu MHz") +} + +// xgetbv reads an extended control register. +func xgetbv(reg uintptr) uint64 + +// archInitialize initializes hostFeatureSet. +func archInitialize() { + hostFeatureSet = FeatureSet{ + Function: &Native{}, + }.Fixed() + + readMaxCPUFreq() + initHWCap() +} diff --git a/pkg/cpuid/native_amd64.s b/pkg/cpuid/native_amd64.s new file mode 100644 index 0000000..04a1433 --- /dev/null +++ b/pkg/cpuid/native_amd64.s @@ -0,0 +1,38 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +TEXT ·native(SB),NOSPLIT|NOFRAME,$0-24 + MOVL arg_Eax+0(FP), AX + MOVL arg_Ecx+4(FP), CX + CPUID + MOVL AX, ret_Eax+8(FP) + MOVL BX, ret_Ebx+12(FP) + MOVL CX, ret_Ecx+16(FP) + MOVL DX, ret_Edx+20(FP) + RET + +// xgetbv reads an extended control register. +// +// The code corresponds to: +// +// xgetbv +// +TEXT ·xgetbv(SB),NOSPLIT|NOFRAME,$0-16 + MOVQ reg+0(FP), CX + BYTE $0x0f; BYTE $0x01; BYTE $0xd0; + MOVL AX, ret+8(FP) + MOVL DX, ret+12(FP) + RET diff --git a/pkg/cpuid/native_arm64.go b/pkg/cpuid/native_arm64.go new file mode 100644 index 0000000..031ca79 --- /dev/null +++ b/pkg/cpuid/native_arm64.go @@ -0,0 +1,165 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package cpuid + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + + "github.com/sagernet/gvisor/pkg/log" +) + +// hostFeatureSet is initialized at startup. +// +// This is copied for HostFeatureSet, below. +var hostFeatureSet FeatureSet + +// HostFeatureSet returns a copy of the host FeatureSet. +func HostFeatureSet() FeatureSet { + return hostFeatureSet +} + +// Fixed returns the same feature set. +func (fs FeatureSet) Fixed() FeatureSet { + return fs +} + +// Intersect returns the intersection of features between self and allowedFeatures. +// +// Just return error as there is no ARM64 equivalent to cpuid.Static.Remove(). +func (fs FeatureSet) Intersect(allowedFeatures map[Feature]struct{}) (FeatureSet, error) { + return FeatureSet{}, fmt.Errorf("FeatureSet intersection is not supported on ARM64") +} + +// Reads CPU information from host /proc/cpuinfo. +// +// Must run before syscall filter installation. This value is used to create +// the fake /proc/cpuinfo from a FeatureSet. +func initCPUInfo() { + if runtime.GOOS != "linux" { + // Don't try to read Linux-specific /proc files or + // warn about them not existing. + return + } + cpuinfob, err := os.ReadFile("/proc/cpuinfo") + if err != nil { + // Leave everything at 0, nothing can be done. + log.Warningf("Could not read /proc/cpuinfo: %v", err) + return + } + cpuinfo := string(cpuinfob) + + // We get the value straight from host /proc/cpuinfo. + for _, line := range strings.Split(cpuinfo, "\n") { + switch { + case strings.Contains(line, "BogoMIPS"): + splitMHz := strings.Split(line, ":") + if len(splitMHz) < 2 { + log.Warningf("Could not read /proc/cpuinfo: malformed BogoMIPS") + break + } + + // If there was a problem, leave cpuFreqMHz as 0. + var err error + hostFeatureSet.cpuFreqMHz, err = strconv.ParseFloat(strings.TrimSpace(splitMHz[1]), 64) + if err != nil { + hostFeatureSet.cpuFreqMHz = 0.0 + log.Warningf("Could not parse BogoMIPS value %v: %v", splitMHz[1], err) + } + case strings.Contains(line, "CPU implementer"): + splitImpl := strings.Split(line, ":") + if len(splitImpl) < 2 { + log.Warningf("Could not read /proc/cpuinfo: malformed CPU implementer") + break + } + + // If there was a problem, leave cpuImplHex as 0. + var err error + hostFeatureSet.cpuImplHex, err = strconv.ParseUint(strings.TrimSpace(splitImpl[1]), 0, 64) + if err != nil { + hostFeatureSet.cpuImplHex = 0 + log.Warningf("Could not parse CPU implementer value %v: %v", splitImpl[1], err) + } + case strings.Contains(line, "CPU architecture"): + splitArch := strings.Split(line, ":") + if len(splitArch) < 2 { + log.Warningf("Could not read /proc/cpuinfo: malformed CPU architecture") + break + } + + // If there was a problem, leave cpuArchDec as 0. + var err error + hostFeatureSet.cpuArchDec, err = strconv.ParseUint(strings.TrimSpace(splitArch[1]), 0, 64) + if err != nil { + hostFeatureSet.cpuArchDec = 0 + log.Warningf("Could not parse CPU architecture value %v: %v", splitArch[1], err) + } + case strings.Contains(line, "CPU variant"): + splitVar := strings.Split(line, ":") + if len(splitVar) < 2 { + log.Warningf("Could not read /proc/cpuinfo: malformed CPU variant") + break + } + + // If there was a problem, leave cpuVarHex as 0. + var err error + hostFeatureSet.cpuVarHex, err = strconv.ParseUint(strings.TrimSpace(splitVar[1]), 0, 64) + if err != nil { + hostFeatureSet.cpuVarHex = 0 + log.Warningf("Could not parse CPU variant value %v: %v", splitVar[1], err) + } + case strings.Contains(line, "CPU part"): + splitPart := strings.Split(line, ":") + if len(splitPart) < 2 { + log.Warningf("Could not read /proc/cpuinfo: malformed CPU part") + break + } + + // If there was a problem, leave cpuPartHex as 0. + var err error + hostFeatureSet.cpuPartHex, err = strconv.ParseUint(strings.TrimSpace(splitPart[1]), 0, 64) + if err != nil { + hostFeatureSet.cpuPartHex = 0 + log.Warningf("Could not parse CPU part value %v: %v", splitPart[1], err) + } + case strings.Contains(line, "CPU revision"): + splitRev := strings.Split(line, ":") + if len(splitRev) < 2 { + log.Warningf("Could not read /proc/cpuinfo: malformed CPU revision") + break + } + + // If there was a problem, leave cpuRevDec as 0. + var err error + hostFeatureSet.cpuRevDec, err = strconv.ParseUint(strings.TrimSpace(splitRev[1]), 0, 64) + if err != nil { + hostFeatureSet.cpuRevDec = 0 + log.Warningf("Could not parse CPU revision value %v: %v", splitRev[1], err) + } + } + } +} + +// archInitialize initializes hostFeatureSet. +func archInitialize() { + initCPUInfo() + initHWCap() +} diff --git a/pkg/cpuid/static_amd64.go b/pkg/cpuid/static_amd64.go new file mode 100644 index 0000000..6012169 --- /dev/null +++ b/pkg/cpuid/static_amd64.go @@ -0,0 +1,135 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package cpuid + +import "context" + +// Static is a static CPUID function. +// +// +stateify savable +type Static map[In]Out + +// Fixed converts the FeatureSet to a fixed set. +func (fs FeatureSet) Fixed() FeatureSet { + sfs := fs.ToStatic().ToFeatureSet() + sfs.hwCap = fs.hwCap + return sfs +} + +// ToStatic converts a FeatureSet to a Static function. +// +// You can create a new static feature set as: +// +// fs := otherFeatureSet.ToStatic().ToFeatureSet() +func (fs FeatureSet) ToStatic() Static { + s := make(Static) + + // Save all allowed top-level functions. + for fn, allowed := range allowedBasicFunctions { + if allowed { + in := In{Eax: uint32(fn)} + s[in] = fs.Query(in) + } + } + + // Save all allowed extended functions. + for fn, allowed := range allowedExtendedFunctions { + if allowed { + in := In{Eax: uint32(fn) + uint32(extendedStart)} + s[in] = fs.Query(in) + } + } + + // Save all features (may be redundant). + for feature := range allFeatures { + feature.set(s, fs.HasFeature(feature)) + } + + // Processor Extended State Enumeration. + for i := uint32(0); i < xSaveInfoNumLeaves; i++ { + in := In{Eax: uint32(xSaveInfo), Ecx: i} + s[in] = fs.Query(in) + } + + // Save all cache information. + out := fs.Query(In{Eax: uint32(featureInfo)}) + for i := uint32(0); i < out.Ecx; i++ { + in := In{Eax: uint32(intelDeterministicCacheParams), Ecx: i} + out := fs.Query(in) + s[in] = out + if CacheType(out.Eax&0xf) == cacheNull { + break + } + } + + return s +} + +// ToFeatureSet converts a static specification to a FeatureSet. +// +// This overloads some local values, where required. +func (s Static) ToFeatureSet() FeatureSet { + // Make a copy. + ns := make(Static) + for k, v := range s { + ns[k] = v + } + ns.normalize() + return FeatureSet{ns, hwCap{}} +} + +// afterLoad calls normalize. +func (s Static) afterLoad(context.Context) { + s.normalize() +} + +// normalize normalizes FPU sizes. +func (s Static) normalize() { + // Override local FPU sizes, which must be fixed. + fs := FeatureSet{s, hwCap{}} + if fs.HasFeature(X86FeatureXSAVE) { + in := In{Eax: uint32(xSaveInfo)} + out := s[in] + out.Ecx = max(out.Ecx, maxXsaveSize) + out.Ebx = max(out.Ebx, xsaveSize) + s[in] = out + } +} + +// Add adds a feature. +func (s Static) Add(feature Feature) Static { + feature.set(s, true) + return s +} + +// Remove removes a feature. +func (s Static) Remove(feature Feature) Static { + feature.set(s, false) + return s +} + +// Set implements ChangeableSet.Set. +func (s Static) Set(in In, out Out) { + s[in] = out +} + +// Query implements Function.Query. +func (s Static) Query(in In) Out { + in.normalize() + return s[in] +} diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 0000000..81d4531 --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,40 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errors holds the standardized error definition for gVisor. +package errors + +import ( + "github.com/sagernet/gvisor/pkg/abi/linux/errno" +) + +// Error represents a syscall errno with a descriptive message. +type Error struct { + errno errno.Errno + message string +} + +// New creates a new *Error. +func New(err errno.Errno, message string) *Error { + return &Error{ + errno: err, + message: message, + } +} + +// Error implements error.Error. +func (e *Error) Error() string { return e.message } + +// Errno returns the underlying errno.Errno value. +func (e *Error) Errno() errno.Errno { return e.errno } diff --git a/pkg/errors/errors_state_autogen.go b/pkg/errors/errors_state_autogen.go new file mode 100644 index 0000000..fd45384 --- /dev/null +++ b/pkg/errors/errors_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package errors diff --git a/pkg/errors/linuxerr/internal.go b/pkg/errors/linuxerr/internal.go new file mode 100644 index 0000000..004d4d7 --- /dev/null +++ b/pkg/errors/linuxerr/internal.go @@ -0,0 +1,129 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"),; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linuxerr + +import ( + "github.com/sagernet/gvisor/pkg/abi/linux/errno" + "github.com/sagernet/gvisor/pkg/errors" +) + +var ( + // ErrWouldBlock is an internal error used to indicate that an operation + // cannot be satisfied immediately, and should be retried at a later + // time, possibly when the caller has received a notification that the + // operation may be able to complete. It is used by implementations of + // the kio.File interface. + ErrWouldBlock = errors.New(errno.EWOULDBLOCK, "request would block") + + // ErrInterrupted is returned if a request is interrupted before it can + // complete. + ErrInterrupted = errors.New(errno.EINTR, "request was interrupted") + + // ErrExceedsFileSizeLimit is returned if a request would exceed the + // file's size limit. + ErrExceedsFileSizeLimit = errors.New(errno.E2BIG, "exceeds file size limit") +) + +var errorMap = map[error]*errors.Error{ + ErrWouldBlock: EWOULDBLOCK, + ErrInterrupted: EINTR, + ErrExceedsFileSizeLimit: EFBIG, +} + +// errorUnwrappers is an array of unwrap functions to extract typed errors. +var errorUnwrappers = []func(error) (*errors.Error, bool){} + +// AddErrorUnwrapper registers an unwrap method that can extract a concrete error +// from a typed, but not initialized, error. +func AddErrorUnwrapper(unwrap func(e error) (*errors.Error, bool)) { + errorUnwrappers = append(errorUnwrappers, unwrap) +} + +// TranslateError translates errors to errnos for registered internal errors. +// It will return false if the error was not registered. +func TranslateError(from error) (*errors.Error, bool) { + if err, ok := errorMap[from]; ok { + return err, true + } + // Try to unwrap the error if we couldn't match an error + // exactly. This might mean that a package has its own + // error type. + for _, unwrap := range errorUnwrappers { + if err, ok := unwrap(from); ok { + return err, true + } + } + return nil, false +} + +// These errors are significant because ptrace syscall exit tracing can +// observe them. +// +// For all of the following errors, if the syscall is not interrupted by a +// signal delivered to a user handler, the syscall is restarted. +var ( + // ERESTARTSYS is returned by an interrupted syscall to indicate that it + // should be converted to EINTR if interrupted by a signal delivered to a + // user handler without SA_RESTART set, and restarted otherwise. + ERESTARTSYS = errors.New(errno.ERESTARTSYS, "to be restarted if SA_RESTART is set") + + // ERESTARTNOINTR is returned by an interrupted syscall to indicate that it + // should always be restarted. + ERESTARTNOINTR = errors.New(errno.ERESTARTNOINTR, "to be restarted") + + // ERESTARTNOHAND is returned by an interrupted syscall to indicate that it + // should be converted to EINTR if interrupted by a signal delivered to a + // user handler, and restarted otherwise. + ERESTARTNOHAND = errors.New(errno.ERESTARTNOHAND, "to be restarted if no handler") + + // ERESTART_RESTARTBLOCK is returned by an interrupted syscall to indicate + // that it should be restarted using a custom function. The interrupted + // syscall must register a custom restart function by calling + // Task.SetRestartSyscallFn. + ERESTART_RESTARTBLOCK = errors.New(errno.ERESTART_RESTARTBLOCK, "interrupted by signal") +) + +var restartMap = map[int]*errors.Error{ + -int(errno.ERESTARTSYS): ERESTARTSYS, + -int(errno.ERESTARTNOINTR): ERESTARTNOINTR, + -int(errno.ERESTARTNOHAND): ERESTARTNOHAND, + -int(errno.ERESTART_RESTARTBLOCK): ERESTART_RESTARTBLOCK, +} + +// IsRestartError checks if a given error is a restart error. +func IsRestartError(err error) bool { + switch err { + case ERESTARTSYS, ERESTARTNOINTR, ERESTARTNOHAND, ERESTART_RESTARTBLOCK: + return true + default: + return false + } +} + +// SyscallRestartErrorFromReturn returns the SyscallRestartErrno represented by +// rv, the value in a syscall return register. +func SyscallRestartErrorFromReturn(rv uintptr) (*errors.Error, bool) { + err, ok := restartMap[int(rv)] + return err, ok +} + +// ConvertIntr converts the provided error code (err) to another one (intr) if +// the first error corresponds to an interrupted operation. +func ConvertIntr(err, intr error) error { + if err == ErrInterrupted { + return intr + } + return err +} diff --git a/pkg/errors/linuxerr/linuxerr.go b/pkg/errors/linuxerr/linuxerr.go new file mode 100644 index 0000000..e78f23b --- /dev/null +++ b/pkg/errors/linuxerr/linuxerr.go @@ -0,0 +1,368 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"),; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package linuxerr contains syscall error codes exported as an error interface +// pointers. This allows for fast comparison and return operations comparable +// to unix.Errno constants. +package linuxerr + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/abi/linux/errno" + "github.com/sagernet/gvisor/pkg/errors" + "golang.org/x/sys/unix" +) + +const maxErrno uint32 = errno.EHWPOISON + 1 + +// The following errors are semantically identical to Errno of type unix.Errno +// or sycall.Errno. However, since the type are distinct ( these are +// *errors.Error), they are not directly comparable. However, the Errno method +// returns an Errno number such that the error can be compared to unix/syscall.Errno +// (e.g. unix.Errno(EPERM.Errno()) == unix.EPERM is true). Converting unix/syscall.Errno +// to the errors should be done via the lookup methods provided. +var ( + noError *errors.Error = nil + EPERM = errors.New(errno.EPERM, "operation not permitted") + ENOENT = errors.New(errno.ENOENT, "no such file or directory") + ESRCH = errors.New(errno.ESRCH, "no such process") + EINTR = errors.New(errno.EINTR, "interrupted system call") + EIO = errors.New(errno.EIO, "I/O error") + ENXIO = errors.New(errno.ENXIO, "no such device or address") + E2BIG = errors.New(errno.E2BIG, "argument list too long") + ENOEXEC = errors.New(errno.ENOEXEC, "exec format error") + EBADF = errors.New(errno.EBADF, "bad file number") + ECHILD = errors.New(errno.ECHILD, "no child processes") + EAGAIN = errors.New(errno.EAGAIN, "try again") + ENOMEM = errors.New(errno.ENOMEM, "out of memory") + EACCES = errors.New(errno.EACCES, "permission denied") + EFAULT = errors.New(errno.EFAULT, "bad address") + ENOTBLK = errors.New(errno.ENOTBLK, "block device required") + EBUSY = errors.New(errno.EBUSY, "device or resource busy") + EEXIST = errors.New(errno.EEXIST, "file exists") + EXDEV = errors.New(errno.EXDEV, "cross-device link") + ENODEV = errors.New(errno.ENODEV, "no such device") + ENOTDIR = errors.New(errno.ENOTDIR, "not a directory") + EISDIR = errors.New(errno.EISDIR, "is a directory") + EINVAL = errors.New(errno.EINVAL, "invalid argument") + ENFILE = errors.New(errno.ENFILE, "file table overflow") + EMFILE = errors.New(errno.EMFILE, "too many open files") + ENOTTY = errors.New(errno.ENOTTY, "not a typewriter") + ETXTBSY = errors.New(errno.ETXTBSY, "text file busy") + EFBIG = errors.New(errno.EFBIG, "file too large") + ENOSPC = errors.New(errno.ENOSPC, "no space left on device") + ESPIPE = errors.New(errno.ESPIPE, "illegal seek") + EROFS = errors.New(errno.EROFS, "read-only file system") + EMLINK = errors.New(errno.EMLINK, "too many links") + EPIPE = errors.New(errno.EPIPE, "broken pipe") + EDOM = errors.New(errno.EDOM, "math argument out of domain of func") + ERANGE = errors.New(errno.ERANGE, "math result not representable") + + // Errno values from include/uapi/asm-generic/errno.h. + EDEADLK = errors.New(errno.EDEADLK, "resource deadlock would occur") + ENAMETOOLONG = errors.New(errno.ENAMETOOLONG, "file name too long") + ENOLCK = errors.New(errno.ENOLCK, "no record locks available") + ENOSYS = errors.New(errno.ENOSYS, "invalid system call number") + ENOTEMPTY = errors.New(errno.ENOTEMPTY, "directory not empty") + ELOOP = errors.New(errno.ELOOP, "too many symbolic links encountered") + ENOMSG = errors.New(errno.ENOMSG, "no message of desired type") + EIDRM = errors.New(errno.EIDRM, "identifier removed") + ECHRNG = errors.New(errno.ECHRNG, "channel number out of range") + EL2NSYNC = errors.New(errno.EL2NSYNC, "level 2 not synchronized") + EL3HLT = errors.New(errno.EL3HLT, "level 3 halted") + EL3RST = errors.New(errno.EL3RST, "level 3 reset") + ELNRNG = errors.New(errno.ELNRNG, "link number out of range") + EUNATCH = errors.New(errno.EUNATCH, "protocol driver not attached") + ENOCSI = errors.New(errno.ENOCSI, "no CSI structure available") + EL2HLT = errors.New(errno.EL2HLT, "level 2 halted") + EBADE = errors.New(errno.EBADE, "invalid exchange") + EBADR = errors.New(errno.EBADR, "invalid request descriptor") + EXFULL = errors.New(errno.EXFULL, "exchange full") + ENOANO = errors.New(errno.ENOANO, "no anode") + EBADRQC = errors.New(errno.EBADRQC, "invalid request code") + EBADSLT = errors.New(errno.EBADSLT, "invalid slot") + EBFONT = errors.New(errno.EBFONT, "bad font file format") + ENOSTR = errors.New(errno.ENOSTR, "device not a stream") + ENODATA = errors.New(errno.ENODATA, "no data available") + ETIME = errors.New(errno.ETIME, "timer expired") + ENOSR = errors.New(errno.ENOSR, "out of streams resources") + ENOPKG = errors.New(errno.ENOPKG, "package not installed") + EREMOTE = errors.New(errno.EREMOTE, "object is remote") + ENOLINK = errors.New(errno.ENOLINK, "link has been severed") + EADV = errors.New(errno.EADV, "advertise error") + ESRMNT = errors.New(errno.ESRMNT, "srmount error") + ECOMM = errors.New(errno.ECOMM, "communication error on send") + EPROTO = errors.New(errno.EPROTO, "protocol error") + EMULTIHOP = errors.New(errno.EMULTIHOP, "multihop attempted") + EDOTDOT = errors.New(errno.EDOTDOT, "RFS specific error") + EBADMSG = errors.New(errno.EBADMSG, "not a data message") + EOVERFLOW = errors.New(errno.EOVERFLOW, "value too large for defined data type") + ENOTUNIQ = errors.New(errno.ENOTUNIQ, "name not unique on network") + EBADFD = errors.New(errno.EBADFD, "file descriptor in bad state") + EREMCHG = errors.New(errno.EREMCHG, "remote address changed") + ELIBACC = errors.New(errno.ELIBACC, "can not access a needed shared library") + ELIBBAD = errors.New(errno.ELIBBAD, "accessing a corrupted shared library") + ELIBSCN = errors.New(errno.ELIBSCN, ".lib section in a.out corrupted") + ELIBMAX = errors.New(errno.ELIBMAX, "attempting to link in too many shared libraries") + ELIBEXEC = errors.New(errno.ELIBEXEC, "cannot exec a shared library directly") + EILSEQ = errors.New(errno.EILSEQ, "illegal byte sequence") + ERESTART = errors.New(errno.ERESTART, "interrupted system call should be restarted") + ESTRPIPE = errors.New(errno.ESTRPIPE, "streams pipe error") + EUSERS = errors.New(errno.EUSERS, "too many users") + ENOTSOCK = errors.New(errno.ENOTSOCK, "socket operation on non-socket") + EDESTADDRREQ = errors.New(errno.EDESTADDRREQ, "destination address required") + EMSGSIZE = errors.New(errno.EMSGSIZE, "message too long") + EPROTOTYPE = errors.New(errno.EPROTOTYPE, "protocol wrong type for socket") + ENOPROTOOPT = errors.New(errno.ENOPROTOOPT, "protocol not available") + EPROTONOSUPPORT = errors.New(errno.EPROTONOSUPPORT, "protocol not supported") + ESOCKTNOSUPPORT = errors.New(errno.ESOCKTNOSUPPORT, "socket type not supported") + EOPNOTSUPP = errors.New(errno.EOPNOTSUPP, "operation not supported on transport endpoint") + EPFNOSUPPORT = errors.New(errno.EPFNOSUPPORT, "protocol family not supported") + EAFNOSUPPORT = errors.New(errno.EAFNOSUPPORT, "address family not supported by protocol") + EADDRINUSE = errors.New(errno.EADDRINUSE, "address already in use") + EADDRNOTAVAIL = errors.New(errno.EADDRNOTAVAIL, "cannot assign requested address") + ENETDOWN = errors.New(errno.ENETDOWN, "network is down") + ENETUNREACH = errors.New(errno.ENETUNREACH, "network is unreachable") + ENETRESET = errors.New(errno.ENETRESET, "network dropped connection because of reset") + ECONNABORTED = errors.New(errno.ECONNABORTED, "software caused connection abort") + ECONNRESET = errors.New(errno.ECONNRESET, "connection reset by peer") + ENOBUFS = errors.New(errno.ENOBUFS, "no buffer space available") + EISCONN = errors.New(errno.EISCONN, "transport endpoint is already connected") + ENOTCONN = errors.New(errno.ENOTCONN, "transport endpoint is not connected") + ESHUTDOWN = errors.New(errno.ESHUTDOWN, "cannot send after transport endpoint shutdown") + ETOOMANYREFS = errors.New(errno.ETOOMANYREFS, "too many references: cannot splice") + ETIMEDOUT = errors.New(errno.ETIMEDOUT, "connection timed out") + ECONNREFUSED = errors.New(errno.ECONNREFUSED, "connection refused") + EHOSTDOWN = errors.New(errno.EHOSTDOWN, "host is down") + EHOSTUNREACH = errors.New(errno.EHOSTUNREACH, "no route to host") + EALREADY = errors.New(errno.EALREADY, "operation already in progress") + EINPROGRESS = errors.New(errno.EINPROGRESS, "operation now in progress") + ESTALE = errors.New(errno.ESTALE, "stale file handle") + EUCLEAN = errors.New(errno.EUCLEAN, "structure needs cleaning") + ENOTNAM = errors.New(errno.ENOTNAM, "not a XENIX named type file") + ENAVAIL = errors.New(errno.ENAVAIL, "no XENIX semaphores available") + EISNAM = errors.New(errno.EISNAM, "is a named type file") + EREMOTEIO = errors.New(errno.EREMOTEIO, "remote I/O error") + EDQUOT = errors.New(errno.EDQUOT, "quota exceeded") + ENOMEDIUM = errors.New(errno.ENOMEDIUM, "no medium found") + EMEDIUMTYPE = errors.New(errno.EMEDIUMTYPE, "wrong medium type") + ECANCELED = errors.New(errno.ECANCELED, "operation Canceled") + ENOKEY = errors.New(errno.ENOKEY, "required key not available") + EKEYEXPIRED = errors.New(errno.EKEYEXPIRED, "key has expired") + EKEYREVOKED = errors.New(errno.EKEYREVOKED, "key has been revoked") + EKEYREJECTED = errors.New(errno.EKEYREJECTED, "key was rejected by service") + EOWNERDEAD = errors.New(errno.EOWNERDEAD, "owner died") + ENOTRECOVERABLE = errors.New(errno.ENOTRECOVERABLE, "state not recoverable") + ERFKILL = errors.New(errno.ERFKILL, "operation not possible due to RF-kill") + EHWPOISON = errors.New(errno.EHWPOISON, "memory page has hardware error") + + // Errors equivalent to other errors. + EWOULDBLOCK = EAGAIN + EDEADLOCK = EDEADLK + ENONET = ENOENT + ENOATTR = ENODATA + ENOTSUP = EOPNOTSUPP +) + +// A nil *errors.Error denotes no error and is placed at the 0 index of +// errorSlice. Thus, any other empty index should not be nil or a valid error. +// This marks that index as an invalid error so any comparison to nil or a +// valid linuxerr fails. +var errNotValidError = errors.New(errno.Errno(maxErrno), "not a valid error") + +// The following errorSlice holds errors by errno for fast translation between +// errnos (especially uint32(sycall.Errno)) and *errors.Error. +var errorSlice = []*errors.Error{ + // Errno values from include/uapi/asm-generic/errno-base.h. + errno.NOERRNO: noError, + errno.EPERM: EPERM, + errno.ENOENT: ENOENT, + errno.ESRCH: ESRCH, + errno.EINTR: EINTR, + errno.EIO: EIO, + errno.ENXIO: ENXIO, + errno.E2BIG: E2BIG, + errno.ENOEXEC: ENOEXEC, + errno.EBADF: EBADF, + errno.ECHILD: ECHILD, + errno.EAGAIN: EAGAIN, + errno.ENOMEM: ENOMEM, + errno.EACCES: EACCES, + errno.EFAULT: EFAULT, + errno.ENOTBLK: ENOTBLK, + errno.EBUSY: EBUSY, + errno.EEXIST: EEXIST, + errno.EXDEV: EXDEV, + errno.ENODEV: ENODEV, + errno.ENOTDIR: ENOTDIR, + errno.EISDIR: EISDIR, + errno.EINVAL: EINVAL, + errno.ENFILE: ENFILE, + errno.EMFILE: EMFILE, + errno.ENOTTY: ENOTTY, + errno.ETXTBSY: ETXTBSY, + errno.EFBIG: EFBIG, + errno.ENOSPC: ENOSPC, + errno.ESPIPE: ESPIPE, + errno.EROFS: EROFS, + errno.EMLINK: EMLINK, + errno.EPIPE: EPIPE, + errno.EDOM: EDOM, + errno.ERANGE: ERANGE, + + // Errno values from include/uapi/asm-generic/errno.h. + errno.EDEADLK: EDEADLK, + errno.ENAMETOOLONG: ENAMETOOLONG, + errno.ENOLCK: ENOLCK, + errno.ENOSYS: ENOSYS, + errno.ENOTEMPTY: ENOTEMPTY, + errno.ELOOP: ELOOP, + errno.ELOOP + 1: errNotValidError, // No valid errno between ELOOP and ENOMSG. + errno.ENOMSG: ENOMSG, + errno.EIDRM: EIDRM, + errno.ECHRNG: ECHRNG, + errno.EL2NSYNC: EL2NSYNC, + errno.EL3HLT: EL3HLT, + errno.EL3RST: EL3RST, + errno.ELNRNG: ELNRNG, + errno.EUNATCH: EUNATCH, + errno.ENOCSI: ENOCSI, + errno.EL2HLT: EL2HLT, + errno.EBADE: EBADE, + errno.EBADR: EBADR, + errno.EXFULL: EXFULL, + errno.ENOANO: ENOANO, + errno.EBADRQC: EBADRQC, + errno.EBADSLT: EBADSLT, + errno.EBADSLT + 1: errNotValidError, // No valid errno between EBADSLT and ENOPKG. + errno.EBFONT: EBFONT, + errno.ENOSTR: ENOSTR, + errno.ENODATA: ENODATA, + errno.ETIME: ETIME, + errno.ENOSR: ENOSR, + errno.ENOSR + 1: errNotValidError, // No valid errno between ENOSR and ENOPKG. + errno.ENOPKG: ENOPKG, + errno.EREMOTE: EREMOTE, + errno.ENOLINK: ENOLINK, + errno.EADV: EADV, + errno.ESRMNT: ESRMNT, + errno.ECOMM: ECOMM, + errno.EPROTO: EPROTO, + errno.EMULTIHOP: EMULTIHOP, + errno.EDOTDOT: EDOTDOT, + errno.EBADMSG: EBADMSG, + errno.EOVERFLOW: EOVERFLOW, + errno.ENOTUNIQ: ENOTUNIQ, + errno.EBADFD: EBADFD, + errno.EREMCHG: EREMCHG, + errno.ELIBACC: ELIBACC, + errno.ELIBBAD: ELIBBAD, + errno.ELIBSCN: ELIBSCN, + errno.ELIBMAX: ELIBMAX, + errno.ELIBEXEC: ELIBEXEC, + errno.EILSEQ: EILSEQ, + errno.ERESTART: ERESTART, + errno.ESTRPIPE: ESTRPIPE, + errno.EUSERS: EUSERS, + errno.ENOTSOCK: ENOTSOCK, + errno.EDESTADDRREQ: EDESTADDRREQ, + errno.EMSGSIZE: EMSGSIZE, + errno.EPROTOTYPE: EPROTOTYPE, + errno.ENOPROTOOPT: ENOPROTOOPT, + errno.EPROTONOSUPPORT: EPROTONOSUPPORT, + errno.ESOCKTNOSUPPORT: ESOCKTNOSUPPORT, + errno.EOPNOTSUPP: EOPNOTSUPP, + errno.EPFNOSUPPORT: EPFNOSUPPORT, + errno.EAFNOSUPPORT: EAFNOSUPPORT, + errno.EADDRINUSE: EADDRINUSE, + errno.EADDRNOTAVAIL: EADDRNOTAVAIL, + errno.ENETDOWN: ENETDOWN, + errno.ENETUNREACH: ENETUNREACH, + errno.ENETRESET: ENETRESET, + errno.ECONNABORTED: ECONNABORTED, + errno.ECONNRESET: ECONNRESET, + errno.ENOBUFS: ENOBUFS, + errno.EISCONN: EISCONN, + errno.ENOTCONN: ENOTCONN, + errno.ESHUTDOWN: ESHUTDOWN, + errno.ETOOMANYREFS: ETOOMANYREFS, + errno.ETIMEDOUT: ETIMEDOUT, + errno.ECONNREFUSED: ECONNREFUSED, + errno.EHOSTDOWN: EHOSTDOWN, + errno.EHOSTUNREACH: EHOSTUNREACH, + errno.EALREADY: EALREADY, + errno.EINPROGRESS: EINPROGRESS, + errno.ESTALE: ESTALE, + errno.EUCLEAN: EUCLEAN, + errno.ENOTNAM: ENOTNAM, + errno.ENAVAIL: ENAVAIL, + errno.EISNAM: EISNAM, + errno.EREMOTEIO: EREMOTEIO, + errno.EDQUOT: EDQUOT, + errno.ENOMEDIUM: ENOMEDIUM, + errno.EMEDIUMTYPE: EMEDIUMTYPE, + errno.ECANCELED: ECANCELED, + errno.ENOKEY: ENOKEY, + errno.EKEYEXPIRED: EKEYEXPIRED, + errno.EKEYREVOKED: EKEYREVOKED, + errno.EKEYREJECTED: EKEYREJECTED, + errno.EOWNERDEAD: EOWNERDEAD, + errno.ENOTRECOVERABLE: ENOTRECOVERABLE, + errno.ERFKILL: ERFKILL, + errno.EHWPOISON: EHWPOISON, +} + +// ErrorFromUnix returns a linuxerr from a unix.Errno. +func ErrorFromUnix(err unix.Errno) error { + if err == unix.Errno(0) { + return nil + } + e := errorSlice[errno.Errno(err)] + // Done this way because a single comparison in benchmarks is 2-3 faster + // than something like ( if err == nil && err > 0 ). + if e == errNotValidError { + panic(fmt.Sprintf("invalid error requested with errno: %v", e)) + } + return e +} + +// ToError converts a linuxerr to an error type. +func ToError(err *errors.Error) error { + if err == noError { + return nil + } + return err +} + +// ToUnix converts a linuxerr to a unix.Errno. +func ToUnix(e *errors.Error) unix.Errno { + var unixErr unix.Errno + if e != noError { + unixErr = unix.Errno(e.Errno()) + } + return unixErr +} + +// Equals compares a linuxerr to a given error. +func Equals(e *errors.Error, err error) bool { + var unixErr unix.Errno + if e != noError { + unixErr = unix.Errno(e.Errno()) + } + if err == nil { + err = noError + } + return e == err || unixErr == err +} diff --git a/pkg/errors/linuxerr/linuxerr_state_autogen.go b/pkg/errors/linuxerr/linuxerr_state_autogen.go new file mode 100644 index 0000000..8106a9d --- /dev/null +++ b/pkg/errors/linuxerr/linuxerr_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package linuxerr diff --git a/pkg/eventfd/eventfd.go b/pkg/eventfd/eventfd.go new file mode 100644 index 0000000..f01851e --- /dev/null +++ b/pkg/eventfd/eventfd.go @@ -0,0 +1,179 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package eventfd wraps Linux's eventfd(2) syscall. +package eventfd + +import ( + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/safecopy" + "golang.org/x/sys/unix" +) + +const sizeofUint64 = 8 + +// Eventfd represents a Linux eventfd object. +type Eventfd struct { + fd int + mmioAddr uintptr + mmioCtrl MMIOController +} + +// Create returns an initialized eventfd. +func Create() (Eventfd, error) { + fd, _, err := unix.RawSyscall(unix.SYS_EVENTFD2, 0, 0, 0) + if err != 0 { + return Eventfd{}, fmt.Errorf("failed to create eventfd: %v", error(err)) + } + if err := unix.SetNonblock(int(fd), true); err != nil { + unix.Close(int(fd)) + return Eventfd{}, err + } + return Eventfd{fd: int(fd)}, nil +} + +// Wrap returns an initialized Eventfd using the provided fd. +func Wrap(fd int) Eventfd { + return Eventfd{fd: fd} +} + +// Close closes the eventfd, after which it should not be used. +func (ev Eventfd) Close() error { + if ev.mmioCtrl != nil { + ev.mmioCtrl.Close(ev) + } + return unix.Close(ev.fd) +} + +// Dup copies the eventfd, calling dup(2) on the underlying file descriptor. +func (ev Eventfd) Dup() (Eventfd, error) { + other, err := unix.Dup(ev.fd) + if err != nil { + return Eventfd{}, fmt.Errorf("failed to dup: %v", other) + } + return Eventfd{fd: other}, nil +} + +// Notify alerts other users of the eventfd. Users can receive alerts by +// calling Wait or Read. +func (ev Eventfd) Notify() error { + return ev.Write(1) +} + +// Write writes a specific value to the eventfd. +func (ev Eventfd) Write(val uint64) error { + var buf [sizeofUint64]byte + hostarch.ByteOrder.PutUint64(buf[:], val) + if ev.mmioAddr != 0 && ev.mmioCtrl.Enabled() { + if _, err := safecopy.CopyOut(ev.mmioPtr(), buf[:]); err == nil { + return nil + } + // Fall back to using a syscall. + } + for { + n, err := nonBlockingWrite(ev.fd, buf[:]) + if err == unix.EINTR { + continue + } + if err != nil || n != sizeofUint64 { + panic(fmt.Sprintf("bad write to eventfd: got %d bytes, wanted %d with error %v", n, sizeofUint64, err)) + } + return err + } +} + +// MMIOWrite is equivalent to Write, but returns an error if the write cannot be +// implemented by writing to the address set by EnableMMIO. This is primarily +// useful for testing. +func (ev Eventfd) MMIOWrite(val uint64) error { + var buf [sizeofUint64]byte + hostarch.ByteOrder.PutUint64(buf[:], val) + if ev.mmioAddr == 0 { + return fmt.Errorf("no MMIO address set") + } + if !ev.mmioCtrl.Enabled() { + return fmt.Errorf("MMIO is temporarily disabled") + } + _, err := safecopy.CopyOut(ev.mmioPtr(), buf[:]) + return err +} + +// Wait blocks until eventfd is non-zero (i.e. someone calls Notify or Write). +func (ev Eventfd) Wait() error { + _, err := ev.Read() + return err +} + +// Read blocks until eventfd is non-zero (i.e. someone calls Notify or Write) +// and returns the value read. +func (ev Eventfd) Read() (uint64, error) { + var tmp [sizeofUint64]byte + n, errno := rawfile.BlockingRead(ev.fd, tmp[:]) + if errno != 0 { + return 0, errno + } + if n == 0 { + return 0, io.EOF + } + if n != sizeofUint64 { + panic(fmt.Sprintf("short read from eventfd: got %d bytes, wanted %d", n, sizeofUint64)) + } + return hostarch.ByteOrder.Uint64(tmp[:]), nil +} + +// FD returns the underlying file descriptor. Use with care, as this breaks the +// Eventfd abstraction. +func (ev Eventfd) FD() int { + return ev.fd +} + +// MMIOController controls eventfd memory-mapped I/O. +type MMIOController interface { + // Enabled returns true if writing to the associated MMIO address can + // succeed. This is inherently racy, so if the memory-mapped write faults, + // the eventfd will fall back to writing using a syscall. + Enabled() bool + + // Close is called when the associated Eventfd is closed. + Close(ev Eventfd) +} + +// EnableMMIO causes future calls to ev.Write() to use memory-mapped writes to +// addr, subject to ctrl. EnableMMIO cannot be called concurrently with Write, +// MMIOWrite, or MMIOAddr. +// +// This feature is used to support KVM ioeventfds. Since this requires that +// addr is mapped read-only or with no permissions in the host virtual address +// space (so that writes in host mode fault), it cannot reasonably be +// Go-managed memory, so it's safe to type as uintptr rather than a pointer. +func (ev *Eventfd) EnableMMIO(addr uintptr, ctrl MMIOController) { + ev.mmioAddr = addr + ev.mmioCtrl = ctrl +} + +// DisableMMIO undoes the effect of a previous call to EnableMMIO. DisableMMIO +// cannot be called concurrently with Write, MMIOWrite, or MMIOAddr. +func (ev *Eventfd) DisableMMIO() { + ev.mmioAddr = 0 + ev.mmioCtrl = nil +} + +// MMIOAddr returns the address set by the last call to EnableMMIO. +func (ev Eventfd) MMIOAddr() uintptr { + return ev.mmioAddr +} diff --git a/pkg/eventfd/eventfd_state_autogen.go b/pkg/eventfd/eventfd_state_autogen.go new file mode 100644 index 0000000..e83f6ae --- /dev/null +++ b/pkg/eventfd/eventfd_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package eventfd diff --git a/pkg/eventfd/eventfd_unsafe.go b/pkg/eventfd/eventfd_unsafe.go new file mode 100644 index 0000000..b1eaf9c --- /dev/null +++ b/pkg/eventfd/eventfd_unsafe.go @@ -0,0 +1,40 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventfd + +import ( + "unsafe" + + "golang.org/x/sys/unix" +) + +func (ev Eventfd) mmioPtr() unsafe.Pointer { + return unsafe.Pointer(ev.mmioAddr) +} + +// nonBlockingWrite writes the given buffer to a file descriptor. It fails if +// partial data is written. +func nonBlockingWrite(fd int, buf []byte) (int, error) { + var ptr unsafe.Pointer + if len(buf) > 0 { + ptr = unsafe.Pointer(&buf[0]) + } + + nwritten, _, errno := unix.RawSyscall(unix.SYS_WRITE, uintptr(fd), uintptr(ptr), uintptr(len(buf))) + if errno != 0 { + return int(nwritten), errno + } + return int(nwritten), nil +} diff --git a/pkg/eventfd/eventfd_unsafe_state_autogen.go b/pkg/eventfd/eventfd_unsafe_state_autogen.go new file mode 100644 index 0000000..e83f6ae --- /dev/null +++ b/pkg/eventfd/eventfd_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package eventfd diff --git a/pkg/fd/fd.go b/pkg/fd/fd.go new file mode 100644 index 0000000..4f1d039 --- /dev/null +++ b/pkg/fd/fd.go @@ -0,0 +1,273 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fd provides types for working with file descriptors. +package fd + +import ( + "fmt" + "io" + "os" + "runtime" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "golang.org/x/sys/unix" +) + +// ReadWriter implements io.ReadWriter, io.ReaderAt, and io.WriterAt for fd. It +// does not take ownership of fd. +type ReadWriter struct { + // fd is accessed atomically so FD.Close/Release can swap it. + fd atomicbitops.Int64 +} + +var ( + _ io.ReadWriter = (*ReadWriter)(nil) + _ io.ReaderAt = (*ReadWriter)(nil) + _ io.WriterAt = (*ReadWriter)(nil) +) + +// NewReadWriter creates a ReadWriter for fd. +func NewReadWriter(fd int) *ReadWriter { + return &ReadWriter{ + fd: atomicbitops.FromInt64(int64(fd)), + } +} + +func fixCount(n int, err error) (int, error) { + if n < 0 { + n = 0 + } + return n, err +} + +// Read implements io.Reader. +func (r *ReadWriter) Read(b []byte) (int, error) { + c, err := fixCount(unix.Read(r.FD(), b)) + if c == 0 && len(b) > 0 && err == nil { + return 0, io.EOF + } + return c, err +} + +// ReadAt implements io.ReaderAt. +// +// ReadAt always returns a non-nil error when c < len(b). +func (r *ReadWriter) ReadAt(b []byte, off int64) (c int, err error) { + for len(b) > 0 { + var m int + m, err = fixCount(unix.Pread(r.FD(), b, off)) + if m == 0 && err == nil { + return c, io.EOF + } + if err != nil { + return c, err + } + c += m + b = b[m:] + off += int64(m) + } + return +} + +// Write implements io.Writer. +func (r *ReadWriter) Write(b []byte) (int, error) { + var err error + var n, remaining int + for remaining = len(b); remaining > 0; { + woff := len(b) - remaining + n, err = unix.Write(r.FD(), b[woff:]) + + if n > 0 { + // unix.Write wrote some bytes. This is the common case. + remaining -= n + } else { + if err == nil { + // unix.Write did not write anything nor did it return an error. + // + // There is no way to guarantee that a subsequent unix.Write will + // make forward progress so just panic. + panic(fmt.Sprintf("unix.Write returned %d with no error", n)) + } + + if err != unix.EINTR { + // If the write failed for anything other than a signal, bail out. + break + } + } + } + + return len(b) - remaining, err +} + +// WriteAt implements io.WriterAt. +func (r *ReadWriter) WriteAt(b []byte, off int64) (c int, err error) { + for len(b) > 0 { + var m int + m, err = fixCount(unix.Pwrite(r.FD(), b, off)) + if err != nil { + break + } + c += m + b = b[m:] + off += int64(m) + } + return +} + +// FD returns the owned file descriptor. Ownership remains unchanged. +func (r *ReadWriter) FD() int { + return int(r.fd.Load()) +} + +// String implements Stringer.String(). +func (r *ReadWriter) String() string { + return fmt.Sprintf("FD: %d", r.FD()) +} + +// FD owns a host file descriptor. +// +// It is similar to os.File, with a few important distinctions: +// +// FD provides a Release() method which relinquishes ownership. Like os.File, +// FD adds a finalizer to close the backing FD. However, the finalizer cannot +// be removed from os.File, forever pinning the lifetime of an FD to its +// os.File. +// +// FD supports both blocking and non-blocking operation. os.File only +// supports blocking operation. +type FD struct { + ReadWriter +} + +// New creates a new FD. +// +// New takes ownership of fd. +func New(fd int) *FD { + if fd < 0 { + return &FD{ + ReadWriter: ReadWriter{ + fd: atomicbitops.FromInt64(-1), + }, + } + } + f := &FD{ + ReadWriter: ReadWriter{ + fd: atomicbitops.FromInt64(int64(fd)), + }, + } + runtime.SetFinalizer(f, (*FD).Close) + return f +} + +// NewFromFile creates a new FD from an os.File. +// +// NewFromFile does not transfer ownership of the file descriptor (it will be +// duplicated, so both the os.File and FD will eventually need to be closed +// and some (but not all) changes made to the FD will be applied to the +// os.File as well). +// +// The returned FD is always blocking (Go 1.9+). +func NewFromFile(file *os.File) (*FD, error) { + fd, err := unix.Dup(int(file.Fd())) + // Technically, the runtime may call the finalizer on file as soon as + // Fd() returns. + runtime.KeepAlive(file) + if err != nil { + return &FD{ + ReadWriter: ReadWriter{ + fd: atomicbitops.FromInt64(-1), + }, + }, err + } + return New(fd), nil +} + +// NewFromFiles creates new FDs for each file in the slice. +func NewFromFiles(files []*os.File) ([]*FD, error) { + rv := make([]*FD, 0, len(files)) + for _, f := range files { + new, err := NewFromFile(f) + if err != nil { + // Cleanup on error. + for _, fd := range rv { + fd.Close() + } + return nil, err + } + rv = append(rv, new) + } + return rv, nil +} + +// Open is equivalent to open(2). +func Open(path string, openmode int, perm uint32) (*FD, error) { + f, err := unix.Open(path, openmode|unix.O_LARGEFILE, perm) + if err != nil { + return nil, err + } + return New(f), nil +} + +// OpenAt is equivalent to openat(2). +func OpenAt(dir *FD, path string, flags int, mode uint32) (*FD, error) { + f, err := unix.Openat(dir.FD(), path, flags, mode) + if err != nil { + return nil, err + } + return New(f), nil +} + +// Close closes the file descriptor contained in the FD. +// +// Close is safe to call multiple times, but will return an error after the +// first call. +// +// Concurrently calling Close and any other method is undefined. +func (f *FD) Close() error { + runtime.SetFinalizer(f, nil) + return unix.Close(int(f.fd.Swap(-1))) +} + +// Release relinquishes ownership of the contained file descriptor. +// +// Concurrently calling Release and any other method is undefined. +func (f *FD) Release() int { + runtime.SetFinalizer(f, nil) + return int(f.fd.Swap(-1)) +} + +// File converts the FD to an os.File. +// +// FD does not transfer ownership of the file descriptor (it will be +// duplicated, so both the FD and os.File will eventually need to be closed +// and some (but not all) changes made to the os.File will be applied to the +// FD as well). +// +// This operation is somewhat expensive, so care should be taken to minimize +// its use. +func (f *FD) File() (*os.File, error) { + fd, err := unix.Dup(f.FD()) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), ""), nil +} + +// ReleaseToFile returns an os.File that takes ownership of the FD. +// +// name is passed to os.NewFile. +func (f *FD) ReleaseToFile(name string) *os.File { + return os.NewFile(uintptr(f.Release()), name) +} diff --git a/pkg/fd/fd_state_autogen.go b/pkg/fd/fd_state_autogen.go new file mode 100644 index 0000000..5ad4129 --- /dev/null +++ b/pkg/fd/fd_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package fd diff --git a/pkg/fdchannel/fdchannel_unsafe.go b/pkg/fdchannel/fdchannel_unsafe.go new file mode 100644 index 0000000..1eb8496 --- /dev/null +++ b/pkg/fdchannel/fdchannel_unsafe.go @@ -0,0 +1,143 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris + +// Package fdchannel implements passing file descriptors between processes over +// Unix domain sockets. +package fdchannel + +import ( + "fmt" + "unsafe" + + "golang.org/x/sys/unix" +) + +// int32 is the real type of a file descriptor. +const sizeofInt32 = int(unsafe.Sizeof(int32(0))) + +// NewConnectedSockets returns a pair of file descriptors, owned by the caller, +// representing connected sockets that may be passed to separate calls to +// NewEndpoint to create connected Endpoints. +func NewConnectedSockets() ([2]int, error) { + return unix.Socketpair(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) +} + +// Endpoint sends file descriptors to, and receives them from, another +// connected Endpoint. +// +// Endpoint is not copyable or movable by value. +type Endpoint struct { + sockfd int32 + msghdr unix.Msghdr + cmsg *unix.Cmsghdr // followed by sizeofInt32 bytes of data +} + +// Init must be called on zero-value Endpoints before first use. sockfd must be +// a blocking AF_UNIX SOCK_SEQPACKET socket. +func (ep *Endpoint) Init(sockfd int) { + // "Datagram sockets in various domains (e.g., the UNIX and Internet + // domains) permit zero-length datagrams." - recv(2). Experimentally, + // sendmsg+recvmsg for a zero-length datagram is slightly faster than + // sendmsg+recvmsg for a single byte over a stream socket. + cmsgSlice := make([]byte, unix.CmsgSpace(sizeofInt32)) + ep.sockfd = int32(sockfd) + ep.msghdr.Control = (*byte)(unsafe.Pointer(&cmsgSlice[0])) + ep.cmsg = (*unix.Cmsghdr)(unsafe.Pointer(&cmsgSlice[0])) + // ep.msghdr.Controllen and ep.cmsg.* are mutated by recvmsg(2), so they're + // set before calling sendmsg/recvmsg. +} + +// NewEndpoint is a convenience function that returns an initialized Endpoint +// allocated on the heap. +func NewEndpoint(sockfd int) *Endpoint { + ep := &Endpoint{} + ep.Init(sockfd) + return ep +} + +// Destroy releases resources owned by ep. No other Endpoint methods may be +// called after Destroy. +func (ep *Endpoint) Destroy() { + unix.Close(int(ep.sockfd)) + ep.sockfd = -1 +} + +// Shutdown causes concurrent and future calls to ep.SendFD(), ep.RecvFD(), and +// ep.RecvFDNonblock(), as well as the same calls in the connected Endpoint, to +// unblock and return errors. It does not wait for concurrent calls to return. +// +// Shutdown is the only Endpoint method that may be called concurrently with +// other methods. +func (ep *Endpoint) Shutdown() { + unix.Shutdown(int(ep.sockfd), unix.SHUT_RDWR) +} + +// SendFD sends the open file description represented by the given file +// descriptor to the connected Endpoint. +func (ep *Endpoint) SendFD(fd int) error { + cmsgLen := unix.CmsgLen(sizeofInt32) + ep.cmsg.Level = unix.SOL_SOCKET + ep.cmsg.Type = unix.SCM_RIGHTS + ep.cmsg.SetLen(cmsgLen) + *ep.cmsgData() = int32(fd) + ep.msghdr.SetControllen(cmsgLen) + _, _, e := unix.Syscall(unix.SYS_SENDMSG, uintptr(ep.sockfd), uintptr(unsafe.Pointer(&ep.msghdr)), 0) + if e != 0 { + return e + } + return nil +} + +// RecvFD receives an open file description from the connected Endpoint and +// returns a file descriptor representing it, owned by the caller. +func (ep *Endpoint) RecvFD() (int, error) { + return ep.recvFD(false) +} + +// RecvFDNonblock receives an open file description from the connected Endpoint +// and returns a file descriptor representing it, owned by the caller. If there +// are no pending receivable open file descriptions, RecvFDNonblock returns +// (, EAGAIN or EWOULDBLOCK). +func (ep *Endpoint) RecvFDNonblock() (int, error) { + return ep.recvFD(true) +} + +func (ep *Endpoint) recvFD(nonblock bool) (int, error) { + cmsgLen := unix.CmsgLen(sizeofInt32) + ep.msghdr.SetControllen(cmsgLen) + var e unix.Errno + if nonblock { + _, _, e = unix.RawSyscall(unix.SYS_RECVMSG, uintptr(ep.sockfd), uintptr(unsafe.Pointer(&ep.msghdr)), unix.MSG_TRUNC|unix.MSG_DONTWAIT) + } else { + _, _, e = unix.Syscall(unix.SYS_RECVMSG, uintptr(ep.sockfd), uintptr(unsafe.Pointer(&ep.msghdr)), unix.MSG_TRUNC) + } + if e != 0 { + return -1, e + } + if int(ep.msghdr.Controllen) != cmsgLen { + return -1, fmt.Errorf("received control message has incorrect length: got %d, wanted %d", ep.msghdr.Controllen, cmsgLen) + } + if ep.cmsg.Level != unix.SOL_SOCKET || ep.cmsg.Type != unix.SCM_RIGHTS { + return -1, fmt.Errorf("received control message has incorrect (level, type): got (%v, %v), wanted (%v, %v)", ep.cmsg.Level, ep.cmsg.Type, unix.SOL_SOCKET, unix.SCM_RIGHTS) + } + return int(*ep.cmsgData()), nil +} + +func (ep *Endpoint) cmsgData() *int32 { + // unix.CmsgLen(0) == unix.cmsgAlignOf(unix.SizeofCmsghdr) + return (*int32)(unsafe.Pointer(uintptr(unsafe.Pointer(ep.cmsg)) + uintptr(unix.CmsgLen(0)))) +} diff --git a/pkg/fdchannel/fdchannel_unsafe_state_autogen.go b/pkg/fdchannel/fdchannel_unsafe_state_autogen.go new file mode 100644 index 0000000..6ad5b7b --- /dev/null +++ b/pkg/fdchannel/fdchannel_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris + +package fdchannel diff --git a/pkg/fdnotifier/fdnotifier.go b/pkg/fdnotifier/fdnotifier.go new file mode 100644 index 0000000..da936cc --- /dev/null +++ b/pkg/fdnotifier/fdnotifier.go @@ -0,0 +1,215 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package fdnotifier contains an adapter that translates IO events (e.g., a +// file became readable/writable) from native FDs to the notifications in the +// waiter package. It uses epoll in edge-triggered mode to receive notifications +// for registered FDs. +package fdnotifier + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/waiter" + "golang.org/x/sys/unix" +) + +type fdInfo struct { + queue *waiter.Queue + waiting bool +} + +// notifier holds all the state necessary to issue notifications when IO events +// occur in the observed FDs. +type notifier struct { + // epFD is the epoll file descriptor used to register for io + // notifications. + epFD int + + // mu protects fdMap. + mu sync.Mutex + + // fdMap maps file descriptors to their notification queues and waiting + // status. + fdMap map[int32]*fdInfo +} + +// newNotifier creates a new notifier object. +func newNotifier() (*notifier, error) { + epfd, err := unix.EpollCreate1(0) + if err != nil { + return nil, err + } + + w := ¬ifier{ + epFD: epfd, + fdMap: make(map[int32]*fdInfo), + } + + go w.waitAndNotify() // S/R-SAFE: no waiter exists during save / load. + + return w, nil +} + +// waitFD waits on mask for fd. The fdMap mutex must be hold. +func (n *notifier) waitFD(fd int32, fi *fdInfo, mask waiter.EventMask) error { + if !fi.waiting && mask == 0 { + return nil + } + + e := unix.EpollEvent{ + Events: mask.ToLinux() | unix.EPOLLET, + Fd: fd, + } + + switch { + case !fi.waiting && mask != 0: + if err := unix.EpollCtl(n.epFD, unix.EPOLL_CTL_ADD, int(fd), &e); err != nil { + return err + } + fi.waiting = true + case fi.waiting && mask == 0: + unix.EpollCtl(n.epFD, unix.EPOLL_CTL_DEL, int(fd), nil) + fi.waiting = false + case fi.waiting && mask != 0: + if err := unix.EpollCtl(n.epFD, unix.EPOLL_CTL_MOD, int(fd), &e); err != nil { + return err + } + } + + return nil +} + +// addFD adds an FD to the list of FDs observed by n. +func (n *notifier) addFD(fd int32, queue *waiter.Queue) error { + n.mu.Lock() + defer n.mu.Unlock() + + // Panic if we're already notifying on this FD. + if _, ok := n.fdMap[fd]; ok { + panic(fmt.Sprintf("File descriptor %v added twice", fd)) + } + + info := &fdInfo{queue: queue} + // We might already have something in queue to wait for. + if err := n.waitFD(fd, info, queue.Events()); err != nil { + return err + } + // Add it to the map. + n.fdMap[fd] = info + return nil +} + +// updateFD updates the set of events the fd needs to be notified on. +func (n *notifier) updateFD(fd int32) error { + n.mu.Lock() + defer n.mu.Unlock() + + if fi, ok := n.fdMap[fd]; ok { + return n.waitFD(fd, fi, fi.queue.Events()) + } + + return nil +} + +// RemoveFD removes an FD from the list of FDs observed by n. +func (n *notifier) removeFD(fd int32) { + n.mu.Lock() + defer n.mu.Unlock() + + // Remove from map, then from epoll object. + n.waitFD(fd, n.fdMap[fd], 0) + delete(n.fdMap, fd) +} + +// hasFD returns true if the fd is in the list of observed FDs. +func (n *notifier) hasFD(fd int32) bool { + n.mu.Lock() + defer n.mu.Unlock() + + _, ok := n.fdMap[fd] + return ok +} + +// waitAndNotify run is its own goroutine and loops waiting for io event +// notifications from the epoll object. Once notifications arrive, they are +// dispatched to the registered queue. +func (n *notifier) waitAndNotify() error { + e := make([]unix.EpollEvent, 100) + for { + v, err := epollWait(n.epFD, e, -1) + if err == unix.EINTR { + continue + } + + if err != nil { + return err + } + + notified := false + n.mu.Lock() + for i := 0; i < v; i++ { + if fi, ok := n.fdMap[e[i].Fd]; ok { + fi.queue.Notify(waiter.EventMaskFromLinux(e[i].Events)) + notified = true + } + } + n.mu.Unlock() + if notified { + // Let goroutines woken by Notify get a chance to run before we + // epoll_wait again. + sync.Goyield() + } + } +} + +var shared struct { + notifier *notifier + once sync.Once + initErr error +} + +// AddFD adds an FD to the list of observed FDs. +func AddFD(fd int32, queue *waiter.Queue) error { + shared.once.Do(func() { + shared.notifier, shared.initErr = newNotifier() + }) + + if shared.initErr != nil { + return shared.initErr + } + + return shared.notifier.addFD(fd, queue) +} + +// UpdateFD updates the set of events the fd needs to be notified on. +func UpdateFD(fd int32) error { + return shared.notifier.updateFD(fd) +} + +// RemoveFD removes an FD from the list of observed FDs. +func RemoveFD(fd int32) { + shared.notifier.removeFD(fd) +} + +// HasFD returns true if the FD is in the list of observed FDs. +// +// This should only be used by tests to assert that FDs are correctly registered. +func HasFD(fd int32) bool { + return shared.notifier.hasFD(fd) +} diff --git a/pkg/fdnotifier/fdnotifier_state_autogen.go b/pkg/fdnotifier/fdnotifier_state_autogen.go new file mode 100644 index 0000000..70dfa86 --- /dev/null +++ b/pkg/fdnotifier/fdnotifier_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package fdnotifier diff --git a/pkg/fdnotifier/fdnotifier_unsafe_state_autogen.go b/pkg/fdnotifier/fdnotifier_unsafe_state_autogen.go new file mode 100644 index 0000000..70dfa86 --- /dev/null +++ b/pkg/fdnotifier/fdnotifier_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package fdnotifier diff --git a/pkg/fdnotifier/poll_unsafe.go b/pkg/fdnotifier/poll_unsafe.go new file mode 100644 index 0000000..29ad49f --- /dev/null +++ b/pkg/fdnotifier/poll_unsafe.go @@ -0,0 +1,82 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package fdnotifier + +import ( + "unsafe" + + "github.com/sagernet/gvisor/pkg/waiter" + "golang.org/x/sys/unix" +) + +// NonBlockingPoll polls the given FD in non-blocking fashion. It is used just +// to query the FD's current state. +func NonBlockingPoll(fd int32, mask waiter.EventMask) waiter.EventMask { + e := struct { + fd int32 + events int16 + revents int16 + }{ + fd: fd, + events: int16(mask.ToLinux()), + } + + ts := unix.Timespec{ + Sec: 0, + Nsec: 0, + } + + for { + n, _, err := unix.RawSyscall6(unix.SYS_PPOLL, uintptr(unsafe.Pointer(&e)), 1, + uintptr(unsafe.Pointer(&ts)), 0, 0, 0) + // Interrupted by signal, try again. + if err == unix.EINTR { + continue + } + // If an error occur we'll conservatively say the FD is ready for + // whatever is being checked. + if err != 0 { + return mask + } + + // If no FDs were returned, it wasn't ready for anything. + if n == 0 { + return 0 + } + + // Otherwise we got the ready events in the revents field. + return waiter.EventMaskFromLinux(uint32(e.revents)) + } +} + +// epollWait performs a blocking wait on epfd. +// +// Preconditions: len(events) > 0 +func epollWait(epfd int, events []unix.EpollEvent, msec int) (int, error) { + if len(events) == 0 { + panic("Empty events passed to EpollWait") + } + + // We actually use epoll_pwait with NULL sigmask instead of epoll_wait + // since that is what the Go >= 1.11 runtime prefers. + r, _, e := unix.Syscall6(unix.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) + if e != 0 { + return 0, e + } + return int(r), nil +} diff --git a/pkg/flipcall/ctrl_futex.go b/pkg/flipcall/ctrl_futex.go new file mode 100644 index 0000000..7efb92f --- /dev/null +++ b/pkg/flipcall/ctrl_futex.go @@ -0,0 +1,215 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !false +// +build !false + +package flipcall + +import ( + "encoding/json" + "fmt" + "math" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/log" +) + +type endpointControlImpl struct { + state atomicbitops.Int32 +} + +// Bits in endpointControlImpl.state. +const ( + epsBlocked = 1 << iota + epsShutdown +) + +func (ep *Endpoint) ctrlInit(opts ...EndpointOption) error { + if len(opts) != 0 { + return fmt.Errorf("unknown EndpointOption: %T", opts[0]) + } + return nil +} + +func (ep *Endpoint) ctrlConnect() error { + if err := ep.enterFutexWait(); err != nil { + return err + } + defer ep.exitFutexWait() + + // Write the connection request. + w := ep.NewWriter() + if err := json.NewEncoder(w).Encode(struct{}{}); err != nil { + return fmt.Errorf("error writing connection request: %v", err) + } + *ep.dataLen() = atomicbitops.FromUint32(w.Len()) + + // Exchange control with the server. + if err := ep.futexSetPeerActive(); err != nil { + return err + } + if err := ep.futexWakePeer(); err != nil { + return err + } + if err := ep.futexWaitUntilActive(); err != nil { + return err + } + + // Read the connection response. + var resp struct{} + respLen := ep.dataLen().Load() + if respLen > ep.dataCap { + return fmt.Errorf("invalid connection response length %d (maximum %d)", respLen, ep.dataCap) + } + if err := json.NewDecoder(ep.NewReader(respLen)).Decode(&resp); err != nil { + return fmt.Errorf("error reading connection response: %v", err) + } + + return nil +} + +func (ep *Endpoint) ctrlWaitFirst() error { + if err := ep.enterFutexWait(); err != nil { + return err + } + defer ep.exitFutexWait() + + // Wait for the connection request. + if err := ep.futexWaitUntilActive(); err != nil { + return err + } + + // Read the connection request. + reqLen := ep.dataLen().Load() + if reqLen > ep.dataCap { + return fmt.Errorf("invalid connection request length %d (maximum %d)", reqLen, ep.dataCap) + } + var req struct{} + if err := json.NewDecoder(ep.NewReader(reqLen)).Decode(&req); err != nil { + return fmt.Errorf("error reading connection request: %v", err) + } + + // Write the connection response. + w := ep.NewWriter() + if err := json.NewEncoder(w).Encode(struct{}{}); err != nil { + return fmt.Errorf("error writing connection response: %v", err) + } + *ep.dataLen() = atomicbitops.FromUint32(w.Len()) + + // Return control to the client. + raceBecomeInactive() + if err := ep.futexSetPeerActive(); err != nil { + return err + } + if err := ep.futexWakePeer(); err != nil { + return err + } + + // Wait for the first non-connection message. + return ep.futexWaitUntilActive() +} + +func (ep *Endpoint) ctrlRoundTrip(mayRetainP bool) error { + if err := ep.enterFutexWait(); err != nil { + return err + } + defer ep.exitFutexWait() + + if err := ep.futexSetPeerActive(); err != nil { + return err + } + if err := ep.futexWakePeer(); err != nil { + return err + } + // Since we don't know if the peer Endpoint is in the same process as this + // one (in which case it may need our P to run), we allow our P to be + // retaken regardless of mayRetainP. + return ep.futexWaitUntilActive() +} + +func (ep *Endpoint) ctrlWakeLast() error { + if err := ep.futexSetPeerActive(); err != nil { + return err + } + return ep.futexWakePeer() +} + +func (ep *Endpoint) enterFutexWait() error { + switch eps := ep.ctrl.state.Add(epsBlocked); eps { + case epsBlocked: + return nil + case epsBlocked | epsShutdown: + ep.ctrl.state.Add(-epsBlocked) + return ShutdownError{} + default: + // Most likely due to ep.enterFutexWait() being called concurrently + // from multiple goroutines. + panic(fmt.Sprintf("invalid flipcall.Endpoint.ctrl.state before flipcall.Endpoint.enterFutexWait(): %v", eps-epsBlocked)) + } +} + +func (ep *Endpoint) exitFutexWait() { + switch eps := ep.ctrl.state.Add(-epsBlocked); eps { + case 0: + return + case epsShutdown: + // ep.ctrlShutdown() was called while we were blocked, so we are + // responsible for indicating connection shutdown. + ep.shutdownConn() + default: + panic(fmt.Sprintf("invalid flipcall.Endpoint.ctrl.state after flipcall.Endpoint.exitFutexWait(): %v", eps+epsBlocked)) + } +} + +func (ep *Endpoint) ctrlShutdown() { + // Set epsShutdown to ensure that future calls to ep.enterFutexWait() fail. + if ep.ctrl.state.Add(epsShutdown)&epsBlocked != 0 { + // Wake the blocked thread. This must loop because it's possible that + // FUTEX_WAKE occurs after the waiter sets epsBlocked, but before it + // blocks in FUTEX_WAIT. + for { + // Wake MaxInt32 threads to prevent a broken or malicious peer from + // swallowing our wakeup by FUTEX_WAITing from multiple threads. + if err := ep.futexWakeConnState(math.MaxInt32); err != nil { + log.Warningf("failed to FUTEX_WAKE Endpoints: %v", err) + break + } + yieldThread() + if ep.ctrl.state.Load()&epsBlocked == 0 { + break + } + } + } else { + // There is no blocked thread, so we are responsible for indicating + // connection shutdown. + ep.shutdownConn() + } +} + +func (ep *Endpoint) shutdownConn() { + switch cs := ep.connState().Swap(csShutdown); cs { + case ep.activeState: + if err := ep.futexWakeConnState(1); err != nil { + log.Warningf("failed to FUTEX_WAKE peer Endpoint for shutdown: %v", err) + } + case ep.inactiveState: + // The peer is currently active and will detect shutdown when it tries + // to update the connection state. + case csShutdown: + // The peer also called Endpoint.Shutdown(). + default: + log.Warningf("unexpected connection state before Endpoint.shutdownConn(): %v", cs) + } +} diff --git a/pkg/flipcall/flipcall.go b/pkg/flipcall/flipcall.go new file mode 100644 index 0000000..05c57d1 --- /dev/null +++ b/pkg/flipcall/flipcall.go @@ -0,0 +1,282 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package flipcall implements a protocol providing Fast Local Interprocess +// Procedure Calls between mutually-distrusting processes. +package flipcall + +import ( + "fmt" + "math" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/memutil" + "golang.org/x/sys/unix" +) + +// An Endpoint provides the ability to synchronously transfer data and control +// to a connected peer Endpoint, which may be in another process. +// +// Since the Endpoint control transfer model is synchronous, at any given time +// one Endpoint "has control" (designated the active Endpoint), and the other +// is "waiting for control" (designated the inactive Endpoint). Users of the +// flipcall package designate one Endpoint as the client, which is initially +// active, and the other as the server, which is initially inactive. See +// flipcall_example_test.go for usage. +type Endpoint struct { + // packet is a pointer to the beginning of the packet window. (Since this + // is a raw OS memory mapping and not a Go object, it does not need to be + // represented as an unsafe.Pointer.) packet is immutable. + packet uintptr + + // dataCap is the size of the datagram part of the packet window in bytes. + // dataCap is immutable. + dataCap uint32 + + // activeState is csClientActive if this is a client Endpoint and + // csServerActive if this is a server Endpoint. + activeState uint32 + + // inactiveState is csServerActive if this is a client Endpoint and + // csClientActive if this is a server Endpoint. + inactiveState uint32 + + // shutdown is non-zero if Endpoint.Shutdown() has been called, or if the + // Endpoint has acknowledged shutdown initiated by the peer. + shutdown atomicbitops.Uint32 + + ctrl endpointControlImpl +} + +// EndpointSide indicates which side of a connection an Endpoint belongs to. +type EndpointSide int + +const ( + // ClientSide indicates that an Endpoint is a client (initially-active; + // first method call should be Connect). + ClientSide EndpointSide = iota + + // ServerSide indicates that an Endpoint is a server (initially-inactive; + // first method call should be RecvFirst.) + ServerSide +) + +// Init must be called on zero-value Endpoints before first use. If it +// succeeds, ep.Destroy() must be called once the Endpoint is no longer in use. +// +// pwd represents the packet window used to exchange data with the peer +// Endpoint. FD may differ between Endpoints if they are in different +// processes, but must represent the same file. The packet window must +// initially be filled with zero bytes. +func (ep *Endpoint) Init(side EndpointSide, pwd PacketWindowDescriptor, opts ...EndpointOption) error { + switch side { + case ClientSide: + ep.activeState = csClientActive + ep.inactiveState = csServerActive + case ServerSide: + ep.activeState = csServerActive + ep.inactiveState = csClientActive + default: + return fmt.Errorf("invalid EndpointSide: %v", side) + } + if pwd.Length < pageSize { + return fmt.Errorf("packet window size (%d) less than minimum (%d)", pwd.Length, pageSize) + } + if pwd.Length > math.MaxUint32 { + return fmt.Errorf("packet window size (%d) exceeds maximum (%d)", pwd.Length, math.MaxUint32) + } + m, err := memutil.MapFile(0, uintptr(pwd.Length), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED, uintptr(pwd.FD), uintptr(pwd.Offset)) + if err != nil { + return fmt.Errorf("failed to mmap packet window: %v", err) + } + ep.packet = m + ep.dataCap = uint32(pwd.Length) - uint32(PacketHeaderBytes) + if err := ep.ctrlInit(opts...); err != nil { + ep.unmapPacket() + return err + } + return nil +} + +// NewEndpoint is a convenience function that returns an initialized Endpoint +// allocated on the heap. +func NewEndpoint(side EndpointSide, pwd PacketWindowDescriptor, opts ...EndpointOption) (*Endpoint, error) { + var ep Endpoint + if err := ep.Init(side, pwd, opts...); err != nil { + return nil, err + } + return &ep, nil +} + +// An EndpointOption configures an Endpoint. +type EndpointOption interface { + isEndpointOption() +} + +// Destroy releases resources owned by ep. No other Endpoint methods may be +// called after Destroy. +func (ep *Endpoint) Destroy() { + ep.unmapPacket() +} + +func (ep *Endpoint) unmapPacket() { + unix.RawSyscall(unix.SYS_MUNMAP, ep.packet, uintptr(ep.dataCap)+PacketHeaderBytes, 0) + ep.packet = 0 +} + +// Shutdown causes concurrent and future calls to ep.Connect(), ep.SendRecv(), +// ep.RecvFirst(), and ep.SendLast(), as well as the same calls in the peer +// Endpoint, to unblock and return ShutdownErrors. It does not wait for +// concurrent calls to return. Successive calls to Shutdown have no effect. +// +// Shutdown is the only Endpoint method that may be called concurrently with +// other methods on the same Endpoint. +func (ep *Endpoint) Shutdown() { + if ep.shutdown.Swap(1) != 0 { + // ep.Shutdown() has previously been called. + return + } + ep.ctrlShutdown() +} + +// isShutdownLocally returns true if ep.Shutdown() has been called. +func (ep *Endpoint) isShutdownLocally() bool { + return ep.shutdown.Load() != 0 +} + +// ShutdownError is returned by most Endpoint methods after Endpoint.Shutdown() +// has been called. +type ShutdownError struct{} + +// Error implements error.Error. +func (ShutdownError) Error() string { + return "flipcall connection shutdown" +} + +// DataCap returns the maximum datagram size supported by ep. Equivalently, +// DataCap returns len(ep.Data()). +func (ep *Endpoint) DataCap() uint32 { + return ep.dataCap +} + +// Connection state. +const ( + // The client is, by definition, initially active, so this must be 0. + csClientActive = 0 + csServerActive = 1 + csShutdown = 2 +) + +// Connect blocks until the peer Endpoint has called Endpoint.RecvFirst(). +// +// Preconditions: +// - ep is a client Endpoint. +// - ep.Connect(), ep.RecvFirst(), ep.SendRecv(), and ep.SendLast() have never +// been called. +func (ep *Endpoint) Connect() error { + err := ep.ctrlConnect() + if err == nil { + raceBecomeActive() + } + return err +} + +// RecvFirst blocks until the peer Endpoint calls Endpoint.SendRecv(), then +// returns the datagram length specified by that call. +// +// Preconditions: +// - ep is a server Endpoint. +// - ep.SendRecv(), ep.RecvFirst(), and ep.SendLast() have never been called. +func (ep *Endpoint) RecvFirst() (uint32, error) { + if err := ep.ctrlWaitFirst(); err != nil { + return 0, err + } + raceBecomeActive() + recvDataLen := ep.dataLen().Load() + if recvDataLen > ep.dataCap { + return 0, fmt.Errorf("received packet with invalid datagram length %d (maximum %d)", recvDataLen, ep.dataCap) + } + return recvDataLen, nil +} + +// SendRecv transfers control to the peer Endpoint, causing its call to +// Endpoint.SendRecv() or Endpoint.RecvFirst() to return with the given +// datagram length, then blocks until the peer Endpoint calls +// Endpoint.SendRecv() or Endpoint.SendLast(). +// +// Preconditions: +// - dataLen <= ep.DataCap(). +// - No previous call to ep.SendRecv() or ep.RecvFirst() has returned an error. +// - ep.SendLast() has never been called. +// - If ep is a client Endpoint, ep.Connect() has previously been called and +// returned nil. +func (ep *Endpoint) SendRecv(dataLen uint32) (uint32, error) { + return ep.sendRecv(dataLen, false /* mayRetainP */) +} + +// SendRecvFast is equivalent to SendRecv, but may prevent the caller's runtime +// P from being released, in which case the calling goroutine continues to +// count against GOMAXPROCS while waiting for the peer Endpoint to return +// control to the caller. +// +// SendRecvFast is appropriate if the peer Endpoint is expected to consistently +// return control in a short amount of time (less than ~10ms). +// +// Preconditions: As for SendRecv. +func (ep *Endpoint) SendRecvFast(dataLen uint32) (uint32, error) { + return ep.sendRecv(dataLen, true /* mayRetainP */) +} + +func (ep *Endpoint) sendRecv(dataLen uint32, mayRetainP bool) (uint32, error) { + if dataLen > ep.dataCap { + panic(fmt.Sprintf("attempting to send packet with datagram length %d (maximum %d)", dataLen, ep.dataCap)) + } + // This store can safely be non-atomic: Under correct operation we should + // be the only thread writing ep.dataLen(), and ep.ctrlRoundTrip() will + // synchronize with the receiver. We will not read from ep.dataLen() until + // after ep.ctrlRoundTrip(), so if the peer is mutating it concurrently then + // they can only shoot themselves in the foot. + ep.dataLen().RacyStore(dataLen) + raceBecomeInactive() + if err := ep.ctrlRoundTrip(mayRetainP); err != nil { + return 0, err + } + raceBecomeActive() + recvDataLen := ep.dataLen().Load() + if recvDataLen > ep.dataCap { + return 0, fmt.Errorf("received packet with invalid datagram length %d (maximum %d)", recvDataLen, ep.dataCap) + } + return recvDataLen, nil +} + +// SendLast causes the peer Endpoint's call to Endpoint.SendRecv() or +// Endpoint.RecvFirst() to return with the given datagram length. +// +// Preconditions: +// - dataLen <= ep.DataCap(). +// - No previous call to ep.SendRecv() or ep.RecvFirst() has returned an error. +// - ep.SendLast() has never been called. +// - If ep is a client Endpoint, ep.Connect() has previously been called and +// returned nil. +func (ep *Endpoint) SendLast(dataLen uint32) error { + if dataLen > ep.dataCap { + panic(fmt.Sprintf("attempting to send packet with datagram length %d (maximum %d)", dataLen, ep.dataCap)) + } + ep.dataLen().RacyStore(dataLen) + raceBecomeInactive() + if err := ep.ctrlWakeLast(); err != nil { + return err + } + return nil +} diff --git a/pkg/flipcall/flipcall_linux_state_autogen.go b/pkg/flipcall/flipcall_linux_state_autogen.go new file mode 100644 index 0000000..7c9d0e9 --- /dev/null +++ b/pkg/flipcall/flipcall_linux_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package flipcall diff --git a/pkg/flipcall/flipcall_state_autogen.go b/pkg/flipcall/flipcall_state_autogen.go new file mode 100644 index 0000000..9be2aa3 --- /dev/null +++ b/pkg/flipcall/flipcall_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build !false +// +build !false + +package flipcall diff --git a/pkg/flipcall/flipcall_unsafe.go b/pkg/flipcall/flipcall_unsafe.go new file mode 100644 index 0000000..c2928c6 --- /dev/null +++ b/pkg/flipcall/flipcall_unsafe.go @@ -0,0 +1,87 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flipcall + +import ( + "reflect" + "unsafe" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/sync" +) + +// Packets consist of a 16-byte header followed by an arbitrarily-sized +// datagram. The header consists of: +// +// - A 4-byte native-endian connection state. +// +// - A 4-byte native-endian datagram length in bytes. +// +// - 8 reserved bytes. +const ( + // PacketHeaderBytes is the size of a flipcall packet header in bytes. The + // maximum datagram size supported by a flipcall connection is equal to the + // length of the packet window minus PacketHeaderBytes. + // + // PacketHeaderBytes is exported to support its use in constant + // expressions. Non-constant expressions may prefer to use + // PacketWindowLengthForDataCap(). + PacketHeaderBytes = 16 +) + +func (ep *Endpoint) connState() *atomicbitops.Uint32 { + return (*atomicbitops.Uint32)(unsafe.Pointer(ep.packet)) +} + +func (ep *Endpoint) dataLen() *atomicbitops.Uint32 { + return (*atomicbitops.Uint32)(unsafe.Pointer(ep.packet + 4)) +} + +// Data returns the datagram part of ep's packet window as a byte slice. +// +// Note that the packet window is shared with the potentially-untrusted peer +// Endpoint, which may concurrently mutate the contents of the packet window. +// Thus: +// +// - Readers must not assume that two reads of the same byte in Data() will +// return the same result. In other words, readers should read any given byte +// in Data() at most once. +// +// - Writers must not assume that they will read back the same data that they +// have written. In other words, writers should avoid reading from Data() at +// all. +func (ep *Endpoint) Data() (bs []byte) { + bshdr := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) + bshdr.Data = ep.packet + PacketHeaderBytes + bshdr.Len = int(ep.dataCap) + bshdr.Cap = int(ep.dataCap) + return +} + +// ioSync is a dummy variable used to indicate synchronization to the Go race +// detector. Compare syscall.ioSync. +var ioSync int64 + +func raceBecomeActive() { + if sync.RaceEnabled { + sync.RaceAcquire(unsafe.Pointer(&ioSync)) + } +} + +func raceBecomeInactive() { + if sync.RaceEnabled { + sync.RaceReleaseMerge(unsafe.Pointer(&ioSync)) + } +} diff --git a/pkg/flipcall/flipcall_unsafe_state_autogen.go b/pkg/flipcall/flipcall_unsafe_state_autogen.go new file mode 100644 index 0000000..0e03c2a --- /dev/null +++ b/pkg/flipcall/flipcall_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package flipcall diff --git a/pkg/flipcall/futex_linux.go b/pkg/flipcall/futex_linux.go new file mode 100644 index 0000000..68a0572 --- /dev/null +++ b/pkg/flipcall/futex_linux.go @@ -0,0 +1,88 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package flipcall + +import ( + "fmt" + "runtime" + + "github.com/sagernet/gvisor/pkg/abi/linux" + "golang.org/x/sys/unix" +) + +func (ep *Endpoint) futexSetPeerActive() error { + if ep.connState().CompareAndSwap(ep.activeState, ep.inactiveState) { + return nil + } + switch cs := ep.connState().Load(); cs { + case csShutdown: + return ShutdownError{} + default: + return fmt.Errorf("unexpected connection state before FUTEX_WAKE: %v", cs) + } +} + +func (ep *Endpoint) futexWakePeer() error { + if err := ep.futexWakeConnState(1); err != nil { + return fmt.Errorf("failed to FUTEX_WAKE peer Endpoint: %v", err) + } + return nil +} + +func (ep *Endpoint) futexWaitUntilActive() error { + for { + switch cs := ep.connState().Load(); cs { + case ep.activeState: + return nil + case ep.inactiveState: + if ep.isShutdownLocally() { + return ShutdownError{} + } + if err := ep.futexWaitConnState(ep.inactiveState); err != nil { + return fmt.Errorf("failed to FUTEX_WAIT for peer Endpoint: %v", err) + } + continue + case csShutdown: + return ShutdownError{} + default: + return fmt.Errorf("unexpected connection state before FUTEX_WAIT: %v", cs) + } + } +} + +func (ep *Endpoint) futexWakeConnState(numThreads int32) error { + if _, _, e := unix.RawSyscall(unix.SYS_FUTEX, ep.packet, linux.FUTEX_WAKE, uintptr(numThreads)); e != 0 { + return e + } + return nil +} + +func (ep *Endpoint) futexWaitConnState(curState uint32) error { + _, _, e := unix.Syscall6(unix.SYS_FUTEX, ep.packet, linux.FUTEX_WAIT, uintptr(curState), 0, 0, 0) + if e != 0 && e != unix.EAGAIN && e != unix.EINTR { + return e + } + return nil +} + +func yieldThread() { + unix.Syscall(unix.SYS_SCHED_YIELD, 0, 0, 0) + // The thread we're trying to yield to may be waiting for a Go runtime P. + // runtime.Gosched() will hand off ours if necessary. + runtime.Gosched() +} diff --git a/pkg/flipcall/io.go b/pkg/flipcall/io.go new file mode 100644 index 0000000..85e40b9 --- /dev/null +++ b/pkg/flipcall/io.go @@ -0,0 +1,113 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flipcall + +import ( + "fmt" + "io" +) + +// DatagramReader implements io.Reader by reading a datagram from an Endpoint's +// packet window. Its use is optional; users that can use Endpoint.Data() more +// efficiently are advised to do so. +type DatagramReader struct { + ep *Endpoint + off uint32 + end uint32 +} + +// Init must be called on zero-value DatagramReaders before first use. +// +// Preconditions: dataLen is 0, or was returned by a previous call to +// ep.RecvFirst() or ep.SendRecv(). +func (r *DatagramReader) Init(ep *Endpoint, dataLen uint32) { + r.ep = ep + r.Reset(dataLen) +} + +// Reset causes r to begin reading a new datagram of the given length from the +// associated Endpoint. +// +// Preconditions: dataLen is 0, or was returned by a previous call to the +// associated Endpoint's RecvFirst() or SendRecv() methods. +func (r *DatagramReader) Reset(dataLen uint32) { + if dataLen > r.ep.dataCap { + panic(fmt.Sprintf("invalid dataLen (%d) > ep.dataCap (%d)", dataLen, r.ep.dataCap)) + } + r.off = 0 + r.end = dataLen +} + +// NewReader is a convenience function that returns an initialized +// DatagramReader allocated on the heap. +// +// Preconditions: dataLen was returned by a previous call to ep.RecvFirst() or +// ep.SendRecv(). +func (ep *Endpoint) NewReader(dataLen uint32) *DatagramReader { + r := &DatagramReader{} + r.Init(ep, dataLen) + return r +} + +// Read implements io.Reader.Read. +func (r *DatagramReader) Read(dst []byte) (int, error) { + n := copy(dst, r.ep.Data()[r.off:r.end]) + r.off += uint32(n) + if r.off == r.end { + return n, io.EOF + } + return n, nil +} + +// DatagramWriter implements io.Writer by writing a datagram to an Endpoint's +// packet window. Its use is optional; users that can use Endpoint.Data() more +// efficiently are advised to do so. +type DatagramWriter struct { + ep *Endpoint + off uint32 +} + +// Init must be called on zero-value DatagramWriters before first use. +func (w *DatagramWriter) Init(ep *Endpoint) { + w.ep = ep +} + +// Reset causes w to begin writing a new datagram to the associated Endpoint. +func (w *DatagramWriter) Reset() { + w.off = 0 +} + +// NewWriter is a convenience function that returns an initialized +// DatagramWriter allocated on the heap. +func (ep *Endpoint) NewWriter() *DatagramWriter { + w := &DatagramWriter{} + w.Init(ep) + return w +} + +// Write implements io.Writer.Write. +func (w *DatagramWriter) Write(src []byte) (int, error) { + n := copy(w.ep.Data()[w.off:w.ep.dataCap], src) + w.off += uint32(n) + if n != len(src) { + return n, fmt.Errorf("datagram would exceed maximum size of %d bytes", w.ep.dataCap) + } + return n, nil +} + +// Len returns the length of the written datagram. +func (w *DatagramWriter) Len() uint32 { + return w.off +} diff --git a/pkg/flipcall/packet_window.go b/pkg/flipcall/packet_window.go new file mode 100644 index 0000000..e75317f --- /dev/null +++ b/pkg/flipcall/packet_window.go @@ -0,0 +1,166 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flipcall + +import ( + "fmt" + "math/bits" + "os" + + "github.com/sagernet/gvisor/pkg/abi/linux" + "github.com/sagernet/gvisor/pkg/memutil" + "golang.org/x/sys/unix" +) + +var ( + pageSize = os.Getpagesize() + pageMask = pageSize - 1 +) + +func init() { + if bits.OnesCount(uint(pageSize)) != 1 { + // This is depended on by roundUpToPage(). + panic(fmt.Sprintf("system page size (%d) is not a power of 2", pageSize)) + } + if uintptr(pageSize) < PacketHeaderBytes { + // This is required since Endpoint.Init() imposes a minimum packet + // window size of 1 page. + panic(fmt.Sprintf("system page size (%d) is less than packet header size (%d)", pageSize, PacketHeaderBytes)) + } +} + +// PacketWindowDescriptor represents a packet window, a range of pages in a +// shared memory file that is used to exchange packets between partner +// Endpoints. +type PacketWindowDescriptor struct { + // FD is the file descriptor representing the shared memory file. + FD int + + // Offset is the offset into the shared memory file at which the packet + // window begins. + Offset int64 + + // Length is the size of the packet window in bytes. + Length int +} + +// PacketWindowLengthForDataCap returns the minimum packet window size required +// to accommodate datagrams of the given size in bytes. +func PacketWindowLengthForDataCap(dataCap uint32) int { + return roundUpToPage(int(dataCap) + int(PacketHeaderBytes)) +} + +func roundUpToPage(x int) int { + return (x + pageMask) &^ pageMask +} + +// A PacketWindowAllocator owns a shared memory file, and allocates packet +// windows from it. +type PacketWindowAllocator struct { + fd int + nextAlloc int64 + fileSize int64 +} + +// Init must be called on zero-value PacketWindowAllocators before first use. +// If it succeeds, Destroy() must be called once the PacketWindowAllocator is +// no longer in use. +func (pwa *PacketWindowAllocator) Init() error { + fd, err := memutil.CreateMemFD("flipcall_packet_windows", linux.MFD_CLOEXEC|linux.MFD_ALLOW_SEALING) + if err != nil { + return fmt.Errorf("failed to create memfd: %v", err) + } + // Apply F_SEAL_SHRINK to prevent either party from causing SIGBUS in the + // other by truncating the file, and F_SEAL_SEAL to prevent either party + // from applying F_SEAL_GROW or F_SEAL_WRITE. + if _, _, e := unix.RawSyscall(unix.SYS_FCNTL, uintptr(fd), linux.F_ADD_SEALS, linux.F_SEAL_SHRINK|linux.F_SEAL_SEAL); e != 0 { + unix.Close(fd) + return fmt.Errorf("failed to apply memfd seals: %v", e) + } + pwa.fd = fd + return nil +} + +// NewPacketWindowAllocator is a convenience function that returns an +// initialized PacketWindowAllocator allocated on the heap. +func NewPacketWindowAllocator() (*PacketWindowAllocator, error) { + var pwa PacketWindowAllocator + if err := pwa.Init(); err != nil { + return nil, err + } + return &pwa, nil +} + +// Destroy releases resources owned by pwa. This invalidates file descriptors +// previously returned by pwa.FD() and pwd.Allocate(). +func (pwa *PacketWindowAllocator) Destroy() { + unix.Close(pwa.fd) +} + +// FD represents the file descriptor of the shared memory file backing pwa. +func (pwa *PacketWindowAllocator) FD() int { + return pwa.fd +} + +// Allocate allocates a new packet window of at least the given size and +// returns a PacketWindowDescriptor representing it. +// +// Preconditions: size > 0. +func (pwa *PacketWindowAllocator) Allocate(size int) (PacketWindowDescriptor, error) { + if size <= 0 { + return PacketWindowDescriptor{}, fmt.Errorf("invalid size: %d", size) + } + // Page-align size to ensure that pwa.nextAlloc remains page-aligned. + size = roundUpToPage(size) + if size <= 0 { + return PacketWindowDescriptor{}, fmt.Errorf("size %d overflows after rounding up to page size", size) + } + end := pwa.nextAlloc + int64(size) // overflow checked by ensureFileSize + if err := pwa.ensureFileSize(end); err != nil { + return PacketWindowDescriptor{}, err + } + start := pwa.nextAlloc + pwa.nextAlloc = end + return PacketWindowDescriptor{ + FD: pwa.FD(), + Offset: start, + Length: size, + }, nil +} + +func (pwa *PacketWindowAllocator) ensureFileSize(min int64) error { + if min <= 0 { + return fmt.Errorf("file size would overflow") + } + if pwa.fileSize >= min { + return nil + } + newSize := 2 * pwa.fileSize + if newSize == 0 { + newSize = int64(pageSize) + } + for newSize < min { + newNewSize := newSize * 2 + if newNewSize <= 0 { + return fmt.Errorf("file size would overflow") + } + newSize = newNewSize + } + if err := unix.Ftruncate(pwa.FD(), newSize); err != nil { + return fmt.Errorf("ftruncate failed: %v", err) + } + pwa.fileSize = newSize + return nil +} diff --git a/pkg/fspath/builder.go b/pkg/fspath/builder.go new file mode 100644 index 0000000..9f6abd4 --- /dev/null +++ b/pkg/fspath/builder.go @@ -0,0 +1,112 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fspath + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/gohacks" +) + +// Builder is similar to strings.Builder, but is used to produce pathnames +// given path components in reverse order (from leaf to root). This is useful +// in the common case where a filesystem is represented by a tree of named +// nodes, and the path to a given node must be produced by walking upward from +// that node to a given root. +type Builder struct { + buf []byte + start int + needSep bool +} + +// Reset resets the Builder to be empty. +func (b *Builder) Reset() { + b.start = len(b.buf) + b.needSep = false +} + +// Len returns the number of accumulated bytes. +func (b *Builder) Len() int { + return len(b.buf) - b.start +} + +func (b *Builder) needToGrow(n int) bool { + return b.start < n +} + +func (b *Builder) grow(n int) { + newLen := b.Len() + n + var newCap int + if len(b.buf) == 0 { + newCap = 64 // arbitrary + } else { + newCap = 2 * len(b.buf) + } + for newCap < newLen { + newCap *= 2 + if newCap == 0 { + panic(fmt.Sprintf("required length (%d) causes buffer size to overflow", newLen)) + } + } + newBuf := make([]byte, newCap) + copy(newBuf[newCap-b.Len():], b.buf[b.start:]) + b.start += newCap - len(b.buf) + b.buf = newBuf +} + +// PrependComponent prepends the given path component to b's buffer. A path +// separator is automatically inserted if appropriate. +func (b *Builder) PrependComponent(pc string) { + if b.needSep { + b.PrependByte('/') + } + b.PrependString(pc) + b.needSep = true +} + +// PrependString prepends the given string to b's buffer. +func (b *Builder) PrependString(str string) { + if b.needToGrow(len(str)) { + b.grow(len(str)) + } + b.start -= len(str) + copy(b.buf[b.start:], str) +} + +// PrependByte prepends the given byte to b's buffer. +func (b *Builder) PrependByte(c byte) { + if b.needToGrow(1) { + b.grow(1) + } + b.start-- + b.buf[b.start] = c +} + +// AppendString appends the given string to b's buffer. +func (b *Builder) AppendString(str string) { + if b.needToGrow(len(str)) { + b.grow(len(str)) + } + oldStart := b.start + b.start -= len(str) + copy(b.buf[b.start:], b.buf[oldStart:]) + copy(b.buf[len(b.buf)-len(str):], str) +} + +// String returns the accumulated string. No other methods should be called +// after String. +func (b *Builder) String() string { + return gohacks.StringFromImmutableBytes(b.buf[b.start:]) +} diff --git a/pkg/fspath/fspath.go b/pkg/fspath/fspath.go new file mode 100644 index 0000000..4c983d5 --- /dev/null +++ b/pkg/fspath/fspath.go @@ -0,0 +1,187 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fspath provides efficient tools for working with file paths in +// Linux-compatible filesystem implementations. +package fspath + +import ( + "strings" +) + +const pathSep = '/' + +// Parse parses a pathname as described by path_resolution(7), except that +// empty pathnames will be parsed successfully to a Path for which +// Path.Absolute == Path.Dir == Path.HasComponents() == false. (This is +// necessary to support AT_EMPTY_PATH.) +func Parse(pathname string) Path { + if len(pathname) == 0 { + return Path{} + } + // Skip leading path separators. + i := 0 + for pathname[i] == pathSep { + i++ + if i == len(pathname) { + // pathname consists entirely of path separators. + return Path{ + Absolute: true, + Dir: true, + } + } + } + // Skip trailing path separators. This is required by Iterator.Next. This + // loop is guaranteed to terminate with j >= 0 because otherwise the + // pathname would consist entirely of path separators, so we would have + // returned above. + j := len(pathname) - 1 + for pathname[j] == pathSep { + j-- + } + // Find the end of the first path component. + firstEnd := i + 1 + for firstEnd != len(pathname) && pathname[firstEnd] != pathSep { + firstEnd++ + } + return Path{ + Begin: Iterator{ + partialPathname: pathname[i : j+1], + end: firstEnd - i, + }, + Absolute: i != 0, + Dir: j != len(pathname)-1, + } +} + +// Path contains the information contained in a pathname string. +// +// Path is copyable by value. The zero value for Path is equivalent to +// fspath.Parse(""), i.e. the empty path. +type Path struct { + // Begin is an iterator to the first path component in the relative part of + // the path. + // + // Path doesn't store information about path components after the first + // since this would require allocation. + Begin Iterator + + // If true, the path is absolute, such that lookup should begin at the + // filesystem root. If false, the path is relative, such that where lookup + // begins is unspecified. + Absolute bool + + // If true, the pathname contains trailing path separators, so the last + // path component must exist and resolve to a directory. + Dir bool +} + +// String returns a pathname string equivalent to p. Note that the returned +// string is not necessarily equal to the string p was parsed from; in +// particular, redundant path separators will not be present. +func (p Path) String() string { + var b strings.Builder + if p.Absolute { + b.WriteByte(pathSep) + } + sep := false + for pit := p.Begin; pit.Ok(); pit = pit.Next() { + if sep { + b.WriteByte(pathSep) + } + b.WriteString(pit.String()) + sep = true + } + // Don't return "//" for Parse("/"). + if p.Dir && p.Begin.Ok() { + b.WriteByte(pathSep) + } + return b.String() +} + +// HasComponents returns true if p contains a non-zero number of path +// components. +func (p Path) HasComponents() bool { + return p.Begin.Ok() +} + +// An Iterator represents either a path component in a Path or a terminal +// iterator indicating that the end of the path has been reached. +// +// Iterator is immutable and copyable by value. The zero value of Iterator is +// valid, and represents a terminal iterator. +type Iterator struct { + // partialPathname is a substring of the original pathname beginning at the + // start of the represented path component and ending immediately after the + // end of the last path component in the pathname. If partialPathname is + // empty, the PathnameIterator is terminal. + // + // See TestParseIteratorPartialPathnames in fspath_test.go for a worked + // example. + partialPathname string + + // end is the offset into partialPathname of the first byte after the end + // of the represented path component. + end int +} + +// Ok returns true if it is not terminal. +func (it Iterator) Ok() bool { + return len(it.partialPathname) != 0 +} + +// String returns the path component represented by it. +// +// Preconditions: it.Ok(). +func (it Iterator) String() string { + return it.partialPathname[:it.end] +} + +// Next returns an iterator to the path component after it. If it is the last +// component in the path, Next returns a terminal iterator. +// +// Preconditions: it.Ok(). +func (it Iterator) Next() Iterator { + if it.end == len(it.partialPathname) { + // End of the path. + return Iterator{} + } + // Skip path separators. Since Parse trims trailing path separators, if we + // aren't at the end of the path, there is definitely another path + // component. + i := it.end + 1 + for { + if it.partialPathname[i] != pathSep { + break + } + i++ + } + nextPartialPathname := it.partialPathname[i:] + // Find the end of this path component. + nextEnd := 1 + for nextEnd < len(nextPartialPathname) && nextPartialPathname[nextEnd] != pathSep { + nextEnd++ + } + return Iterator{ + partialPathname: nextPartialPathname, + end: nextEnd, + } +} + +// NextOk is equivalent to it.Next().Ok(), but is faster. +// +// Preconditions: it.Ok(). +func (it Iterator) NextOk() bool { + return it.end != len(it.partialPathname) +} diff --git a/pkg/fspath/fspath_state_autogen.go b/pkg/fspath/fspath_state_autogen.go new file mode 100644 index 0000000..6ceea80 --- /dev/null +++ b/pkg/fspath/fspath_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package fspath diff --git a/pkg/fsutil/fsutil.go b/pkg/fsutil/fsutil.go new file mode 100644 index 0000000..e331b9e --- /dev/null +++ b/pkg/fsutil/fsutil.go @@ -0,0 +1,48 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fsutil contains filesystem utilities that can be shared between the +// sentry and other sandbox components. +package fsutil + +import "golang.org/x/sys/unix" + +// DirentHandler is a function that handles a dirent. +type DirentHandler func(ino uint64, off int64, ftype uint8, name string, reclen uint16) + +// ForEachDirent retrieves all dirents from dirfd using getdents64(2) and +// invokes handleDirent on them. +func ForEachDirent(dirfd int, handleDirent DirentHandler) error { + var direntsBuf [8192]byte + for { + n, err := unix.Getdents(dirfd, direntsBuf[:]) + if err != nil { + return err + } + if n <= 0 { + return nil + } + ParseDirents(direntsBuf[:n], handleDirent) + } +} + +// DirentNames retrieves all dirents from dirfd using getdents64(2) and returns +// all the recorded dirent names. +func DirentNames(dirfd int) ([]string, error) { + var names []string + err := ForEachDirent(dirfd, func(_ uint64, _ int64, _ uint8, name string, _ uint16) { + names = append(names, name) + }) + return names, err +} diff --git a/pkg/fsutil/fsutil_amd64_unsafe.go b/pkg/fsutil/fsutil_amd64_unsafe.go new file mode 100644 index 0000000..ac222e1 --- /dev/null +++ b/pkg/fsutil/fsutil_amd64_unsafe.go @@ -0,0 +1,49 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +package fsutil + +import ( + "unsafe" + + "github.com/sagernet/gvisor/pkg/syserr" + "golang.org/x/sys/unix" +) + +// StatAt is a convenience wrapper around newfstatat(2). +func StatAt(dirFd int, name string) (unix.Stat_t, error) { + nameBytes, err := unix.BytePtrFromString(name) + if err != nil { + return unix.Stat_t{}, err + } + namePtr := unsafe.Pointer(nameBytes) + + var stat unix.Stat_t + statPtr := unsafe.Pointer(&stat) + + if _, _, errno := unix.Syscall6( + unix.SYS_NEWFSTATAT, + uintptr(dirFd), + uintptr(namePtr), + uintptr(statPtr), + unix.AT_SYMLINK_NOFOLLOW, + 0, + 0); errno != 0 { + return unix.Stat_t{}, syserr.FromHost(errno).ToError() + } + return stat, nil +} diff --git a/pkg/fsutil/fsutil_amd64_unsafe_state_autogen.go b/pkg/fsutil/fsutil_amd64_unsafe_state_autogen.go new file mode 100644 index 0000000..d660bd8 --- /dev/null +++ b/pkg/fsutil/fsutil_amd64_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build amd64 +// +build amd64 + +package fsutil diff --git a/pkg/fsutil/fsutil_arm64_unsafe.go b/pkg/fsutil/fsutil_arm64_unsafe.go new file mode 100644 index 0000000..ec0034f --- /dev/null +++ b/pkg/fsutil/fsutil_arm64_unsafe.go @@ -0,0 +1,49 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package fsutil + +import ( + "unsafe" + + "github.com/sagernet/gvisor/pkg/syserr" + "golang.org/x/sys/unix" +) + +// StatAt is a convenience wrapper around fstatat(2). +func StatAt(dirFd int, name string) (unix.Stat_t, error) { + nameBytes, err := unix.BytePtrFromString(name) + if err != nil { + return unix.Stat_t{}, err + } + namePtr := unsafe.Pointer(nameBytes) + + var stat unix.Stat_t + statPtr := unsafe.Pointer(&stat) + + if _, _, errno := unix.Syscall6( + unix.SYS_FSTATAT, + uintptr(dirFd), + uintptr(namePtr), + uintptr(statPtr), + unix.AT_SYMLINK_NOFOLLOW, + 0, + 0); errno != 0 { + return unix.Stat_t{}, syserr.FromHost(errno).ToError() + } + return stat, nil +} diff --git a/pkg/fsutil/fsutil_arm64_unsafe_state_autogen.go b/pkg/fsutil/fsutil_arm64_unsafe_state_autogen.go new file mode 100644 index 0000000..78a2ac7 --- /dev/null +++ b/pkg/fsutil/fsutil_arm64_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build arm64 +// +build arm64 + +package fsutil diff --git a/pkg/fsutil/fsutil_state_autogen.go b/pkg/fsutil/fsutil_state_autogen.go new file mode 100644 index 0000000..00b0994 --- /dev/null +++ b/pkg/fsutil/fsutil_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package fsutil diff --git a/pkg/fsutil/fsutil_unsafe.go b/pkg/fsutil/fsutil_unsafe.go new file mode 100644 index 0000000..bc898fa --- /dev/null +++ b/pkg/fsutil/fsutil_unsafe.go @@ -0,0 +1,121 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsutil + +import ( + "unsafe" + + "github.com/sagernet/gvisor/pkg/syserr" + "golang.org/x/sys/unix" +) + +// UnixDirentMaxSize is the maximum size of unix.Dirent in bytes. +var UnixDirentMaxSize = int(unsafe.Sizeof(unix.Dirent{})) + +// Utimensat is a convenience wrapper to make the utimensat(2) syscall. It +// additionally handles empty name. +func Utimensat(dirFd int, name string, times [2]unix.Timespec, flags int) error { + // utimensat(2) doesn't accept empty name, instead name must be nil to make it + // operate directly on 'dirFd' unlike other *at syscalls. + var namePtr unsafe.Pointer + if name != "" { + nameBytes, err := unix.BytePtrFromString(name) + if err != nil { + return err + } + namePtr = unsafe.Pointer(nameBytes) + } + + timesPtr := unsafe.Pointer(×[0]) + + if _, _, errno := unix.Syscall6( + unix.SYS_UTIMENSAT, + uintptr(dirFd), + uintptr(namePtr), + uintptr(timesPtr), + uintptr(flags), + 0, + 0); errno != 0 { + return syserr.FromHost(errno).ToError() + } + return nil +} + +// RenameAt is a convenience wrapper to make the renameat(2) syscall. It +// additionally handles empty names. +func RenameAt(oldDirFD int, oldName string, newDirFD int, newName string) error { + var oldNamePtr unsafe.Pointer + if oldName != "" { + nameBytes, err := unix.BytePtrFromString(oldName) + if err != nil { + return err + } + oldNamePtr = unsafe.Pointer(nameBytes) + } + var newNamePtr unsafe.Pointer + if newName != "" { + nameBytes, err := unix.BytePtrFromString(newName) + if err != nil { + return err + } + newNamePtr = unsafe.Pointer(nameBytes) + } + + if _, _, errno := unix.Syscall6( + unix.SYS_RENAMEAT, + uintptr(oldDirFD), + uintptr(oldNamePtr), + uintptr(newDirFD), + uintptr(newNamePtr), + 0, + 0); errno != 0 { + return syserr.FromHost(errno).ToError() + } + return nil +} + +// ParseDirents parses dirents from buf. buf must have been populated by +// getdents64(2) syscall. It calls the handleDirent callback for each dirent. +func ParseDirents(buf []byte, handleDirent DirentHandler) { + for len(buf) > 0 { + // Interpret the buf populated by unix.Getdents as unix.Dirent. + dirent := *(*unix.Dirent)(unsafe.Pointer(&buf[0])) + + // Advance buf for the next dirent. + buf = buf[dirent.Reclen:] + + // Extracting the name is pretty tedious... + var nameBuf [unix.NAME_MAX]byte + var nameLen int + for i := 0; i < len(dirent.Name); i++ { + // The name is null terminated. + if dirent.Name[i] == 0 { + nameLen = i + break + } + nameBuf[i] = byte(dirent.Name[i]) + } + name := string(nameBuf[:nameLen]) + + // Skip `.` and `..` entries. It is anyways ignored by the client. We also + // don't want to leak information about `..`. + if name == "." || name == ".." { + continue + } + + // Deliver results to caller. + handleDirent(dirent.Ino, dirent.Off, dirent.Type, name, dirent.Reclen) + } +} diff --git a/pkg/fsutil/fsutil_unsafe_state_autogen.go b/pkg/fsutil/fsutil_unsafe_state_autogen.go new file mode 100644 index 0000000..00b0994 --- /dev/null +++ b/pkg/fsutil/fsutil_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package fsutil diff --git a/pkg/gohacks/linkname_go113_unsafe.go b/pkg/gohacks/linkname_go113_unsafe.go new file mode 100644 index 0000000..2e8c465 --- /dev/null +++ b/pkg/gohacks/linkname_go113_unsafe.go @@ -0,0 +1,51 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.13 + +// //go:linkname directives type-checked by checklinkname. Any other +// non-linkname assumptions outside the Go 1 compatibility guarantee should +// have an accompanied vet check or version guard build tag. + +// Package gohacks contains utilities for subverting the Go compiler. +package gohacks + +import ( + "unsafe" +) + +// Note that go:linkname silently doesn't work if the local name is exported, +// necessitating an indirection for exported functions. + +// Memmove is runtime.memmove, exported for SeqAtomicLoad/SeqAtomicTryLoad. +// +//go:nosplit +func Memmove(to, from unsafe.Pointer, n uintptr) { + memmove(to, from, n) +} + +//go:linkname memmove runtime.memmove +//go:noescape +func memmove(to, from unsafe.Pointer, n uintptr) + +// Nanotime is runtime.nanotime. +// +//go:nosplit +func Nanotime() int64 { + return nanotime() +} + +//go:linkname nanotime runtime.nanotime +//go:noescape +func nanotime() int64 diff --git a/pkg/gohacks/noescape_unsafe.go b/pkg/gohacks/noescape_unsafe.go new file mode 100644 index 0000000..e6470e3 --- /dev/null +++ b/pkg/gohacks/noescape_unsafe.go @@ -0,0 +1,34 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gohacks + +import ( + "unsafe" +) + +// Noescape hides a pointer from escape analysis. Noescape is the identity +// function but escape analysis doesn't think the output depends on the input. +// Noescape is inlined and currently compiles down to zero instructions. +// USE CAREFULLY! +// +// Noescape is copy/pasted from Go's runtime/stubs.go:noescape(), and is valid +// as of Go 1.20. It is possible that this approach stops working in future +// versions of the toolchain, at which point `p` may still escape. +// +//go:nosplit +func Noescape(p unsafe.Pointer) unsafe.Pointer { + x := uintptr(p) + return unsafe.Pointer(x ^ 0) +} diff --git a/pkg/gohacks/slice_go113_unsafe.go b/pkg/gohacks/slice_go113_unsafe.go new file mode 100644 index 0000000..8ee39f5 --- /dev/null +++ b/pkg/gohacks/slice_go113_unsafe.go @@ -0,0 +1,45 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.13 && !go1.20 +// +build go1.13,!go1.20 + +// TODO(go.dev/issue/8422): Remove this once Go 1.19 is no longer supported, +// and update callers to use unsafe.Slice directly. + +package gohacks + +import ( + "unsafe" +) + +// sliceHeader is equivalent to reflect.SliceHeader, but represents the pointer +// to the underlying array as unsafe.Pointer rather than uintptr, allowing +// sliceHeaders to be directly converted to slice objects. +type sliceHeader struct { + Data unsafe.Pointer + Len int + Cap int +} + +// Slice returns a slice whose underlying array starts at ptr an which length +// and capacity are len. +func Slice[T any](ptr *T, length int) []T { + var s []T + hdr := (*sliceHeader)(unsafe.Pointer(&s)) + hdr.Data = unsafe.Pointer(ptr) + hdr.Len = length + hdr.Cap = length + return s +} diff --git a/pkg/gohacks/slice_go120_unsafe.go b/pkg/gohacks/slice_go120_unsafe.go new file mode 100644 index 0000000..9778db8 --- /dev/null +++ b/pkg/gohacks/slice_go120_unsafe.go @@ -0,0 +1,30 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.20 + +package gohacks + +import ( + "unsafe" +) + +// Slice returns a slice whose underlying array starts at ptr an which length +// and capacity are len. +// +// Slice is a wrapper around unsafe.Slice. Prefer to use unsafe.Slice directly +// if possible. +func Slice[T any](ptr *T, length int) []T { + return unsafe.Slice(ptr, length) +} diff --git a/pkg/gohacks/string_go113_unsafe.go b/pkg/gohacks/string_go113_unsafe.go new file mode 100644 index 0000000..dceeaf5 --- /dev/null +++ b/pkg/gohacks/string_go113_unsafe.go @@ -0,0 +1,51 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.13 && !go1.20 +// +build go1.13,!go1.20 + +// TODO(go.dev/issue/8422): Remove this file once Go 1.19 is no longer +// supported. + +package gohacks + +import ( + "unsafe" +) + +// stringHeader is equivalent to reflect.StringHeader, but represents the +// pointer to the underlying array as unsafe.Pointer rather than uintptr, +// allowing StringHeaders to be directly converted to strings. +type stringHeader struct { + Data unsafe.Pointer + Len int +} + +// ImmutableBytesFromString is equivalent to []byte(s), except that it uses the +// same memory backing s instead of making a heap-allocated copy. This is only +// valid if the returned slice is never mutated. +func ImmutableBytesFromString(s string) []byte { + shdr := (*stringHeader)(unsafe.Pointer(&s)) + return Slice((*byte)(shdr.Data), shdr.Len) +} + +// StringFromImmutableBytes is equivalent to string(bs), except that it uses +// the same memory backing bs instead of making a heap-allocated copy. This is +// only valid if bs is never mutated after StringFromImmutableBytes returns. +func StringFromImmutableBytes(bs []byte) string { + // This is cheaper than messing with StringHeader and SliceHeader, which as + // of this writing produces many dead stores of zeroes. Compare + // strings.Builder.String(). + return *(*string)(unsafe.Pointer(&bs)) +} diff --git a/pkg/gohacks/string_go120_unsafe.go b/pkg/gohacks/string_go120_unsafe.go new file mode 100644 index 0000000..9005efd --- /dev/null +++ b/pkg/gohacks/string_go120_unsafe.go @@ -0,0 +1,39 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.20 + +package gohacks + +import ( + "unsafe" +) + +// ImmutableBytesFromString is equivalent to []byte(s), except that it uses the +// same memory backing s instead of making a heap-allocated copy. This is only +// valid if the returned slice is never mutated. +func ImmutableBytesFromString(s string) []byte { + b := unsafe.StringData(s) + return unsafe.Slice(b, len(s)) +} + +// StringFromImmutableBytes is equivalent to string(bs), except that it uses +// the same memory backing bs instead of making a heap-allocated copy. This is +// only valid if bs is never mutated after StringFromImmutableBytes returns. +func StringFromImmutableBytes(bs []byte) string { + if len(bs) == 0 { + return "" + } + return unsafe.String(&bs[0], len(bs)) +} diff --git a/pkg/goid/goid.go b/pkg/goid/goid.go new file mode 100644 index 0000000..1531761 --- /dev/null +++ b/pkg/goid/goid.go @@ -0,0 +1,28 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package goid provides the Get function. +package goid + +import ( + _ "runtime" // For facts in assembly files. +) + +// goid returns the current goid, it is defined in assembly. +func goid() int64 + +// Get returns the ID of the current goroutine. +func Get() int64 { + return goid() +} diff --git a/pkg/goid/goid_122_amd64.s b/pkg/goid/goid_122_amd64.s new file mode 100644 index 0000000..5039f73 --- /dev/null +++ b/pkg/goid/goid_122_amd64.s @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.23 + +#include "textflag.h" + +#define GOID_OFFSET 152 // +checkoffset runtime g.goid + +// func goid() int64 +TEXT ·goid(SB),NOSPLIT|NOFRAME,$0-8 + MOVQ (TLS), R14 + MOVQ GOID_OFFSET(R14), R14 + MOVQ R14, ret+0(FP) + RET diff --git a/pkg/goid/goid_122_arm64.s b/pkg/goid/goid_122_arm64.s new file mode 100644 index 0000000..ec59b4b --- /dev/null +++ b/pkg/goid/goid_122_arm64.s @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.23 + +#include "textflag.h" + +#define GOID_OFFSET 152 // +checkoffset runtime g.goid + +// func goid() int64 +TEXT ·goid(SB),NOSPLIT,$0-8 + MOVD g, R0 // g + MOVD GOID_OFFSET(R0), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/goid/goid_123_amd64.s b/pkg/goid/goid_123_amd64.s new file mode 100644 index 0000000..21c61a7 --- /dev/null +++ b/pkg/goid/goid_123_amd64.s @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.23 && !go1.25 + +#include "textflag.h" + +#define GOID_OFFSET 160 // +checkoffset runtime g.goid + +// func goid() int64 +TEXT ·goid(SB),NOSPLIT|NOFRAME,$0-8 + MOVQ (TLS), R14 + MOVQ GOID_OFFSET(R14), R14 + MOVQ R14, ret+0(FP) + RET diff --git a/pkg/goid/goid_123_arm64.s b/pkg/goid/goid_123_arm64.s new file mode 100644 index 0000000..c5ea245 --- /dev/null +++ b/pkg/goid/goid_123_arm64.s @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.23 && !go1.25 + +#include "textflag.h" + +#define GOID_OFFSET 160 // +checkoffset runtime g.goid + +// func goid() int64 +TEXT ·goid(SB),NOSPLIT,$0-8 + MOVD g, R0 // g + MOVD GOID_OFFSET(R0), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/goid/goid_125_amd64.s b/pkg/goid/goid_125_amd64.s new file mode 100644 index 0000000..796459d --- /dev/null +++ b/pkg/goid/goid_125_amd64.s @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.25 + +#include "textflag.h" + +#define GOID_OFFSET 152 // +checkoffset runtime g.goid + +// func goid() int64 +TEXT ·goid(SB),NOSPLIT|NOFRAME,$0-8 + MOVQ (TLS), R14 + MOVQ GOID_OFFSET(R14), R14 + MOVQ R14, ret+0(FP) + RET diff --git a/pkg/goid/goid_125_arm64.s b/pkg/goid/goid_125_arm64.s new file mode 100644 index 0000000..e6879a1 --- /dev/null +++ b/pkg/goid/goid_125_arm64.s @@ -0,0 +1,26 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.25 + +#include "textflag.h" + +#define GOID_OFFSET 152 // +checkoffset runtime g.goid + +// func goid() int64 +TEXT ·goid(SB),NOSPLIT,$0-8 + MOVD g, R0 // g + MOVD GOID_OFFSET(R0), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/gomaxprocs/gomaxprocs.go b/pkg/gomaxprocs/gomaxprocs.go new file mode 100644 index 0000000..85df209 --- /dev/null +++ b/pkg/gomaxprocs/gomaxprocs.go @@ -0,0 +1,82 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gomaxprocs synchronizes adjustments to GOMAXPROCS. When this package +// is active (i.e. after the first call to SetBase), it sets the value of +// GOMAXPROCS to a "base" value (which should be set by a single goroutine, +// without races) plus a non-negative "temporary" value (which may be +// concurrently increased or decreased by multiple goroutines). +// +// Note that changing GOMAXPROCS stops the world, so callers should adjust +// GOMAXPROCS infrequently. +// +// TODO: Add gomaxprocs.Get() and check that other gVisor packages don't call +// runtime.GOMAXPROCS() at all. +package gomaxprocs + +import ( + "runtime" + + "github.com/sagernet/gvisor/pkg/log" +) + +var ( + mu gomaxprocsMutex + // +checklocks:mu + base int + // +checklocks:mu + temp int +) + +// SetBase sets base GOMAXPROCS. +func SetBase(n int) { + if n < 1 { + log.Traceback("Invalid base GOMAXPROCS: %d", n) + return + } + mu.Lock() + defer mu.Unlock() + oldBase := base + base = n + updateRuntime(oldBase, temp) +} + +// Add adds n temporary GOMAXPROCS. n may be negative; callers should call Add +// with negative n to remove temporary GOMAXPROCS when they are no longer +// needed. +func Add(n int) { + mu.Lock() + defer mu.Unlock() + t := temp + n + if t < 0 { + log.Traceback("gomaxprocs.Add(%d) would cause temp to become %d", n, t) + return + } + oldTemp := temp + temp = t + if base != 0 { + updateRuntime(base, oldTemp) + } +} + +// +checklocks:mu +func updateRuntime(oldBase, oldTemp int) { + n := base + temp + log.Debugf("Setting GOMAXPROCS to %d", n) + got := runtime.GOMAXPROCS(n) + if want := oldBase + oldTemp; oldBase != 0 && got != want { + // Something changed GOMAXPROCS outside of our control. + log.Warningf("Previous GOMAXPROCS was %d, expected %d = %d + %d", got, want, oldBase, oldTemp) + } +} diff --git a/pkg/gomaxprocs/gomaxprocs_mutex.go b/pkg/gomaxprocs/gomaxprocs_mutex.go new file mode 100644 index 0000000..512e803 --- /dev/null +++ b/pkg/gomaxprocs/gomaxprocs_mutex.go @@ -0,0 +1,64 @@ +package gomaxprocs + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type gomaxprocsMutex struct { + mu sync.Mutex +} + +var gomaxprocsprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var gomaxprocslockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type gomaxprocslockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *gomaxprocsMutex) Lock() { + locking.AddGLock(gomaxprocsprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *gomaxprocsMutex) NestedLock(i gomaxprocslockNameIndex) { + locking.AddGLock(gomaxprocsprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *gomaxprocsMutex) Unlock() { + locking.DelGLock(gomaxprocsprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *gomaxprocsMutex) NestedUnlock(i gomaxprocslockNameIndex) { + locking.DelGLock(gomaxprocsprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func gomaxprocsinitLockNames() {} + +func init() { + gomaxprocsinitLockNames() + gomaxprocsprefixIndex = locking.NewMutexClass(reflect.TypeOf(gomaxprocsMutex{}), gomaxprocslockNames) +} diff --git a/pkg/gomaxprocs/gomaxprocs_state_autogen.go b/pkg/gomaxprocs/gomaxprocs_state_autogen.go new file mode 100644 index 0000000..c17c168 --- /dev/null +++ b/pkg/gomaxprocs/gomaxprocs_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package gomaxprocs diff --git a/pkg/hostarch/access_type.go b/pkg/hostarch/access_type.go new file mode 100644 index 0000000..a085b60 --- /dev/null +++ b/pkg/hostarch/access_type.go @@ -0,0 +1,127 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostarch + +import "golang.org/x/sys/unix" + +// AccessType specifies memory access types. This is used for +// setting mapping permissions, as well as communicating faults. +// +// +stateify savable +type AccessType struct { + // Read is read access. + Read bool + + // Write is write access. + Write bool + + // Execute is executable access. + Execute bool +} + +// String returns a pretty representation of access. This looks like the +// familiar r-x, rw-, etc. and can be relied on as such. +func (a AccessType) String() string { + bits := [3]byte{'-', '-', '-'} + if a.Read { + bits[0] = 'r' + } + if a.Write { + bits[1] = 'w' + } + if a.Execute { + bits[2] = 'x' + } + return string(bits[:]) +} + +// Any returns true iff at least one of Read, Write or Execute is true. +func (a AccessType) Any() bool { + return a.Read || a.Write || a.Execute +} + +// Prot returns the system prot (unix.PROT_READ, etc.) for this access. +func (a AccessType) Prot() int { + var prot int + if a.Read { + prot |= unix.PROT_READ + } + if a.Write { + prot |= unix.PROT_WRITE + } + if a.Execute { + prot |= unix.PROT_EXEC + } + return prot +} + +// SupersetOf returns true iff the access types in a are a superset of the +// access types in other. +func (a AccessType) SupersetOf(other AccessType) bool { + if !a.Read && other.Read { + return false + } + if !a.Write && other.Write { + return false + } + if !a.Execute && other.Execute { + return false + } + return true +} + +// Intersect returns the access types set in both a and other. +func (a AccessType) Intersect(other AccessType) AccessType { + return AccessType{ + Read: a.Read && other.Read, + Write: a.Write && other.Write, + Execute: a.Execute && other.Execute, + } +} + +// Union returns the access types set in either a or other. +func (a AccessType) Union(other AccessType) AccessType { + return AccessType{ + Read: a.Read || other.Read, + Write: a.Write || other.Write, + Execute: a.Execute || other.Execute, + } +} + +// Effective returns the set of effective access types allowed by a, even if +// some types are not explicitly allowed. +func (a AccessType) Effective() AccessType { + // In Linux, Write and Execute access generally imply Read access. See + // mm/mmap.c:protection_map. + // + // The notable exception is get_user_pages, which only checks against + // the original vma flags. That said, most user memory accesses do not + // use GUP. + if a.Write || a.Execute { + a.Read = true + } + return a +} + +// Convenient access types. +var ( + NoAccess = AccessType{} + Read = AccessType{Read: true} + Write = AccessType{Write: true} + Execute = AccessType{Execute: true} + ReadWrite = AccessType{Read: true, Write: true} + ReadExecute = AccessType{Read: true, Execute: true} + AnyAccess = AccessType{Read: true, Write: true, Execute: true} +) diff --git a/pkg/hostarch/addr.go b/pkg/hostarch/addr.go new file mode 100644 index 0000000..2f7dcf1 --- /dev/null +++ b/pkg/hostarch/addr.go @@ -0,0 +1,119 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostarch + +import ( + "fmt" +) + +// Addr represents an address in an unspecified address space. +// +// +stateify savable +type Addr uintptr + +// AddLength adds the given length to start and returns the result. ok is true +// iff adding the length did not overflow the range of Addr. +// +// Note: This function is usually used to get the end of an address range +// defined by its start address and length. Since the resulting end is +// exclusive, end == 0 is technically valid, and corresponds to a range that +// extends to the end of the address space, but ok will be false. This isn't +// expected to ever come up in practice. +func (v Addr) AddLength(length uint64) (end Addr, ok bool) { + end = v + Addr(length) + // As of this writing (Go 1.21), addrAtLeast64b is required to prevent the + // compiler from generating a tautological `length <= MaxUint64` check on + // 64-bit architectures. + ok = end >= v && (addrAtLeast64b || length <= uint64(^Addr(0))) + return +} + +// RoundDown is equivalent to function PageRoundDown. +func (v Addr) RoundDown() Addr { + return PageRoundDown(v) +} + +// RoundUp is equivalent to function PageRoundUp. +func (v Addr) RoundUp() (Addr, bool) { + return PageRoundUp(v) +} + +// MustRoundUp is equivalent to function MustPageRoundUp. +func (v Addr) MustRoundUp() Addr { + return MustPageRoundUp(v) +} + +// HugeRoundDown is equivalent to function HugePageRoundDown. +func (v Addr) HugeRoundDown() Addr { + return HugePageRoundDown(v) +} + +// HugeRoundUp is equivalent to function HugePageRoundUp. +func (v Addr) HugeRoundUp() (Addr, bool) { + return HugePageRoundUp(v) +} + +// MustHugeRoundUp is equivalent to function MustHugePageRoundUp. +func (v Addr) MustHugeRoundUp() Addr { + return MustHugePageRoundUp(v) +} + +// PageOffset is equivalent to function PageOffset, except that it casts the +// result to uint64. +func (v Addr) PageOffset() uint64 { + return uint64(PageOffset(v)) +} + +// IsPageAligned is equivalent to function IsPageAligned. +func (v Addr) IsPageAligned() bool { + return IsPageAligned(v) +} + +// HugePageOffset is equivalent to function HugePageOffset. +func (v Addr) HugePageOffset() uint64 { + return uint64(HugePageOffset(v)) +} + +// IsHugePageAligned is equivalent to function IsHugePageAligned. +func (v Addr) IsHugePageAligned() bool { + return IsHugePageAligned(v) +} + +// AddrRange is a range of Addrs. +// +// type AddrRange + +// ToRange returns [v, v+length). +func (v Addr) ToRange(length uint64) (AddrRange, bool) { + end, ok := v.AddLength(length) + return AddrRange{v, end}, ok +} + +// IsPageAligned returns true if ar.Start.IsPageAligned() and +// ar.End.IsPageAligned(). +func (ar AddrRange) IsPageAligned() bool { + return ar.Start.IsPageAligned() && ar.End.IsPageAligned() +} + +// IsHugePageAligned returns true if ar.Start.IsHugePageAligned() and +// ar.End.IsHugePageAligned(). +func (ar AddrRange) IsHugePageAligned() bool { + return ar.Start.IsHugePageAligned() && ar.End.IsHugePageAligned() +} + +// String implements fmt.Stringer.String. +func (ar AddrRange) String() string { + return fmt.Sprintf("[%#x, %#x)", ar.Start, ar.End) +} diff --git a/pkg/hostarch/addr_range.go b/pkg/hostarch/addr_range.go new file mode 100644 index 0000000..8a771d6 --- /dev/null +++ b/pkg/hostarch/addr_range.go @@ -0,0 +1,76 @@ +package hostarch + +// A Range represents a contiguous range of T. +// +// +stateify savable +type AddrRange struct { + // Start is the inclusive start of the range. + Start Addr + + // End is the exclusive end of the range. + End Addr +} + +// WellFormed returns true if r.Start <= r.End. All other methods on a Range +// require that the Range is well-formed. +// +//go:nosplit +func (r AddrRange) WellFormed() bool { + return r.Start <= r.End +} + +// Length returns the length of the range. +// +//go:nosplit +func (r AddrRange) Length() Addr { + return r.End - r.Start +} + +// Contains returns true if r contains x. +// +//go:nosplit +func (r AddrRange) Contains(x Addr) bool { + return r.Start <= x && x < r.End +} + +// Overlaps returns true if r and r2 overlap. +// +//go:nosplit +func (r AddrRange) Overlaps(r2 AddrRange) bool { + return r.Start < r2.End && r2.Start < r.End +} + +// IsSupersetOf returns true if r is a superset of r2; that is, the range r2 is +// contained within r. +// +//go:nosplit +func (r AddrRange) IsSupersetOf(r2 AddrRange) bool { + return r.Start <= r2.Start && r.End >= r2.End +} + +// Intersect returns a range consisting of the intersection between r and r2. +// If r and r2 do not overlap, Intersect returns a range with unspecified +// bounds, but for which Length() == 0. +// +//go:nosplit +func (r AddrRange) Intersect(r2 AddrRange) AddrRange { + if r.Start < r2.Start { + r.Start = r2.Start + } + if r.End > r2.End { + r.End = r2.End + } + if r.End < r.Start { + r.End = r.Start + } + return r +} + +// CanSplitAt returns true if it is legal to split a segment spanning the range +// r at x; that is, splitting at x would produce two ranges, both of which have +// non-zero length. +// +//go:nosplit +func (r AddrRange) CanSplitAt(x Addr) bool { + return r.Contains(x) && r.Start < x +} diff --git a/pkg/hostarch/addr_range_seq_unsafe.go b/pkg/hostarch/addr_range_seq_unsafe.go new file mode 100644 index 0000000..b0fbf77 --- /dev/null +++ b/pkg/hostarch/addr_range_seq_unsafe.go @@ -0,0 +1,277 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostarch + +import ( + "bytes" + "fmt" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" +) + +// An AddrRangeSeq represents a sequence of AddrRanges. +// +// AddrRangeSeqs are immutable and may be copied by value. The zero value of +// AddrRangeSeq represents an empty sequence. +// +// An AddrRangeSeq may contain AddrRanges with a length of 0. This is necessary +// since zero-length AddrRanges are significant to MM bounds checks. +type AddrRangeSeq struct { + // If length is 0, then the AddrRangeSeq represents no AddrRanges. + // Invariants: data == 0; offset == 0; limit == 0. + // + // If length is 1, then the AddrRangeSeq represents the single + // AddrRange{offset, offset+limit}. Invariants: data == 0. + // + // Otherwise, length >= 2, and the AddrRangeSeq represents the `length` + // AddrRanges in the array of AddrRanges starting at address `data`, + // starting at `offset` bytes into the first AddrRange and limited to the + // following `limit` bytes. (AddrRanges after `limit` are still iterated, + // but are truncated to a length of 0.) Invariants: data != 0; offset <= + // data[0].Length(); limit > 0; offset+limit <= the combined length of all + // AddrRanges in the array. + data unsafe.Pointer + length int + offset Addr + limit Addr +} + +// AddrRangeSeqOf returns an AddrRangeSeq representing the single AddrRange ar. +func AddrRangeSeqOf(ar AddrRange) AddrRangeSeq { + return AddrRangeSeq{ + length: 1, + offset: ar.Start, + limit: ar.Length(), + } +} + +// AddrRangeSeqFromSlice returns an AddrRangeSeq representing all AddrRanges in +// slice. +// +// Whether the returned AddrRangeSeq shares memory with slice is unspecified; +// clients should avoid mutating slices passed to AddrRangeSeqFromSlice. +// +// Preconditions: The combined length of all AddrRanges in slice <= +// math.MaxInt64. +func AddrRangeSeqFromSlice(slice []AddrRange) AddrRangeSeq { + var limit int64 + for _, ar := range slice { + len64 := int64(ar.Length()) + if len64 < 0 { + panic(fmt.Sprintf("Length of AddrRange %v overflows int64", ar)) + } + sum := limit + len64 + if sum < limit { + panic(fmt.Sprintf("Total length of AddrRanges %v overflows int64", slice)) + } + limit = sum + } + return addrRangeSeqFromSliceLimited(slice, limit) +} + +// Preconditions: +// - The combined length of all AddrRanges in slice <= limit. +// - limit >= 0. +// - If len(slice) != 0, then limit > 0. +func addrRangeSeqFromSliceLimited(slice []AddrRange, limit int64) AddrRangeSeq { + switch len(slice) { + case 0: + return AddrRangeSeq{} + case 1: + return AddrRangeSeq{ + length: 1, + offset: slice[0].Start, + limit: Addr(limit), + } + default: + return AddrRangeSeq{ + data: unsafe.Pointer(&slice[0]), + length: len(slice), + limit: Addr(limit), + } + } +} + +// IsEmpty returns true if ars.NumRanges() == 0. +// +// Note that since AddrRangeSeq may contain AddrRanges with a length of zero, +// an AddrRange representing 0 bytes (AddrRangeSeq.NumBytes() == 0) is not +// necessarily empty. +func (ars AddrRangeSeq) IsEmpty() bool { + return ars.length == 0 +} + +// NumRanges returns the number of AddrRanges in ars. +func (ars AddrRangeSeq) NumRanges() int { + return ars.length +} + +// NumBytes returns the number of bytes represented by ars. +func (ars AddrRangeSeq) NumBytes() int64 { + return int64(ars.limit) +} + +// Head returns the first AddrRange in ars. +// +// Preconditions: !ars.IsEmpty(). +func (ars AddrRangeSeq) Head() AddrRange { + if ars.length == 0 { + panic("empty AddrRangeSeq") + } + if ars.length == 1 { + return AddrRange{ars.offset, ars.offset + ars.limit} + } + ar := *(*AddrRange)(ars.data) + ar.Start += ars.offset + if ar.Length() > ars.limit { + ar.End = ar.Start + ars.limit + } + return ar +} + +// Tail returns an AddrRangeSeq consisting of all AddrRanges in ars after the +// first. +// +// Preconditions: !ars.IsEmpty(). +func (ars AddrRangeSeq) Tail() AddrRangeSeq { + if ars.length == 0 { + panic("empty AddrRangeSeq") + } + if ars.length == 1 { + return AddrRangeSeq{} + } + return ars.externalTail() +} + +// Preconditions: ars.length >= 2. +func (ars AddrRangeSeq) externalTail() AddrRangeSeq { + data := (*AddrRange)(ars.data) + headLen := data.Length() - ars.offset + var tailLimit int64 + if ars.limit > headLen { + tailLimit = int64(ars.limit - headLen) + } + extSlice := gohacks.Slice(data, ars.length) + return addrRangeSeqFromSliceLimited(extSlice[1:], tailLimit) +} + +// DropFirst returns an AddrRangeSeq equivalent to ars, but with the first n +// bytes omitted. If n > ars.NumBytes(), DropFirst returns an empty +// AddrRangeSeq. +// +// If !ars.IsEmpty() and ars.Head().Length() == 0, DropFirst will always omit +// at least ars.Head(), even if n == 0. This guarantees that the basic pattern +// of: +// +// for !ars.IsEmpty() { +// n, err = doIOWith(ars.Head()) +// if err != nil { +// return err +// } +// ars = ars.DropFirst(n) +// } +// +// works even in the presence of zero-length AddrRanges. +// +// Preconditions: n >= 0. +func (ars AddrRangeSeq) DropFirst(n int) AddrRangeSeq { + if n < 0 { + panic(fmt.Sprintf("invalid n: %d", n)) + } + return ars.DropFirst64(int64(n)) +} + +// DropFirst64 is equivalent to DropFirst but takes an int64. +func (ars AddrRangeSeq) DropFirst64(n int64) AddrRangeSeq { + if n < 0 { + panic(fmt.Sprintf("invalid n: %d", n)) + } + if Addr(n) > ars.limit { + return AddrRangeSeq{} + } + // Handle initial empty AddrRange. + switch ars.length { + case 0: + return AddrRangeSeq{} + case 1: + if ars.limit == 0 { + return AddrRangeSeq{} + } + default: + if rawHeadLen := (*AddrRange)(ars.data).Length(); ars.offset == rawHeadLen { + ars = ars.externalTail() + } + } + for n != 0 { + // Calling ars.Head() here is surprisingly expensive, so inline getting + // the head's length. + var headLen Addr + if ars.length == 1 { + headLen = ars.limit + } else { + headLen = (*AddrRange)(ars.data).Length() - ars.offset + } + if Addr(n) < headLen { + // Dropping ends partway through the head AddrRange. + ars.offset += Addr(n) + ars.limit -= Addr(n) + return ars + } + n -= int64(headLen) + ars = ars.Tail() + } + return ars +} + +// TakeFirst returns an AddrRangeSeq equivalent to ars, but iterating at most n +// bytes. TakeFirst never removes AddrRanges from ars; AddrRanges beyond the +// first n bytes are reduced to a length of zero, but will still be iterated. +// +// Preconditions: n >= 0. +func (ars AddrRangeSeq) TakeFirst(n int) AddrRangeSeq { + if n < 0 { + panic(fmt.Sprintf("invalid n: %d", n)) + } + return ars.TakeFirst64(int64(n)) +} + +// TakeFirst64 is equivalent to TakeFirst but takes an int64. +func (ars AddrRangeSeq) TakeFirst64(n int64) AddrRangeSeq { + if n < 0 { + panic(fmt.Sprintf("invalid n: %d", n)) + } + if ars.limit > Addr(n) { + ars.limit = Addr(n) + } + return ars +} + +// String implements fmt.Stringer.String. +func (ars AddrRangeSeq) String() string { + // This is deliberately chosen to be the same as fmt's automatic stringer + // for []AddrRange. + var buf bytes.Buffer + buf.WriteByte('[') + var sep string + for !ars.IsEmpty() { + buf.WriteString(sep) + sep = " " + buf.WriteString(ars.Head().String()) + ars = ars.Tail() + } + buf.WriteByte(']') + return buf.String() +} diff --git a/pkg/hostarch/addr_unsafe.go b/pkg/hostarch/addr_unsafe.go new file mode 100644 index 0000000..63f494e --- /dev/null +++ b/pkg/hostarch/addr_unsafe.go @@ -0,0 +1,22 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostarch + +import ( + "unsafe" +) + +// This is used in addr.go:Addr.AddLength(). +const addrAtLeast64b = unsafe.Sizeof(Addr(0)) >= 8 diff --git a/pkg/hostarch/hostarch.go b/pkg/hostarch/hostarch.go new file mode 100644 index 0000000..af0bc93 --- /dev/null +++ b/pkg/hostarch/hostarch.go @@ -0,0 +1,8 @@ +// Copyright 2021 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// Package hostarch contains host arch address operations for user memory. +package hostarch diff --git a/pkg/hostarch/hostarch_arm64.go b/pkg/hostarch/hostarch_arm64.go new file mode 100644 index 0000000..82666eb --- /dev/null +++ b/pkg/hostarch/hostarch_arm64.go @@ -0,0 +1,98 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package hostarch + +import ( + "encoding/binary" + + "golang.org/x/sys/unix" +) + +const ( + // PageSize is the system page size. + // arm64 support 4K/16K/64K page size, + // which can be get by unix.Getpagesize(). + // Currently, only 4K page size is supported. + PageSize = 1 << PageShift + + // HugePageSize is the system huge page size. + HugePageSize = 1 << HugePageShift + + // CacheLineSize is the size of the cache line. + CacheLineSize = 1 << CacheLineShift + + // PageShift is the binary log of the system page size. + PageShift = 12 + + // HugePageShift is the binary log of the system huge page size. + // Should be calculated by "PageShift + (PageShift - 3)" + // when multiple page size support is ready. + HugePageShift = 21 + + // CacheLineShift is the binary log of the cache line size. + CacheLineShift = 6 +) + +// ByteOrder is the native byte order (little endian). +var ByteOrder = binary.LittleEndian + +// Arm64: Exception Syndrome Register EL1. +const ( + _ESR_ELx_EC_SHIFT = 26 + _ESR_ELx_EC_MASK = 0x3F << _ESR_ELx_EC_SHIFT + + _ESR_ELx_EC_IABT_LOW = 0x20 + _ESR_ELx_EC_DABT_LOW = 0x24 + + _ESR_ELx_WNR = 1 << 6 + _ESR_ELx_CM = 1 << 8 +) + +// ESRAccessType returns the memory access type for the given ESR (Exception +// Syndrome Register) code. If code does not represent an invalid memory +// access from a lower exception level, ESRAccessType returns NoAccess. +// +//go:nosplit +func ESRAccessType(code uint64) AccessType { + switch (code & _ESR_ELx_EC_MASK) >> _ESR_ELx_EC_SHIFT { + case _ESR_ELx_EC_IABT_LOW: + return Execute + case _ESR_ELx_EC_DABT_LOW: + // For faults on cache maintenance and address translation + // instructions, _ESR_ELx_WNR is always set. + if code&(_ESR_ELx_WNR|_ESR_ELx_CM) == _ESR_ELx_WNR { + return Write + } + return Read + default: + return NoAccess + } +} + +// UntaggedUserAddr clears the tag from the address pointer. Top-Byte-Ignore (TBI0) +// is enabled in Linux, so bits[63:56] of user space addresses are ignored. +func UntaggedUserAddr(addr Addr) Addr { + return Addr(int64(addr<<8) >> 8) +} + +func init() { + // Make sure the page size is 4K on arm64 platform. + if size := unix.Getpagesize(); size != PageSize { + panic("Only 4K page size is supported on arm64!") + } +} diff --git a/pkg/hostarch/hostarch_arm64_state_autogen.go b/pkg/hostarch/hostarch_arm64_state_autogen.go new file mode 100644 index 0000000..444542b --- /dev/null +++ b/pkg/hostarch/hostarch_arm64_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build arm64 +// +build arm64 + +package hostarch diff --git a/pkg/hostarch/hostarch_state_autogen.go b/pkg/hostarch/hostarch_state_autogen.go new file mode 100644 index 0000000..95d821f --- /dev/null +++ b/pkg/hostarch/hostarch_state_autogen.go @@ -0,0 +1,82 @@ +// automatically generated by stateify. + +package hostarch + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (a *AccessType) StateTypeName() string { + return "pkg/hostarch.AccessType" +} + +func (a *AccessType) StateFields() []string { + return []string{ + "Read", + "Write", + "Execute", + } +} + +func (a *AccessType) beforeSave() {} + +// +checklocksignore +func (a *AccessType) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.Read) + stateSinkObject.Save(1, &a.Write) + stateSinkObject.Save(2, &a.Execute) +} + +func (a *AccessType) afterLoad(context.Context) {} + +// +checklocksignore +func (a *AccessType) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.Read) + stateSourceObject.Load(1, &a.Write) + stateSourceObject.Load(2, &a.Execute) +} + +func (v *Addr) StateTypeName() string { + return "pkg/hostarch.Addr" +} + +func (v *Addr) StateFields() []string { + return nil +} + +func (r *AddrRange) StateTypeName() string { + return "pkg/hostarch.AddrRange" +} + +func (r *AddrRange) StateFields() []string { + return []string{ + "Start", + "End", + } +} + +func (r *AddrRange) beforeSave() {} + +// +checklocksignore +func (r *AddrRange) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.Start) + stateSinkObject.Save(1, &r.End) +} + +func (r *AddrRange) afterLoad(context.Context) {} + +// +checklocksignore +func (r *AddrRange) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.Start) + stateSourceObject.Load(1, &r.End) +} + +func init() { + state.Register((*AccessType)(nil)) + state.Register((*Addr)(nil)) + state.Register((*AddrRange)(nil)) +} diff --git a/pkg/hostarch/hostarch_unsafe_state_autogen.go b/pkg/hostarch/hostarch_unsafe_state_autogen.go new file mode 100644 index 0000000..419f249 --- /dev/null +++ b/pkg/hostarch/hostarch_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package hostarch diff --git a/pkg/hostarch/hostarch_x86.go b/pkg/hostarch/hostarch_x86.go new file mode 100644 index 0000000..eb0405b --- /dev/null +++ b/pkg/hostarch/hostarch_x86.go @@ -0,0 +1,48 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || 386 +// +build amd64 386 + +package hostarch + +import "encoding/binary" + +const ( + // PageSize is the system page size. + PageSize = 1 << PageShift + + // HugePageSize is the system huge page size. + HugePageSize = 1 << HugePageShift + + // CacheLineSize is the size of the cache line. + CacheLineSize = 1 << CacheLineShift + + // PageShift is the binary log of the system page size. + PageShift = 12 + + // HugePageShift is the binary log of the system huge page size. + HugePageShift = 21 + + // CacheLineShift is the binary log of the cache line size. + CacheLineShift = 6 +) + +// ByteOrder is the native byte order (little endian). +var ByteOrder = binary.LittleEndian + +// UntaggedUserAddr is no-op on x86. +func UntaggedUserAddr(addr Addr) Addr { + return addr +} diff --git a/pkg/hostarch/hostarch_x86_state_autogen.go b/pkg/hostarch/hostarch_x86_state_autogen.go new file mode 100644 index 0000000..239b6ba --- /dev/null +++ b/pkg/hostarch/hostarch_x86_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build amd64 || 386 +// +build amd64 386 + +package hostarch diff --git a/pkg/hostarch/memory_type.go b/pkg/hostarch/memory_type.go new file mode 100644 index 0000000..82d530b --- /dev/null +++ b/pkg/hostarch/memory_type.go @@ -0,0 +1,84 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostarch + +import "fmt" + +// MemoryType specifies CPU memory access behavior. +type MemoryType uint8 + +const ( + // MemoryTypeWriteBack is equivalent to Linux's default pgprot, or the + // following architectural memory types: + // + // - x86: Write-back (WB) + // + // - ARM64: Normal write-back cacheable + // + // This memory type is appropriate for typical application memory and must + // be the zero value for MemoryType. + MemoryTypeWriteBack MemoryType = iota + + // MemoryTypeWriteCombine is equivalent to Linux's pgprot_writecombine(), + // or the following architectural memory types: + // + // - x86: Write-combining (WC) + // + // - ARM64: Normal non-cacheable + MemoryTypeWriteCombine + + // MemoryTypeUncached is equivalent to Linux's pgprot_noncached(), or the + // following architectural memory types: + // + // - x86: Strong Uncacheable (UC) or Uncacheable (UC-); these differ in + // that UC- may be "downgraded" to WC by a setting of WC or (Intel only) WP + // in MTRR or EPT/NPT, but gVisor does not use MTRRs and KVM never sets WC + // or WP in EPT/NPT. + // + // - ARM64: Device-nGnRnE + MemoryTypeUncached + + // NumMemoryTypes is the number of memory types. + NumMemoryTypes +) + +// String implements fmt.Stringer.String. +func (mt MemoryType) String() string { + switch mt { + case MemoryTypeWriteBack: + return "WriteBack" + case MemoryTypeWriteCombine: + return "WriteCombine" + case MemoryTypeUncached: + return "Uncached" + default: + return fmt.Sprintf("%d", mt) + } +} + +// ShortString returns a two-character string compactly representing the +// MemoryType. +func (mt MemoryType) ShortString() string { + switch mt { + case MemoryTypeWriteBack: + return "WB" + case MemoryTypeWriteCombine: + return "WC" + case MemoryTypeUncached: + return "UC" + default: + return fmt.Sprintf("%02d", mt) + } +} diff --git a/pkg/hostarch/sizes_util.go b/pkg/hostarch/sizes_util.go new file mode 100644 index 0000000..d798675 --- /dev/null +++ b/pkg/hostarch/sizes_util.go @@ -0,0 +1,114 @@ +// Copyright 2022 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package hostarch + +// Masks often used when working with alignment in constant expressions. +const ( + PageMask = PageSize - 1 + HugePageMask = HugePageSize - 1 + CacheLineMask = CacheLineSize - 1 +) + +type bytecount interface { + ~uint | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +type hugebytecount interface { + ~uint | ~uint32 | ~uint64 | ~uintptr +} + +// PageRoundDown returns x rounded down to the nearest multiple of PageSize. +func PageRoundDown[T bytecount](x T) T { + return x &^ PageMask +} + +// PageRoundUp returns x rounded up to the nearest multiple of PageSize. ok is +// true iff rounding up does not overflow the range of T. +func PageRoundUp[T bytecount](x T) (val T, ok bool) { + val = PageRoundDown(x + PageMask) + ok = val >= x + return +} + +// MustPageRoundUp is equivalent to PageRoundUp, but panics if rounding up +// overflows. +func MustPageRoundUp[T bytecount](x T) T { + val, ok := PageRoundUp(x) + if !ok { + panic("PageRoundUp overflows") + } + return val +} + +// PageOffset returns the offset of x into its containing page. +func PageOffset[T bytecount](x T) T { + return x & PageMask +} + +// IsPageAligned returns true if x is a multiple of PageSize. +func IsPageAligned[T bytecount](x T) bool { + return PageOffset(x) == 0 +} + +// ToPagesRoundUp returns (the number of pages equal to x bytes rounded up, +// true). If rounding x up to a multiple of PageSize overflows the range of T, +// ToPagesRoundUp returns (unspecified, false). +func ToPagesRoundUp[T bytecount](x T) (T, bool) { + y := x + PageMask + if y < x { + return x, false + } + return y / PageSize, true +} + +// HugePageRoundDown returns x rounded down to the nearest multiple of +// HugePageSize. +func HugePageRoundDown[T hugebytecount](x T) T { + return x &^ HugePageMask +} + +// HugePageRoundUp returns x rounded up to the nearest multiple of +// HugePageSize. ok is true iff rounding up does not overflow the range of T. +func HugePageRoundUp[T hugebytecount](x T) (val T, ok bool) { + val = HugePageRoundDown(x + HugePageMask) + ok = val >= x + return +} + +// MustHugePageRoundUp is equivalent to HugePageRoundUp, but panics if rounding +// up overflows. +func MustHugePageRoundUp[T hugebytecount](x T) T { + val, ok := HugePageRoundUp(x) + if !ok { + panic("HugePageRoundUp overflows") + } + return val +} + +// HugePageOffset returns the offset of x into its containing page. +func HugePageOffset[T hugebytecount](x T) T { + return x & HugePageMask +} + +// IsHugePageAligned returns true if x is a multiple of HugePageSize. +func IsHugePageAligned[T hugebytecount](x T) bool { + return HugePageOffset(x) == 0 +} + +// CacheLineRoundDown returns the offset rounded down to the nearest multiple +// of CacheLineSize. +func CacheLineRoundDown[T bytecount](x T) T { + return x &^ CacheLineMask +} + +// CacheLineRoundUp returns the offset rounded up to the nearest multiple of +// CacheLineSize. ok is true iff rounding up does not overflow the range of T. +func CacheLineRoundUp[T bytecount](x T) (val T, ok bool) { + val = CacheLineRoundDown(x + CacheLineMask) + ok = val >= x + return +} diff --git a/pkg/hostsyscall/hostsyscall.go b/pkg/hostsyscall/hostsyscall.go new file mode 100644 index 0000000..c4c70a1 --- /dev/null +++ b/pkg/hostsyscall/hostsyscall.go @@ -0,0 +1,45 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package hostsyscall provides functions like unix.RawSyscall, but without the +// overhead of multiple stack frame allocations. +// +// This is mostly relevant for platform/kvm which needs to execute some function +// call chains in a go:nosplit environment. Debug builds specifically make using +// unix.RawSyscall variants infeasible. +package hostsyscall + +import ( + "golang.org/x/sys/unix" +) + +// RawSyscall6 is a copy of runtime.Syscall6. +func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1 uintptr, errno unix.Errno) + +// RawSyscall is a copy of runtime.Syscall6, but only uses the first three arguments. +func RawSyscall(trap, a1, a2, a3 uintptr) (r1 uintptr, errno unix.Errno) + +// Variants of runtime.Syscall6 that use slightly less stack space by only +// returning errno. + +// RawSyscallErrno6 is like RawSyscall6, but only returns errno, +// and 0 if successful. +func RawSyscallErrno6(trap, a1, a2, a3, a4, a5, a6 uintptr) unix.Errno + +// RawSyscallErrno is like RawSyscall, but only returns errno, +// and 0 if successful. +func RawSyscallErrno(trap, a1, a2, a3 uintptr) unix.Errno diff --git a/pkg/hostsyscall/hostsyscall_amd64.s b/pkg/hostsyscall/hostsyscall_amd64.s new file mode 100644 index 0000000..fa5f4dd --- /dev/null +++ b/pkg/hostsyscall/hostsyscall_amd64.s @@ -0,0 +1,96 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, errno) +TEXT ·RawSyscall6(SB),NOSPLIT,$0-72 + MOVQ a1+8(FP), DI + MOVQ a2+16(FP), SI + MOVQ a3+24(FP), DX + MOVQ a4+32(FP), R10 + MOVQ a5+40(FP), R8 + MOVQ a6+48(FP), R9 + MOVQ trap+0(FP), AX // syscall entry + SYSCALL + CMPQ AX, $0xfffffffffffff001 + JLS ok + MOVQ $-1, r1+56(FP) + NEGQ AX + MOVQ AX, errno+64(FP) + RET +ok: + MOVQ AX, r1+56(FP) + MOVQ $0, errno+64(FP) + RET + +// func RawSyscall(trap, a1, a2, a3 uintptr) (r1, errno) +TEXT ·RawSyscall(SB),NOSPLIT,$0-48 + MOVQ a1+8(FP), DI + MOVQ a2+16(FP), SI + MOVQ a3+24(FP), DX + MOVQ $0, R10 + MOVQ $0, R8 + MOVQ $0, R9 + MOVQ trap+0(FP), AX // syscall entry + SYSCALL + CMPQ AX, $0xfffffffffffff001 + JLS ok + MOVQ $-1, r1+32(FP) + NEGQ AX + MOVQ AX, errno+40(FP) + RET +ok: + MOVQ AX, r1+32(FP) + MOVQ $0, errno+40(FP) + RET + + +// func RawSyscallErrno6(trap, a1, a2, a3, a4, a5, a6 uintptr) (ret unix.Errno) +TEXT ·RawSyscallErrno6(SB),NOSPLIT,$0-64 + MOVQ a1+8(FP), DI + MOVQ a2+16(FP), SI + MOVQ a3+24(FP), DX + MOVQ a4+32(FP), R10 + MOVQ a5+40(FP), R8 + MOVQ a6+48(FP), R9 + MOVQ trap+0(FP), AX // syscall entry + SYSCALL + CMPQ AX, $0xfffffffffffff001 + JLS ok + NEGQ AX + MOVQ AX, ret+56(FP) + RET +ok: + MOVQ $0, ret+56(FP) + RET + +// func RawSyscallErrno(trap, a1, a2, a3 uintptr) (ret unix.Errno) +TEXT ·RawSyscallErrno(SB),NOSPLIT,$0-40 + MOVQ a1+8(FP), DI + MOVQ a2+16(FP), SI + MOVQ a3+24(FP), DX + MOVQ $0, R10 + MOVQ $0, R8 + MOVQ $0, R9 + MOVQ trap+0(FP), AX // syscall entry + SYSCALL + CMPQ AX, $0xfffffffffffff001 + JLS ok + NEGQ AX + MOVQ AX, ret+32(FP) + RET +ok: + MOVQ $0, ret+32(FP) + RET diff --git a/pkg/hostsyscall/hostsyscall_arm64.s b/pkg/hostsyscall/hostsyscall_arm64.s new file mode 100644 index 0000000..1d0beed --- /dev/null +++ b/pkg/hostsyscall/hostsyscall_arm64.s @@ -0,0 +1,97 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, errno) +TEXT ·RawSyscall6(SB),NOSPLIT,$0-72 + MOVD trap+0(FP), R8 // syscall entry + MOVD a1+8(FP), R0 + MOVD a2+16(FP), R1 + MOVD a3+24(FP), R2 + MOVD a4+32(FP), R3 + MOVD a5+40(FP), R4 + MOVD a6+48(FP), R5 + SVC + CMN $4095, R0 + BCC ok + MOVD $-1, R4 + MOVD R4, r1+56(FP) + NEG R0, R0 + MOVD R0, errno+64(FP) + RET +ok: + MOVD R0, r1+56(FP) + MOVD ZR, errno+64(FP) + RET + +// func RawSyscall(trap, a1, a2, a3 uintptr) (r1, errno) +TEXT ·RawSyscall(SB),NOSPLIT,$0-48 + MOVD trap+0(FP), R8 // syscall entry + MOVD a1+8(FP), R0 + MOVD a2+16(FP), R1 + MOVD a3+24(FP), R2 + MOVD ZR, R3 + MOVD ZR, R4 + MOVD ZR, R5 + SVC + CMN $4095, R0 + BCC ok + MOVD $-1, R4 + MOVD R4, r1+32(FP) + NEG R0, R0 + MOVD R0, errno+40(FP) + RET +ok: + MOVD R0, r1+32(FP) + MOVD ZR, errno+40(FP) + RET + +// func RawSyscallErrno6(trap, a1, a2, a3, a4, a5, a6 uintptr) (errno unix.Errno) +TEXT ·RawSyscallErrno6(SB),NOSPLIT,$0-64 + MOVD trap+0(FP), R8 // syscall entry + MOVD a1+8(FP), R0 + MOVD a2+16(FP), R1 + MOVD a3+24(FP), R2 + MOVD a4+32(FP), R3 + MOVD a5+40(FP), R4 + MOVD a6+48(FP), R5 + SVC + CMN $4095, R0 + BCC ok + NEG R0, R0 + MOVD R0, ret+56(FP) // errno + RET +ok: + MOVD ZR, ret+56(FP) // errno + RET + +// func RawSyscallErrno(trap, a1, a2, a3 uintptr) (errno unix.Errno) +TEXT ·RawSyscallErrno(SB),NOSPLIT,$0-40 + MOVD trap+0(FP), R8 // syscall entry + MOVD a1+8(FP), R0 + MOVD a2+16(FP), R1 + MOVD a3+24(FP), R2 + MOVD ZR, R3 + MOVD ZR, R4 + MOVD ZR, R5 + SVC + CMN $4095, R0 + BCC ok + NEG R0, R0 + MOVD R0, ret+32(FP) // errno + RET +ok: + MOVD ZR, ret+32(FP) // errno + RET diff --git a/pkg/hostsyscall/hostsyscall_state_autogen.go b/pkg/hostsyscall/hostsyscall_state_autogen.go new file mode 100644 index 0000000..dcfdacf --- /dev/null +++ b/pkg/hostsyscall/hostsyscall_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package hostsyscall diff --git a/pkg/hosttid/hosttid.go b/pkg/hosttid/hosttid.go new file mode 100644 index 0000000..a556b19 --- /dev/null +++ b/pkg/hosttid/hosttid.go @@ -0,0 +1,30 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hosttid provides the Current function. +package hosttid + +import ( + "runtime" +) + +// Dummy references for facts. +const _ = runtime.Compiler + +// Current returns the caller's host thread ID. Unless runtime.LockOSThread() +// is in effect, this function is inherently racy since the Go runtime may +// migrate the calling goroutine to another thread at any time. +// +// Current is equivalent to unix.Gettid(), but faster. +func Current() uint64 diff --git a/pkg/hosttid/hosttid_124_amd64.s b/pkg/hosttid/hosttid_124_amd64.s new file mode 100644 index 0000000..3047ce0 --- /dev/null +++ b/pkg/hosttid/hosttid_124_amd64.s @@ -0,0 +1,29 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.25 + +#include "textflag.h" + +#define M_OFFSET 48 // +checkoffset runtime g.m +#define PROCID_OFFSET 72 // +checkoffset runtime m.procid + +TEXT ·Current(SB),NOSPLIT|NOFRAME,$0-8 + // procid is in getg().m.procid. + MOVQ TLS, AX + MOVQ 0(AX)(TLS*1), AX + MOVQ M_OFFSET(AX), AX // gp.m + MOVQ PROCID_OFFSET(AX), AX // mp.procid + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/hosttid/hosttid_124_arm64.s b/pkg/hosttid/hosttid_124_arm64.s new file mode 100644 index 0000000..9dd1011 --- /dev/null +++ b/pkg/hosttid/hosttid_124_arm64.s @@ -0,0 +1,28 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.25 + +#include "textflag.h" + +#define M_OFFSET 48 // +checkoffset runtime g.m +#define PROCID_OFFSET 72 // +checkoffset runtime m.procid + +TEXT ·Current(SB),NOSPLIT,$0-8 + // procid is in getg().m.procid. + MOVD g, R0 // g + MOVD M_OFFSET(R0), R0 // gp.m + MOVD PROCID_OFFSET(R0), R0 // mp.procid + MOVD R0, ret+0(FP) + RET diff --git a/pkg/hosttid/hosttid_125_amd64.s b/pkg/hosttid/hosttid_125_amd64.s new file mode 100644 index 0000000..37c915a --- /dev/null +++ b/pkg/hosttid/hosttid_125_amd64.s @@ -0,0 +1,29 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.25 + +#include "textflag.h" + +#define M_OFFSET 48 // +checkoffset runtime g.m +#define PROCID_OFFSET 64 // +checkoffset runtime m.procid + +TEXT ·Current(SB),NOSPLIT|NOFRAME,$0-8 + // procid is in getg().m.procid. + MOVQ TLS, AX + MOVQ 0(AX)(TLS*1), AX + MOVQ M_OFFSET(AX), AX // gp.m + MOVQ PROCID_OFFSET(AX), AX // mp.procid + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/hosttid/hosttid_125_arm64.s b/pkg/hosttid/hosttid_125_arm64.s new file mode 100644 index 0000000..e450ad0 --- /dev/null +++ b/pkg/hosttid/hosttid_125_arm64.s @@ -0,0 +1,28 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.25 + +#include "textflag.h" + +#define M_OFFSET 48 // +checkoffset runtime g.m +#define PROCID_OFFSET 64 // +checkoffset runtime m.procid + +TEXT ·Current(SB),NOSPLIT,$0-8 + // procid is in getg().m.procid. + MOVD g, R0 // g + MOVD M_OFFSET(R0), R0 // gp.m + MOVD PROCID_OFFSET(R0), R0 // mp.procid + MOVD R0, ret+0(FP) + RET diff --git a/pkg/hosttid/hosttid_state_autogen.go b/pkg/hosttid/hosttid_state_autogen.go new file mode 100644 index 0000000..5711dab --- /dev/null +++ b/pkg/hosttid/hosttid_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package hosttid diff --git a/pkg/ilist/ilist_state_autogen.go b/pkg/ilist/ilist_state_autogen.go new file mode 100644 index 0000000..49349ac --- /dev/null +++ b/pkg/ilist/ilist_state_autogen.go @@ -0,0 +1,70 @@ +// automatically generated by stateify. + +package ilist + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (l *List) StateTypeName() string { + return "pkg/ilist.List" +} + +func (l *List) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *List) beforeSave() {} + +// +checklocksignore +func (l *List) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *List) afterLoad(context.Context) {} + +// +checklocksignore +func (l *List) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *Entry) StateTypeName() string { + return "pkg/ilist.Entry" +} + +func (e *Entry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *Entry) beforeSave() {} + +// +checklocksignore +func (e *Entry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *Entry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Entry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*List)(nil)) + state.Register((*Entry)(nil)) +} diff --git a/pkg/ilist/interface_list.go b/pkg/ilist/interface_list.go new file mode 100644 index 0000000..faa73e3 --- /dev/null +++ b/pkg/ilist/interface_list.go @@ -0,0 +1,258 @@ +package ilist + +// Linker is the interface that objects must implement if they want to be added +// to and/or removed from List objects. +// +// N.B. When substituted in a template instantiation, Linker doesn't need to +// be an interface, and in most cases won't be. +type Linker interface { + Next() Element + Prev() Element + SetNext(Element) + SetPrev(Element) +} + +// Element the item that is used at the API level. +// +// N.B. Like Linker, this is unlikely to be an interface in most cases. +type Element interface { + Linker +} + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type ElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (ElementMapper) linkerFor(elem Element) Linker { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type List struct { + head Element + tail Element +} + +// Reset resets list l to the empty state. +func (l *List) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *List) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *List) Front() Element { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *List) Back() Element { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *List) Len() (count int) { + for e := l.Front(); e != nil; e = (ElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *List) PushFront(e Element) { + linker := ElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + ElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *List) PushFrontList(m *List) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + ElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + ElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *List) PushBack(e Element) { + linker := ElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + ElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *List) PushBackList(m *List) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + ElementMapper{}.linkerFor(l.tail).SetNext(m.head) + ElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *List) InsertAfter(b, e Element) { + bLinker := ElementMapper{}.linkerFor(b) + eLinker := ElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + ElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *List) InsertBefore(a, e Element) { + aLinker := ElementMapper{}.linkerFor(a) + eLinker := ElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + ElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *List) Remove(e Element) { + linker := ElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + ElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + ElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type Entry struct { + next Element + prev Element +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *Entry) Next() Element { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *Entry) Prev() Element { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *Entry) SetNext(elem Element) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *Entry) SetPrev(elem Element) { + e.prev = elem +} diff --git a/pkg/linewriter/linewriter.go b/pkg/linewriter/linewriter.go new file mode 100644 index 0000000..7fe4ca1 --- /dev/null +++ b/pkg/linewriter/linewriter.go @@ -0,0 +1,79 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package linewriter provides an io.Writer which calls an emitter on each line. +package linewriter + +import ( + "bytes" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Writer is an io.Writer which buffers input, flushing +// individual lines through an emitter function. +type Writer struct { + // the mutex locks buf. + sync.Mutex + + // buf holds the data we haven't emitted yet. + buf bytes.Buffer + + // emit is used to flush individual lines. + emit func(p []byte) +} + +// NewWriter creates a Writer which emits using emitter. +// The emitter must not retain p. It may change after emitter returns. +func NewWriter(emitter func(p []byte)) *Writer { + return &Writer{emit: emitter} +} + +// Write implements io.Writer.Write. +// It calls emit on each line of input, not including the newline. +// Write may be called concurrently. +func (w *Writer) Write(p []byte) (int, error) { + w.Lock() + defer w.Unlock() + + total := 0 + for len(p) > 0 { + emit := true + i := bytes.IndexByte(p, '\n') + if i < 0 { + // No newline, we will buffer everything. + i = len(p) + emit = false + } + + n, err := w.buf.Write(p[:i]) + if err != nil { + return total, err + } + total += n + + p = p[i:] + + if emit { + // Skip the newline, but still count it. + p = p[1:] + total++ + + w.emit(w.buf.Bytes()) + w.buf.Reset() + } + } + + return total, nil +} diff --git a/pkg/log/bug.go b/pkg/log/bug.go new file mode 100644 index 0000000..234d63e --- /dev/null +++ b/pkg/log/bug.go @@ -0,0 +1,161 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "fmt" + "runtime" + "strings" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// This file contains helper functions analogous to the Linux kernel's WARN* +// macros. Should be used for non-fatal errors that should be treated as bugs +// none the less. + +const ( + warnFmtStr = "WARNING: BUG on %s:%d\n" + warnUnknownLineStr = "WARNING: BUG on unknown line\n" + catchAllMagic = "runtime.Caller failed" +) + +//go:noinline +func reportBugErr(caller int, err error) { + reportBug(caller+1, err.Error(), nil) +} + +func reportBug(caller int, msg string, vars []any) { + var b strings.Builder + if _, file, line, ok := runtime.Caller(caller); ok { + b.WriteString(fmt.Sprintf(warnFmtStr, file, line)) + } else { + b.WriteString(warnUnknownLineStr) + } + b.WriteByte('\n') + if len(msg) > 0 { + if len(vars) > 0 { + b.WriteString(fmt.Sprintf(msg, vars...)) + } else { + b.WriteString(msg) + } + b.WriteByte('\n') + } + TracebackAll(b.String()) +} + +var ( + // warnedMu protects the variables below. + warnedMu sync.Mutex + // warnedSet is used to keep track of which WarnOnOnce calls have fired. + warnedSet map[string]struct{} +) + +//go:noinline +func reportBugErrOnce(caller int, err error) { + reportBugOnce(caller+1, err.Error(), nil) +} + +func reportBugOnce(caller int, msg string, vars []any) { + var b strings.Builder + if _, file, line, ok := runtime.Caller(caller); ok { + key := fmt.Sprintf("%s:%d", file, line) + + warnedMu.Lock() + defer warnedMu.Unlock() + + if _, ok = warnedSet[key]; !ok { + b.WriteString(fmt.Sprintf(warnFmtStr, file, line)) + b.WriteByte('\n') + if len(msg) > 0 { + if len(vars) > 0 { + b.WriteString(fmt.Sprintf(msg, vars...)) + } else { + b.WriteString(msg) + } + b.WriteByte('\n') + } + + TracebackAll(b.String()) + warnedSet[key] = struct{}{} + } + } else { + warnedMu.Lock() + defer warnedMu.Unlock() + + // Use const string as a catch-all when runtime.Caller fails, + // so as to avoid log-spam since that's the point of WARN_ONCE. + if _, ok := warnedSet[catchAllMagic]; !ok { + b.WriteString(warnUnknownLineStr) + b.WriteByte('\n') + if len(msg) > 0 { + if len(vars) > 0 { + b.WriteString(fmt.Sprintf(msg, vars...)) + } else { + b.WriteString(msg) + } + b.WriteByte('\n') + } + + TracebackAll(b.String()) + warnedSet[catchAllMagic] = struct{}{} + } + } +} + +// BugTraceback will report a bug with a traceback of all goroutines if the +// error isn't nil. Use it for reporting abnormal bugs encountered at runtime +// that should be fixed. +// +// Do not use this for bad user input. Errors reported by this function should +// not be fatal. +func BugTraceback(err error) { + if err != nil { + reportBugErr(2, err) + } +} + +// BugTracebackf will report a bug with a traceback of all goroutines. +// Use it for reporting abnormal bugs encountered at runtime that should be +// fixed. +// +// Do not use this for bad user input. Errors reported by this function should +// not be fatal. +func BugTracebackf(s string, a ...any) { + reportBug(2, s, a) +} + +// BugTracebackOnce will report a bug with a traceback of all goroutines if the +// error isn't nil. Use it for reporting abnormal bugs encountered at runtime +// that should be fixed. If called multiple time from same invocation, will only +// print once. +// +// Do not use this for bad user input. Errors reported by this function should +// not be fatal. +func BugTracebackOnce(err error) { + if err != nil { + reportBugErrOnce(2, err) + } +} + +// BugTracebackfOnce will report a bug with a traceback of all goroutines. +// Use it for reporting abnormal bugs encountered at runtime that should be +// fixed. If called multiple time from same invocation, will only print once. +// +// Do not use this for bad user input. Errors reported by this function should +// not be fatal. +func BugTracebackfOnce(s string, a ...any) { + reportBugOnce(2, s, a) +} diff --git a/pkg/log/glog.go b/pkg/log/glog.go new file mode 100644 index 0000000..553f7fe --- /dev/null +++ b/pkg/log/glog.go @@ -0,0 +1,86 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "fmt" + "os" + "runtime" + "strings" + "time" +) + +// GoogleEmitter is a wrapper that emits logs in a format compatible with +// package github.com/golang/glog. +type GoogleEmitter struct { + *Writer +} + +// pid is used for the threadid component of the header. +var pid = os.Getpid() + +// Emit emits the message, google-style. +// +// Log lines have this form: +// +// Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... +// +// where the fields are defined as follows: +// +// L A single character, representing the log level (eg 'I' for INFO) +// mm The month (zero padded; ie May is '05') +// dd The day (zero padded) +// hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds +// threadid The space-padded thread ID as returned by GetTID() +// file The file name +// line The line number +// msg The user-supplied message +func (g GoogleEmitter) Emit(depth int, level Level, timestamp time.Time, format string, args ...any) { + // Log level. + prefix := byte('?') + switch level { + case Debug: + prefix = byte('D') + case Info: + prefix = byte('I') + case Warning: + prefix = byte('W') + } + + // Timestamp. + _, month, day := timestamp.Date() + hour, minute, second := timestamp.Clock() + microsecond := int(timestamp.Nanosecond() / 1000) + + // 0 = this frame. + _, file, line, ok := runtime.Caller(depth + 1) + if ok { + // Trim any directory path from the file. + slash := strings.LastIndexByte(file, byte('/')) + if slash >= 0 { + file = file[slash+1:] + } + } else { + // We don't have a filename. + file = "???" + line = 0 + } + + // Generate the message. + message := fmt.Sprintf(format, args...) + + // Emit the formatted result. + fmt.Fprintf(g.Writer, "%c%02d%02d %02d:%02d:%02d.%06d % 7d %s:%d] %s\n", prefix, int(month), day, hour, minute, second, microsecond, pid, file, line, message) +} diff --git a/pkg/log/json.go b/pkg/log/json.go new file mode 100644 index 0000000..a57bc10 --- /dev/null +++ b/pkg/log/json.go @@ -0,0 +1,85 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "encoding/json" + "fmt" + "runtime" + "strings" + "time" +) + +type jsonLog struct { + Msg string `json:"msg"` + Level Level `json:"level"` + Time time.Time `json:"time"` +} + +// MarshalJSON implements json.Marshaler.MarashalJSON. +func (l Level) MarshalJSON() ([]byte, error) { + switch l { + case Warning: + return []byte(`"warning"`), nil + case Info: + return []byte(`"info"`), nil + case Debug: + return []byte(`"debug"`), nil + default: + return nil, fmt.Errorf("unknown level %v", l) + } +} + +// UnmarshalJSON implements json.Unmarshaler.UnmarshalJSON. It can unmarshal +// from both string names and integers. +func (l *Level) UnmarshalJSON(b []byte) error { + switch s := string(b); s { + case "0", `"warning"`: + *l = Warning + case "1", `"info"`: + *l = Info + case "2", `"debug"`: + *l = Debug + default: + return fmt.Errorf("unknown level %q", s) + } + return nil +} + +// JSONEmitter logs messages in json format. +type JSONEmitter struct { + *Writer +} + +// Emit implements Emitter.Emit. +func (e JSONEmitter) Emit(depth int, level Level, timestamp time.Time, format string, v ...any) { + logLine := fmt.Sprintf(format, v...) + if _, file, line, ok := runtime.Caller(depth + 1); ok { + if slash := strings.LastIndexByte(file, byte('/')); slash >= 0 { + file = file[slash+1:] // Trim any directory path from the file. + } + logLine = fmt.Sprintf("%s:%d] %s", file, line, logLine) + } + j := jsonLog{ + Msg: logLine, + Level: level, + Time: timestamp, + } + b, err := json.Marshal(j) + if err != nil { + panic(err) + } + e.Writer.Write(b) +} diff --git a/pkg/log/json_k8s.go b/pkg/log/json_k8s.go new file mode 100644 index 0000000..8f5aab5 --- /dev/null +++ b/pkg/log/json_k8s.go @@ -0,0 +1,56 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "encoding/json" + "fmt" + "runtime" + "strings" + "time" +) + +type k8sJSONLog struct { + Log string `json:"log"` + Level Level `json:"level"` + Time time.Time `json:"time"` +} + +// K8sJSONEmitter logs messages in json format that is compatible with +// Kubernetes fluent configuration. +type K8sJSONEmitter struct { + *Writer +} + +// Emit implements Emitter.Emit. +func (e K8sJSONEmitter) Emit(depth int, level Level, timestamp time.Time, format string, v ...any) { + logLine := fmt.Sprintf(format, v...) + if _, file, line, ok := runtime.Caller(depth + 1); ok { + if slash := strings.LastIndexByte(file, byte('/')); slash >= 0 { + file = file[slash+1:] // Trim any directory path from the file. + } + logLine = fmt.Sprintf("%s:%d] %s", file, line, logLine) + } + j := k8sJSONLog{ + Log: logLine, + Level: level, + Time: timestamp, + } + b, err := json.Marshal(j) + if err != nil { + panic(err) + } + e.Writer.Write(b) +} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..7201a19 --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,401 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package log implements a library for logging. +// +// This is separate from the standard logging package because logging may be a +// high-impact activity, and therefore we wanted to provide as much flexibility +// as possible in the underlying implementation. +// +// Note that logging should still be considered high-impact, and should not be +// done in the hot path. If necessary, logging statements should be protected +// with guards regarding the logging level. For example, +// +// if log.IsLogging(log.Debug) { +// log.Debugf(...) +// } +// +// This is because the log.Debugf(...) statement alone will generate a +// significant amount of garbage and churn in many cases, even if no log +// message is ultimately emitted. +// +// +checkalignedignore +package log + +import ( + "fmt" + "io" + stdlog "log" + "os" + "regexp" + "runtime" + "sync/atomic" + "time" + + "github.com/sagernet/gvisor/pkg/linewriter" + "github.com/sagernet/gvisor/pkg/sync" +) + +// Level is the log level. +type Level uint32 + +// The following levels are fixed, and can never be changed. Since some control +// RPCs allow for changing the level as an integer, it is only possible to add +// additional levels, and the existing one cannot be removed. +const ( + // Warning indicates that output should always be emitted. + Warning Level = iota + + // Info indicates that output should normally be emitted. + Info + + // Debug indicates that output should not normally be emitted. + Debug +) + +func (l Level) String() string { + switch l { + case Warning: + return "Warning" + case Info: + return "Info" + case Debug: + return "Debug" + default: + return fmt.Sprintf("Invalid level: %d", l) + } +} + +// Emitter is the final destination for logs. +type Emitter interface { + // Emit emits the given log statement. This allows for control over the + // timestamp used for logging. + Emit(depth int, level Level, timestamp time.Time, format string, v ...any) +} + +// Writer writes the output to the given writer. +type Writer struct { + // Next is where output is written. + Next io.Writer + + // mu protects fields below. + mu sync.Mutex + + // errors counts failures to write log messages so it can be reported + // when writer start to work again. Needs to be accessed using atomics + // to make race detector happy because it's read outside the mutex. + // +checklocks + atomicErrors int32 +} + +// Write writes out the given bytes, handling non-blocking sockets. +func (l *Writer) Write(data []byte) (int, error) { + n := 0 + + for n < len(data) { + w, err := l.Next.Write(data[n:]) + n += w + + // Is it a non-blocking socket? + if pathErr, ok := err.(*os.PathError); ok && pathErr.Timeout() { + runtime.Gosched() + continue + } + + // Some other error? + if err != nil { + l.mu.Lock() + atomic.AddInt32(&l.atomicErrors, 1) + l.mu.Unlock() + return n, err + } + } + + // Do we need to end with a '\n'? + if len(data) == 0 || data[len(data)-1] != '\n' { + l.Write([]byte{'\n'}) + } + + // Dirty read in case there were errors (rare). + if atomic.LoadInt32(&l.atomicErrors) > 0 { + l.mu.Lock() + defer l.mu.Unlock() + + // Recheck condition under lock. + if e := atomic.LoadInt32(&l.atomicErrors); e > 0 { + msg := fmt.Sprintf("\n*** Dropped %d log messages ***\n", e) + if _, err := l.Next.Write([]byte(msg)); err == nil { + atomic.StoreInt32(&l.atomicErrors, 0) + } + } + } + + return n, nil +} + +// Emit emits the message. +func (l *Writer) Emit(_ int, _ Level, _ time.Time, format string, args ...any) { + fmt.Fprintf(l, format, args...) +} + +// MultiEmitter is an emitter that emits to multiple Emitters. +type MultiEmitter []Emitter + +// Emit emits to all emitters. +func (m *MultiEmitter) Emit(depth int, level Level, timestamp time.Time, format string, v ...any) { + for _, e := range *m { + e.Emit(1+depth, level, timestamp, format, v...) + } +} + +// TestLogger is implemented by testing.T and testing.B. +type TestLogger interface { + Logf(format string, v ...any) +} + +// TestEmitter may be used for wrapping tests. +type TestEmitter struct { + TestLogger +} + +// Emit emits to the TestLogger. +func (t *TestEmitter) Emit(_ int, level Level, timestamp time.Time, format string, v ...any) { + t.Logf(format, v...) +} + +// Logger is a high-level logging interface. It is in fact, not used within the +// log package. Rather it is provided for others to provide contextual loggers +// that may append some addition information to log statement. BasicLogger +// satisfies this interface, and may be passed around as a Logger. +type Logger interface { + // Debugf logs a debug statement. + Debugf(format string, v ...any) + + // Infof logs at an info level. + Infof(format string, v ...any) + + // Warningf logs at a warning level. + Warningf(format string, v ...any) + + // IsLogging returns true iff this level is being logged. This may be + // used to short-circuit expensive operations for debugging calls. + IsLogging(level Level) bool +} + +// BasicLogger is the default implementation of Logger. +type BasicLogger struct { + Level + Emitter +} + +// Debugf implements logger.Debugf. +func (l *BasicLogger) Debugf(format string, v ...any) { + l.DebugfAtDepth(1, format, v...) +} + +// Infof implements logger.Infof. +func (l *BasicLogger) Infof(format string, v ...any) { + l.InfofAtDepth(1, format, v...) +} + +// Warningf implements logger.Warningf. +func (l *BasicLogger) Warningf(format string, v ...any) { + l.WarningfAtDepth(1, format, v...) +} + +// DebugfAtDepth logs at a specific depth. +func (l *BasicLogger) DebugfAtDepth(depth int, format string, v ...any) { + if l.IsLogging(Debug) { + l.Emit(1+depth, Debug, time.Now(), format, v...) + } +} + +// InfofAtDepth logs at a specific depth. +func (l *BasicLogger) InfofAtDepth(depth int, format string, v ...any) { + if l.IsLogging(Info) { + l.Emit(1+depth, Info, time.Now(), format, v...) + } +} + +// WarningfAtDepth logs at a specific depth. +func (l *BasicLogger) WarningfAtDepth(depth int, format string, v ...any) { + if l.IsLogging(Warning) { + l.Emit(1+depth, Warning, time.Now(), format, v...) + } +} + +// IsLogging implements logger.IsLogging. +func (l *BasicLogger) IsLogging(level Level) bool { + return atomic.LoadUint32((*uint32)(&l.Level)) >= uint32(level) +} + +// SetLevel sets the logging level. +func (l *BasicLogger) SetLevel(level Level) { + atomic.StoreUint32((*uint32)(&l.Level), uint32(level)) +} + +// logMu protects Log below. We use atomic operations to read the value, but +// updates require logMu to ensure consistency. +var logMu sync.Mutex + +// log is the default logger. +var log atomic.Pointer[BasicLogger] + +// Log retrieves the global logger. +func Log() *BasicLogger { + return log.Load() +} + +// SetTarget sets the log target. +// +// This is not thread safe and shouldn't be called concurrently with any +// logging calls. +// +// SetTarget should be called before any instances of log.Log() to avoid race conditions +func SetTarget(target Emitter) { + logMu.Lock() + defer logMu.Unlock() + oldLog := Log() + log.Store(&BasicLogger{Level: oldLog.Level, Emitter: target}) +} + +// SetLevel sets the log level. +func SetLevel(newLevel Level) { + Log().SetLevel(newLevel) +} + +// Debugf logs to the global logger. +func Debugf(format string, v ...any) { + Log().DebugfAtDepth(1, format, v...) +} + +// Infof logs to the global logger. +func Infof(format string, v ...any) { + Log().InfofAtDepth(1, format, v...) +} + +// Warningf logs to the global logger. +func Warningf(format string, v ...any) { + Log().WarningfAtDepth(1, format, v...) +} + +// DebugfAtDepth logs to the global logger. +func DebugfAtDepth(depth int, format string, v ...any) { + Log().DebugfAtDepth(1+depth, format, v...) +} + +// InfofAtDepth logs to the global logger. +func InfofAtDepth(depth int, format string, v ...any) { + Log().InfofAtDepth(1+depth, format, v...) +} + +// WarningfAtDepth logs to the global logger. +func WarningfAtDepth(depth int, format string, v ...any) { + Log().WarningfAtDepth(1+depth, format, v...) +} + +// defaultStackSize is the default buffer size to allocate for stack traces. +const defaultStackSize = 1 << 16 // 64KB + +// maxStackSize is the maximum buffer size to allocate for stack traces. +const maxStackSize = 1 << 26 // 64MB + +// Stacks returns goroutine stacks, like panic. +func Stacks(all bool) []byte { + var trace []byte + for s := defaultStackSize; s <= maxStackSize; s *= 4 { + trace = make([]byte, s) + nbytes := runtime.Stack(trace, all) + if nbytes == s { + continue + } + return trace[:nbytes] + } + trace = append(trace, []byte("\n\n...")...) + return trace +} + +// stackRegexp matches one level within a stack trace. +var stackRegexp = regexp.MustCompile("(?m)^\\S+\\(.*\\)$\\r?\\n^\\t\\S+:\\d+.*$\\r?\\n") + +// LocalStack returns the local goroutine stack, excluding the top N entries. +// LocalStack's own entry is excluded by default and does not need to be counted in excludeTopN. +func LocalStack(excludeTopN int) []byte { + replaceNext := excludeTopN + 1 + return stackRegexp.ReplaceAllFunc(Stacks(false), func(s []byte) []byte { + if replaceNext > 0 { + replaceNext-- + return nil + } + return s + }) +} + +// Traceback logs the given message and dumps a stacktrace of the current +// goroutine. +// +// This will be print a traceback, tb, as Warningf(format+":\n%s", v..., tb). +func Traceback(format string, v ...any) { + v = append(v, Stacks(false)) + Warningf(format+":\n%s", v...) +} + +// TracebackAll logs the given message and dumps a stacktrace of all goroutines. +// +// This will be print a traceback, tb, as Warningf(format+":\n%s", v..., tb). +func TracebackAll(format string, v ...any) { + v = append(v, Stacks(true)) + Warningf(format+":\n%s", v...) +} + +// IsLogging returns whether the global logger is logging. +func IsLogging(level Level) bool { + return Log().IsLogging(level) +} + +// CopyStandardLogTo redirects the stdlib log package global output to the global +// logger for the specified level. +func CopyStandardLogTo(l Level) error { + var f func(string, ...any) + + switch l { + case Debug: + f = Debugf + case Info: + f = Infof + case Warning: + f = Warningf + default: + return fmt.Errorf("unknown log level %v", l) + } + + stdlog.SetOutput(linewriter.NewWriter(func(p []byte) { + // We must not retain p, but log formatting is not required to + // be synchronous (though the in-package implementations are), + // so we must make a copy. + b := make([]byte, len(p)) + copy(b, p) + + f("%s", b) + })) + + return nil +} + +func init() { + // Store the initial value for the log. + log.Store(&BasicLogger{Level: Info, Emitter: GoogleEmitter{&Writer{Next: os.Stderr}}}) + + warnedSet = make(map[string]struct{}) +} diff --git a/pkg/log/rate_limited.go b/pkg/log/rate_limited.go new file mode 100644 index 0000000..e274238 --- /dev/null +++ b/pkg/log/rate_limited.go @@ -0,0 +1,63 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "time" + + "golang.org/x/time/rate" +) + +type rateLimitedLogger struct { + logger Logger + limit *rate.Limiter +} + +func (rl *rateLimitedLogger) Debugf(format string, v ...any) { + if rl.limit.Allow() { + rl.logger.Debugf(format, v...) + } +} + +func (rl *rateLimitedLogger) Infof(format string, v ...any) { + if rl.limit.Allow() { + rl.logger.Infof(format, v...) + } +} + +func (rl *rateLimitedLogger) Warningf(format string, v ...any) { + if rl.limit.Allow() { + rl.logger.Warningf(format, v...) + } +} + +func (rl *rateLimitedLogger) IsLogging(level Level) bool { + return rl.logger.IsLogging(level) +} + +// BasicRateLimitedLogger returns a Logger that logs to the global logger no +// more than once per the provided duration. +func BasicRateLimitedLogger(every time.Duration) Logger { + return RateLimitedLogger(Log(), every) +} + +// RateLimitedLogger returns a Logger that logs to the provided logger no more +// than once per the provided duration. +func RateLimitedLogger(logger Logger, every time.Duration) Logger { + return &rateLimitedLogger{ + logger: logger, + limit: rate.NewLimiter(rate.Every(every), 1), + } +} diff --git a/pkg/marshal/marshal.go b/pkg/marshal/marshal.go new file mode 100644 index 0000000..8574a91 --- /dev/null +++ b/pkg/marshal/marshal.go @@ -0,0 +1,226 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package marshal defines the Marshallable interface for +// serialize/deserializing go data structures to/from memory, according to the +// Linux ABI. +// +// Implementations of this interface are typically automatically generated by +// tools/go_marshal. See the go_marshal README for details. +package marshal + +import ( + "io" + + "github.com/sagernet/gvisor/pkg/hostarch" +) + +// CopyContext defines the memory operations required to marshal to and from +// user memory. Typically, kernel.Task is used to provide implementations for +// these operations. +type CopyContext interface { + // CopyScratchBuffer provides a task goroutine-local scratch buffer. See + // kernel.CopyScratchBuffer. + CopyScratchBuffer(size int) []byte + + // CopyOutBytes writes the contents of b to the task's memory. See + // kernel.CopyOutBytes. + CopyOutBytes(addr hostarch.Addr, b []byte) (int, error) + + // CopyInBytes reads the contents of the task's memory to b. See + // kernel.CopyInBytes. + CopyInBytes(addr hostarch.Addr, b []byte) (int, error) +} + +// Marshallable represents operations on a type that can be marshalled to and +// from memory. +// +// go-marshal automatically generates implementations for this interface for +// types marked as '+marshal'. +type Marshallable interface { + io.WriterTo + + // SizeBytes is the size of the memory representation of a type in + // marshalled form. + // + // SizeBytes must handle a nil receiver. Practically, this means SizeBytes + // cannot deference any fields on the object implementing it (but will + // likely make use of the type of these fields). + SizeBytes() int + + // MarshalBytes serializes a copy of a type to dst and returns the remaining + // buffer. + // Precondition: dst must be at least SizeBytes() in length. + MarshalBytes(dst []byte) []byte + + // UnmarshalBytes deserializes a type from src and returns the remaining + // buffer. + // Precondition: src must be at least SizeBytes() in length. + UnmarshalBytes(src []byte) []byte + + // Packed returns true if the marshalled size of the type is the same as the + // size it occupies in memory. This happens when the type has no fields + // starting at unaligned addresses (should always be true by default for ABI + // structs, verified by automatically generated tests when using + // go_marshal), and has no fields marked `marshal:"unaligned"`. + // + // Packed must return the same result for all possible values of the type + // implementing it. Violating this constraint implies the type doesn't have + // a static memory layout, and will lead to memory corruption. + // Go-marshal-generated code reuses the result of Packed for multiple values + // of the same type. + Packed() bool + + // MarshalUnsafe serializes a type by bulk copying its in-memory + // representation to the dst buffer. This is only safe to do when the type + // has no implicit padding, see Marshallable.Packed. When Packed would + // return false, MarshalUnsafe should fall back to the safer but slower + // MarshalBytes. + // Precondition: dst must be at least SizeBytes() in length. + MarshalUnsafe(dst []byte) []byte + + // UnmarshalUnsafe deserializes a type by directly copying to the underlying + // memory allocated for the object by the runtime. + // + // This allows much faster unmarshalling of types which have no implicit + // padding, see Marshallable.Packed. When Packed would return false, + // UnmarshalUnsafe should fall back to the safer but slower unmarshal + // mechanism implemented in UnmarshalBytes. + // Precondition: src must be at least SizeBytes() in length. + UnmarshalUnsafe(src []byte) []byte + + // CopyIn deserializes a Marshallable type from a task's memory. This may + // only be called from a task goroutine. This is more efficient than calling + // UnmarshalUnsafe on Marshallable.Packed types, as the type being + // marshalled does not escape. The implementation should avoid creating + // extra copies in memory by directly deserializing to the object's + // underlying memory. + // + // If the copy-in from the task memory is only partially successful, CopyIn + // should still attempt to deserialize as much data as possible. See comment + // for UnmarshalBytes. + CopyIn(cc CopyContext, addr hostarch.Addr) (int, error) + + // CopyInN is like CopyIn, but explicitly requests a partial + // copy-in. Note that this may yield unexpected results for non-packed + // types and the caller may only want to allow this for packed types. See + // comment on UnmarshalBytes. + // + // The limit must be less than or equal to SizeBytes(). + CopyInN(cc CopyContext, addr hostarch.Addr, limit int) (int, error) + + // CopyOut serializes a Marshallable type to a task's memory. This may only + // be called from a task goroutine. This is more efficient than calling + // MarshalUnsafe on Marshallable.Packed types, as the type being serialized + // does not escape. The implementation should avoid creating extra copies in + // memory by directly serializing from the object's underlying memory. + // + // The copy-out to the task memory may be partially successful, in which + // case CopyOut returns how much data was serialized. See comment for + // MarshalBytes for implications. + CopyOut(cc CopyContext, addr hostarch.Addr) (int, error) + + // CopyOutN is like CopyOut, but explicitly requests a partial + // copy-out. Note that this may yield unexpected results for non-packed + // types and the caller may only want to allow this for packed types. See + // comment on MarshalBytes. + // + // The limit must be less than or equal to SizeBytes(). + CopyOutN(cc CopyContext, addr hostarch.Addr, limit int) (int, error) +} + +// CheckedMarshallable represents operations on a type that can be marshalled +// to and from memory and additionally does bound checking. +type CheckedMarshallable interface { + // CheckedMarshal is the same as Marshallable.MarshalUnsafe but without the + // precondition that dst must at least have some appropriate length. Similar + // to Marshallable.MarshalBytes, it returns a shifted slice according to how + // much data is consumed. Additionally it returns a bool indicating whether + // marshalling was successful. Unsuccessful marshalling doesn't consume any + // data. + CheckedMarshal(dst []byte) ([]byte, bool) + + // CheckedUnmarshal is the same as Marshallable.UmarshalUnsafe but without + // the precondition that src must at least have some appropriate length. + // Similar to Marshallable.UnmarshalBytes, it returns a shifted slice + // according to how much data is consumed. Additionally it returns a bool + // indicating whether marshalling was successful. Unsuccessful marshalling + // doesn't consume any data. + CheckedUnmarshal(src []byte) ([]byte, bool) +} + +// go-marshal generates additional functions for a type based on additional +// clauses to the +marshal directive. They are documented below. +// +// Slice API +// ========= +// +// Adding a "slice" clause to the +marshal directive for structs or newtypes on +// primitives like this: +// +// // +marshal slice:FooSlice +// type Foo struct { ... } +// +// Generates four additional functions for marshalling slices of Foos like this: +// +// // MarshalUnsafeFooSlice is like Foo.MarshalUnsafe, buf for a []Foo. It +// // might be more efficient that repeatedly calling Foo.MarshalUnsafe +// // over a []Foo in a loop if the type is Packed. +// // Preconditions: dst must be at least len(src)*Foo.SizeBytes() in length. +// func MarshalUnsafeFooSlice(src []Foo, dst []byte) []byte { ... } +// +// // UnmarshalUnsafeFooSlice is like Foo.UnmarshalUnsafe, buf for a []Foo. It +// // might be more efficient that repeatedly calling Foo.UnmarshalUnsafe +// // over a []Foo in a loop if the type is Packed. +// // Preconditions: src must be at least len(dst)*Foo.SizeBytes() in length. +// func UnmarshalUnsafeFooSlice(dst []Foo, src []byte) []byte { ... } +// +// // CopyFooSliceIn copies in a slice of Foo objects from the task's memory. +// func CopyFooSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []Foo) (int, error) { ... } +// +// // CopyFooSliceIn copies out a slice of Foo objects to the task's memory. +// func CopyFooSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []Foo) (int, error) { ... } +// +// The name of the functions are of the format "Copy%sIn" and "Copy%sOut", where +// %s is the first argument to the slice clause. This directive is not supported +// for newtypes on arrays. +// +// Note: Partial copies are not supported for Slice API UnmarshalUnsafe and +// MarshalUnsafe. +// +// The slice clause also takes an optional second argument, which must be the +// value "inner": +// +// // +marshal slice:Int32Slice:inner +// type Int32 int32 +// +// This is only valid on newtypes on primitives, and causes the generated +// functions to accept slices of the inner type instead: +// +// func CopyInt32SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []int32) (int, error) { ... } +// +// Without "inner", they would instead be: +// +// func CopyInt32SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []Int32) (int, error) { ... } +// +// This may help avoid a cast depending on how the generated functions are used. +// +// Bound Checking +// ============== +// +// Some users might want to do bound checking on marshal and unmarshal. This is +// is useful when the user does not control the buffer size. To prevent +// repeated bound checking code around Marshallable, users can add a +// "boundCheck" clause to the +marshal directive. go_marshal will generate the +// CheckedMarshallable interface methods on the type. diff --git a/pkg/marshal/marshal_state_autogen.go b/pkg/marshal/marshal_state_autogen.go new file mode 100644 index 0000000..a0a9531 --- /dev/null +++ b/pkg/marshal/marshal_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package marshal diff --git a/pkg/marshal/primitive/primitive.go b/pkg/marshal/primitive/primitive.go new file mode 100644 index 0000000..0ec5426 --- /dev/null +++ b/pkg/marshal/primitive/primitive.go @@ -0,0 +1,405 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package primitive defines marshal.Marshallable implementations for primitive +// types. +package primitive + +import ( + "io" + + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Int8 is a marshal.Marshallable implementation for int8. +// +// +marshal boundCheck slice:Int8Slice:inner +type Int8 int8 + +// Uint8 is a marshal.Marshallable implementation for uint8. +// +// +marshal boundCheck slice:Uint8Slice:inner +type Uint8 uint8 + +// Int16 is a marshal.Marshallable implementation for int16. +// +// +marshal boundCheck slice:Int16Slice:inner +type Int16 int16 + +// Uint16 is a marshal.Marshallable implementation for uint16. +// +// +marshal boundCheck slice:Uint16Slice:inner +type Uint16 uint16 + +// Int32 is a marshal.Marshallable implementation for int32. +// +// +marshal boundCheck slice:Int32Slice:inner +type Int32 int32 + +// Uint32 is a marshal.Marshallable implementation for uint32. +// +// +marshal boundCheck slice:Uint32Slice:inner +type Uint32 uint32 + +// Int64 is a marshal.Marshallable implementation for int64. +// +// +marshal boundCheck slice:Int64Slice:inner +type Int64 int64 + +// Uint64 is a marshal.Marshallable implementation for uint64. +// +// +marshal boundCheck slice:Uint64Slice:inner +type Uint64 uint64 + +// ByteSlice is a marshal.Marshallable implementation for []byte. +// This is a convenience wrapper around a dynamically sized type, and can't be +// embedded in other marshallable types because it breaks assumptions made by +// go-marshal internals. It violates the "no dynamically-sized types" +// constraint of the go-marshal library. +type ByteSlice []byte + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (b *ByteSlice) SizeBytes() int { + return len(*b) +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (b *ByteSlice) MarshalBytes(dst []byte) []byte { + return dst[copy(dst, *b):] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (b *ByteSlice) UnmarshalBytes(src []byte) []byte { + return src[copy(*b, src):] +} + +// Packed implements marshal.Marshallable.Packed. +func (b *ByteSlice) Packed() bool { + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (b *ByteSlice) MarshalUnsafe(dst []byte) []byte { + return b.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (b *ByteSlice) UnmarshalUnsafe(src []byte) []byte { + return b.UnmarshalBytes(src) +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (b *ByteSlice) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return cc.CopyInBytes(addr, *b) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (b *ByteSlice) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + return cc.CopyInBytes(addr, (*b)[:limit]) +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (b *ByteSlice) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return cc.CopyOutBytes(addr, *b) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (b *ByteSlice) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + return cc.CopyOutBytes(addr, (*b)[:limit]) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (b *ByteSlice) WriteTo(w io.Writer) (int64, error) { + n, err := w.Write(*b) + return int64(n), err +} + +var _ marshal.Marshallable = (*ByteSlice)(nil) + +// The following set of functions are convenient shorthands for wrapping a +// built-in type in a marshallable primitive type. For example: +// +// func useMarshallable(m marshal.Marshallable) { ... } +// +// // Compare: +// +// buf = []byte{...} +// // useMarshallable(&primitive.ByteSlice(buf)) // Not allowed, can't address temp value. +// bufP := primitive.ByteSlice(buf) +// useMarshallable(&bufP) +// +// // Vs: +// +// useMarshallable(AsByteSlice(buf)) +// +// Note that the argument to these function escapes, so avoid using them on very +// hot code paths. But generally if a function accepts an interface as an +// argument, the argument escapes anyways. + +// AllocateInt8 returns x as a marshallable. +func AllocateInt8(x int8) marshal.Marshallable { + p := Int8(x) + return &p +} + +// AllocateUint8 returns x as a marshallable. +func AllocateUint8(x uint8) marshal.Marshallable { + p := Uint8(x) + return &p +} + +// AllocateInt16 returns x as a marshallable. +func AllocateInt16(x int16) marshal.Marshallable { + p := Int16(x) + return &p +} + +// AllocateUint16 returns x as a marshallable. +func AllocateUint16(x uint16) marshal.Marshallable { + p := Uint16(x) + return &p +} + +// AllocateInt32 returns x as a marshallable. +func AllocateInt32(x int32) marshal.Marshallable { + p := Int32(x) + return &p +} + +// AllocateUint32 returns x as a marshallable. +func AllocateUint32(x uint32) marshal.Marshallable { + p := Uint32(x) + return &p +} + +// AllocateInt64 returns x as a marshallable. +func AllocateInt64(x int64) marshal.Marshallable { + p := Int64(x) + return &p +} + +// AllocateUint64 returns x as a marshallable. +func AllocateUint64(x uint64) marshal.Marshallable { + p := Uint64(x) + return &p +} + +// AsByteSlice returns b as a marshallable. Note that this allocates a new slice +// header, but does not copy the slice contents. +func AsByteSlice(b []byte) marshal.Marshallable { + bs := ByteSlice(b) + return &bs +} + +// Below, we define some convenience functions for marshalling primitive types +// using the newtypes above, without requiring superfluous casts. + +// 8-bit integers + +// CopyInt8In is a convenient wrapper for copying in an int8 from the task's +// memory. +func CopyInt8In(cc marshal.CopyContext, addr hostarch.Addr, dst *int8) (int, error) { + var buf Int8 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = int8(buf) + return n, nil +} + +// CopyInt8Out is a convenient wrapper for copying out an int8 to the task's +// memory. +func CopyInt8Out(cc marshal.CopyContext, addr hostarch.Addr, src int8) (int, error) { + srcP := Int8(src) + return srcP.CopyOut(cc, addr) +} + +// CopyUint8In is a convenient wrapper for copying in a uint8 from the task's +// memory. +func CopyUint8In(cc marshal.CopyContext, addr hostarch.Addr, dst *uint8) (int, error) { + var buf Uint8 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = uint8(buf) + return n, nil +} + +// CopyUint8Out is a convenient wrapper for copying out a uint8 to the task's +// memory. +func CopyUint8Out(cc marshal.CopyContext, addr hostarch.Addr, src uint8) (int, error) { + srcP := Uint8(src) + return srcP.CopyOut(cc, addr) +} + +// 16-bit integers + +// CopyInt16In is a convenient wrapper for copying in an int16 from the task's +// memory. +func CopyInt16In(cc marshal.CopyContext, addr hostarch.Addr, dst *int16) (int, error) { + var buf Int16 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = int16(buf) + return n, nil +} + +// CopyInt16Out is a convenient wrapper for copying out an int16 to the task's +// memory. +func CopyInt16Out(cc marshal.CopyContext, addr hostarch.Addr, src int16) (int, error) { + srcP := Int16(src) + return srcP.CopyOut(cc, addr) +} + +// CopyUint16In is a convenient wrapper for copying in a uint16 from the task's +// memory. +func CopyUint16In(cc marshal.CopyContext, addr hostarch.Addr, dst *uint16) (int, error) { + var buf Uint16 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = uint16(buf) + return n, nil +} + +// CopyUint16Out is a convenient wrapper for copying out a uint16 to the task's +// memory. +func CopyUint16Out(cc marshal.CopyContext, addr hostarch.Addr, src uint16) (int, error) { + srcP := Uint16(src) + return srcP.CopyOut(cc, addr) +} + +// 32-bit integers + +// CopyInt32In is a convenient wrapper for copying in an int32 from the task's +// memory. +func CopyInt32In(cc marshal.CopyContext, addr hostarch.Addr, dst *int32) (int, error) { + var buf Int32 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = int32(buf) + return n, nil +} + +// CopyInt32Out is a convenient wrapper for copying out an int32 to the task's +// memory. +func CopyInt32Out(cc marshal.CopyContext, addr hostarch.Addr, src int32) (int, error) { + srcP := Int32(src) + return srcP.CopyOut(cc, addr) +} + +// CopyUint32In is a convenient wrapper for copying in a uint32 from the task's +// memory. +func CopyUint32In(cc marshal.CopyContext, addr hostarch.Addr, dst *uint32) (int, error) { + var buf Uint32 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = uint32(buf) + return n, nil +} + +// CopyUint32Out is a convenient wrapper for copying out a uint32 to the task's +// memory. +func CopyUint32Out(cc marshal.CopyContext, addr hostarch.Addr, src uint32) (int, error) { + srcP := Uint32(src) + return srcP.CopyOut(cc, addr) +} + +// 64-bit integers + +// CopyInt64In is a convenient wrapper for copying in an int64 from the task's +// memory. +func CopyInt64In(cc marshal.CopyContext, addr hostarch.Addr, dst *int64) (int, error) { + var buf Int64 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = int64(buf) + return n, nil +} + +// CopyInt64Out is a convenient wrapper for copying out an int64 to the task's +// memory. +func CopyInt64Out(cc marshal.CopyContext, addr hostarch.Addr, src int64) (int, error) { + srcP := Int64(src) + return srcP.CopyOut(cc, addr) +} + +// CopyUint64In is a convenient wrapper for copying in a uint64 from the task's +// memory. +func CopyUint64In(cc marshal.CopyContext, addr hostarch.Addr, dst *uint64) (int, error) { + var buf Uint64 + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = uint64(buf) + return n, nil +} + +// CopyUint64Out is a convenient wrapper for copying out a uint64 to the task's +// memory. +func CopyUint64Out(cc marshal.CopyContext, addr hostarch.Addr, src uint64) (int, error) { + srcP := Uint64(src) + return srcP.CopyOut(cc, addr) +} + +// CopyByteSliceIn is a convenient wrapper for copying in a []byte from the +// task's memory. +func CopyByteSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst *[]byte) (int, error) { + var buf ByteSlice + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = []byte(buf) + return n, nil +} + +// CopyByteSliceOut is a convenient wrapper for copying out a []byte to the +// task's memory. +func CopyByteSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []byte) (int, error) { + srcP := ByteSlice(src) + return srcP.CopyOut(cc, addr) +} + +// CopyStringIn is a convenient wrapper for copying in a string from the +// task's memory. +func CopyStringIn(cc marshal.CopyContext, addr hostarch.Addr, dst *string) (int, error) { + var buf ByteSlice + n, err := buf.CopyIn(cc, addr) + if err != nil { + return n, err + } + *dst = string(buf) + return n, nil +} + +// CopyStringOut is a convenient wrapper for copying out a string to the task's +// memory. +func CopyStringOut(cc marshal.CopyContext, addr hostarch.Addr, src string) (int, error) { + srcP := ByteSlice(src) + return srcP.CopyOut(cc, addr) +} diff --git a/pkg/marshal/primitive/primitive_abi_autogen_unsafe.go b/pkg/marshal/primitive/primitive_abi_autogen_unsafe.go new file mode 100644 index 0000000..12140fe --- /dev/null +++ b/pkg/marshal/primitive/primitive_abi_autogen_unsafe.go @@ -0,0 +1,1586 @@ +// Automatically generated marshal implementation. See tools/go_marshal. + +package primitive + +import ( + "io" + "reflect" + "runtime" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/hostarch" + "github.com/sagernet/gvisor/pkg/marshal" +) + +// Marshallable types used by this file. +var ( + _ marshal.Marshallable = (*Int16)(nil) + _ marshal.Marshallable = (*Int32)(nil) + _ marshal.Marshallable = (*Int64)(nil) + _ marshal.Marshallable = (*Int8)(nil) + _ marshal.Marshallable = (*Uint16)(nil) + _ marshal.Marshallable = (*Uint32)(nil) + _ marshal.Marshallable = (*Uint64)(nil) + _ marshal.Marshallable = (*Uint8)(nil) +) + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *Int16) SizeBytes() int { + return 2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Int16) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(*i)) + return dst[2:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Int16) UnmarshalBytes(src []byte) []byte { + *i = Int16(int16(hostarch.ByteOrder.Uint16(src[:2]))) + return src[2:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Int16) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Int16) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Int16) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Int16) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Int16) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Int16) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Int16) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Int16) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (i *Int16) CheckedMarshal(dst []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (i *Int16) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyInt16SliceIn copies in a slice of int16 objects from the task's memory. +func CopyInt16SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []int16) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Int16)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyInt16SliceOut copies a slice of int16 objects to the task's memory. +func CopyInt16SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []int16) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Int16)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeInt16Slice is like Int16.MarshalUnsafe, but for a []Int16. +func MarshalUnsafeInt16Slice(src []Int16, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Int16)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeInt16Slice is like Int16.UnmarshalUnsafe, but for a []Int16. +func UnmarshalUnsafeInt16Slice(dst []Int16, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Int16)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *Int32) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Int32) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*i)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Int32) UnmarshalBytes(src []byte) []byte { + *i = Int32(int32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Int32) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Int32) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Int32) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Int32) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Int32) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Int32) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Int32) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Int32) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (i *Int32) CheckedMarshal(dst []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (i *Int32) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyInt32SliceIn copies in a slice of int32 objects from the task's memory. +func CopyInt32SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []int32) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Int32)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyInt32SliceOut copies a slice of int32 objects to the task's memory. +func CopyInt32SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []int32) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Int32)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeInt32Slice is like Int32.MarshalUnsafe, but for a []Int32. +func MarshalUnsafeInt32Slice(src []Int32, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Int32)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeInt32Slice is like Int32.UnmarshalUnsafe, but for a []Int32. +func UnmarshalUnsafeInt32Slice(dst []Int32, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Int32)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *Int64) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Int64) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*i)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Int64) UnmarshalBytes(src []byte) []byte { + *i = Int64(int64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Int64) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Int64) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Int64) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Int64) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Int64) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Int64) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Int64) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Int64) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (i *Int64) CheckedMarshal(dst []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (i *Int64) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyInt64SliceIn copies in a slice of int64 objects from the task's memory. +func CopyInt64SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []int64) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Int64)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyInt64SliceOut copies a slice of int64 objects to the task's memory. +func CopyInt64SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []int64) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Int64)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeInt64Slice is like Int64.MarshalUnsafe, but for a []Int64. +func MarshalUnsafeInt64Slice(src []Int64, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Int64)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeInt64Slice is like Int64.UnmarshalUnsafe, but for a []Int64. +func UnmarshalUnsafeInt64Slice(dst []Int64, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Int64)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (i *Int8) SizeBytes() int { + return 1 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *Int8) MarshalBytes(dst []byte) []byte { + dst[0] = byte(*i) + return dst[1:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *Int8) UnmarshalBytes(src []byte) []byte { + *i = Int8(int8(src[0])) + return src[1:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *Int8) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *Int8) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *Int8) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *Int8) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *Int8) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *Int8) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *Int8) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *Int8) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (i *Int8) CheckedMarshal(dst []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (i *Int8) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := i.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyInt8SliceIn copies in a slice of int8 objects from the task's memory. +func CopyInt8SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []int8) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Int8)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyInt8SliceOut copies a slice of int8 objects to the task's memory. +func CopyInt8SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []int8) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Int8)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeInt8Slice is like Int8.MarshalUnsafe, but for a []Int8. +func MarshalUnsafeInt8Slice(src []Int8, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Int8)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeInt8Slice is like Int8.UnmarshalUnsafe, but for a []Int8. +func UnmarshalUnsafeInt8Slice(dst []Int8, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Int8)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (u *Uint16) SizeBytes() int { + return 2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *Uint16) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint16(dst[:2], uint16(*u)) + return dst[2:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *Uint16) UnmarshalBytes(src []byte) []byte { + *u = Uint16(uint16(hostarch.ByteOrder.Uint16(src[:2]))) + return src[2:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *Uint16) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *Uint16) MarshalUnsafe(dst []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *Uint16) UnmarshalUnsafe(src []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *Uint16) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *Uint16) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *Uint16) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *Uint16) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *Uint16) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (u *Uint16) CheckedMarshal(dst []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (u *Uint16) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyUint16SliceIn copies in a slice of uint16 objects from the task's memory. +func CopyUint16SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []uint16) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Uint16)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyUint16SliceOut copies a slice of uint16 objects to the task's memory. +func CopyUint16SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []uint16) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Uint16)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeUint16Slice is like Uint16.MarshalUnsafe, but for a []Uint16. +func MarshalUnsafeUint16Slice(src []Uint16, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Uint16)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeUint16Slice is like Uint16.UnmarshalUnsafe, but for a []Uint16. +func UnmarshalUnsafeUint16Slice(dst []Uint16, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Uint16)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (u *Uint32) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *Uint32) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(*u)) + return dst[4:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *Uint32) UnmarshalBytes(src []byte) []byte { + *u = Uint32(uint32(hostarch.ByteOrder.Uint32(src[:4]))) + return src[4:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *Uint32) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *Uint32) MarshalUnsafe(dst []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *Uint32) UnmarshalUnsafe(src []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *Uint32) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *Uint32) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *Uint32) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *Uint32) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *Uint32) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (u *Uint32) CheckedMarshal(dst []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (u *Uint32) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyUint32SliceIn copies in a slice of uint32 objects from the task's memory. +func CopyUint32SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []uint32) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Uint32)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyUint32SliceOut copies a slice of uint32 objects to the task's memory. +func CopyUint32SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []uint32) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Uint32)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeUint32Slice is like Uint32.MarshalUnsafe, but for a []Uint32. +func MarshalUnsafeUint32Slice(src []Uint32, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Uint32)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeUint32Slice is like Uint32.UnmarshalUnsafe, but for a []Uint32. +func UnmarshalUnsafeUint32Slice(dst []Uint32, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Uint32)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (u *Uint64) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *Uint64) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(*u)) + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *Uint64) UnmarshalBytes(src []byte) []byte { + *u = Uint64(uint64(hostarch.ByteOrder.Uint64(src[:8]))) + return src[8:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *Uint64) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *Uint64) MarshalUnsafe(dst []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *Uint64) UnmarshalUnsafe(src []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *Uint64) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *Uint64) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *Uint64) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *Uint64) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *Uint64) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (u *Uint64) CheckedMarshal(dst []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (u *Uint64) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyUint64SliceIn copies in a slice of uint64 objects from the task's memory. +func CopyUint64SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []uint64) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Uint64)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyUint64SliceOut copies a slice of uint64 objects to the task's memory. +func CopyUint64SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []uint64) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Uint64)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeUint64Slice is like Uint64.MarshalUnsafe, but for a []Uint64. +func MarshalUnsafeUint64Slice(src []Uint64, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Uint64)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeUint64Slice is like Uint64.UnmarshalUnsafe, but for a []Uint64. +func UnmarshalUnsafeUint64Slice(dst []Uint64, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Uint64)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +// +//go:nosplit +func (u *Uint8) SizeBytes() int { + return 1 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (u *Uint8) MarshalBytes(dst []byte) []byte { + dst[0] = byte(*u) + return dst[1:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (u *Uint8) UnmarshalBytes(src []byte) []byte { + *u = Uint8(uint8(src[0])) + return src[1:] +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (u *Uint8) Packed() bool { + // Scalar newtypes are always packed. + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (u *Uint8) MarshalUnsafe(dst []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (u *Uint8) UnmarshalUnsafe(src []byte) []byte { + size := u.SizeBytes() + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (u *Uint8) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (u *Uint8) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyOutN(cc, addr, u.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (u *Uint8) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (u *Uint8) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return u.CopyInN(cc, addr, u.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (u *Uint8) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(u))) + hdr.Len = u.SizeBytes() + hdr.Cap = u.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that u + // must live until the use above. + runtime.KeepAlive(u) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CheckedMarshal implements marshal.CheckedMarshallable.CheckedMarshal. +func (u *Uint8) CheckedMarshal(dst []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(dst) { + return dst, false + } + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(u), uintptr(size)) + return dst[size:], true +} + +// CheckedUnmarshal implements marshal.CheckedMarshallable.CheckedUnmarshal. +func (u *Uint8) CheckedUnmarshal(src []byte) ([]byte, bool) { + size := u.SizeBytes() + if size > len(src) { + return src, false + } + gohacks.Memmove(unsafe.Pointer(u), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:], true +} + +// CopyUint8SliceIn copies in a slice of uint8 objects from the task's memory. +func CopyUint8SliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []uint8) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Uint8)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyUint8SliceOut copies a slice of uint8 objects to the task's memory. +func CopyUint8SliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []uint8) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Uint8)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeUint8Slice is like Uint8.MarshalUnsafe, but for a []Uint8. +func MarshalUnsafeUint8Slice(src []Uint8, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + size := (*Uint8)(nil).SizeBytes() + + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeUint8Slice is like Uint8.UnmarshalUnsafe, but for a []Uint8. +func UnmarshalUnsafeUint8Slice(dst []Uint8, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + size := (*Uint8)(nil).SizeBytes() + + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} diff --git a/pkg/marshal/primitive/primitive_state_autogen.go b/pkg/marshal/primitive/primitive_state_autogen.go new file mode 100644 index 0000000..f9db3a9 --- /dev/null +++ b/pkg/marshal/primitive/primitive_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package primitive diff --git a/pkg/marshal/util.go b/pkg/marshal/util.go new file mode 100644 index 0000000..959d692 --- /dev/null +++ b/pkg/marshal/util.go @@ -0,0 +1,44 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshal + +// Marshal returns the serialized contents of m in a newly allocated +// byte slice. +func Marshal(m Marshallable) []byte { + buf := make([]byte, m.SizeBytes()) + m.MarshalUnsafe(buf) + return buf +} + +// MarshalAll returns the serialized contents of all ms in a newly allocated +// byte slice. +func MarshalAll(ms []Marshallable) []byte { + buf := make([]byte, TotalSize(ms)) + var written int + for _, m := range ms { + m.MarshalUnsafe(buf[written:]) + written += m.SizeBytes() + } + return buf +} + +// TotalSize returns the total size of all ms. +func TotalSize(ms []Marshallable) int { + var size int + for _, m := range ms { + size += m.SizeBytes() + } + return size +} diff --git a/pkg/memutil/memfd_linux_unsafe.go b/pkg/memutil/memfd_linux_unsafe.go new file mode 100644 index 0000000..2179c92 --- /dev/null +++ b/pkg/memutil/memfd_linux_unsafe.go @@ -0,0 +1,41 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package memutil + +import ( + "fmt" + "unsafe" + + "golang.org/x/sys/unix" +) + +// CreateMemFD creates a memfd file and returns the fd. +func CreateMemFD(name string, flags int) (int, error) { + p, err := unix.BytePtrFromString(name) + if err != nil { + return -1, err + } + fd, _, e := unix.Syscall(unix.SYS_MEMFD_CREATE, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) + if e != 0 { + if e == unix.ENOSYS { + return -1, fmt.Errorf("memfd_create(2) is not implemented. Check that you have Linux 3.17 or higher") + } + return -1, e + } + return int(fd), nil +} diff --git a/pkg/memutil/memutil_linux_unsafe_state_autogen.go b/pkg/memutil/memutil_linux_unsafe_state_autogen.go new file mode 100644 index 0000000..df954e7 --- /dev/null +++ b/pkg/memutil/memutil_linux_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package memutil diff --git a/pkg/memutil/memutil_state_autogen.go b/pkg/memutil/memutil_state_autogen.go new file mode 100644 index 0000000..0ab6ca5 --- /dev/null +++ b/pkg/memutil/memutil_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build !false +// +build !false + +package memutil diff --git a/pkg/memutil/memutil_unsafe.go b/pkg/memutil/memutil_unsafe.go new file mode 100644 index 0000000..3c5ebd7 --- /dev/null +++ b/pkg/memutil/memutil_unsafe.go @@ -0,0 +1,44 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memutil provides utilities for working with shared memory files. +package memutil + +import ( + "reflect" + "unsafe" + + "golang.org/x/sys/unix" +) + +// MapSlice is like MapFile, but returns a slice instead of a uintptr. +func MapSlice(addr, size, prot, flags, fd, offset uintptr) ([]byte, error) { + addr, err := MapFile(addr, size, prot, flags, fd, offset) + if err != nil { + return nil, err + } + var slice []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + hdr.Data = addr + hdr.Len = int(size) + hdr.Cap = int(size) + return slice, nil +} + +// UnmapSlice unmaps a mapping returned by MapSlice. +func UnmapSlice(slice []byte) error { + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + _, _, err := unix.RawSyscall6(unix.SYS_MUNMAP, uintptr(unsafe.Pointer(hdr.Data)), uintptr(hdr.Cap), 0, 0, 0, 0) + return err +} diff --git a/pkg/memutil/memutil_unsafe_state_autogen.go b/pkg/memutil/memutil_unsafe_state_autogen.go new file mode 100644 index 0000000..8984855 --- /dev/null +++ b/pkg/memutil/memutil_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package memutil diff --git a/pkg/memutil/mmap.go b/pkg/memutil/mmap.go new file mode 100644 index 0000000..b41e1e0 --- /dev/null +++ b/pkg/memutil/mmap.go @@ -0,0 +1,32 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !false +// +build !false + +package memutil + +import ( + "golang.org/x/sys/unix" +) + +// MapFile returns a memory mapping configured by the given options as per +// mmap(2). +func MapFile(addr, size, prot, flags, fd, offset uintptr) (uintptr, error) { + m, _, e := unix.RawSyscall6(unix.SYS_MMAP, addr, size, prot, flags, fd, offset) + if e != 0 { + return 0, e + } + return m, nil +} diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go new file mode 100644 index 0000000..069fbdf --- /dev/null +++ b/pkg/pool/pool.go @@ -0,0 +1,67 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pool provides a trivial integer pool. +package pool + +import ( + "github.com/sagernet/gvisor/pkg/sync" +) + +// Pool is a simple allocator. +type Pool struct { + mu sync.Mutex + + // cache is the set of returned values. + cache []uint64 + + // Start is the starting value (if needed). + Start uint64 + + // max is the current maximum issued. + max uint64 + + // Limit is the upper limit. + Limit uint64 +} + +// Get gets a value from the pool. +func (p *Pool) Get() (uint64, bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // Anything cached? + if len(p.cache) > 0 { + v := p.cache[len(p.cache)-1] + p.cache = p.cache[:len(p.cache)-1] + return v, true + } + + // Over the limit? + if p.Start == p.Limit { + return 0, false + } + + // Generate a new value. + v := p.Start + p.Start++ + return v, true +} + +// Put returns a value to the pool. +func (p *Pool) Put(v uint64) { + p.mu.Lock() + p.cache = append(p.cache, v) + p.mu.Unlock() +} diff --git a/pkg/pool/pool_state_autogen.go b/pkg/pool/pool_state_autogen.go new file mode 100644 index 0000000..1f4164c --- /dev/null +++ b/pkg/pool/pool_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package pool diff --git a/pkg/rand/rand.go b/pkg/rand/rand.go new file mode 100644 index 0000000..94d2764 --- /dev/null +++ b/pkg/rand/rand.go @@ -0,0 +1,28 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux +// +build !linux + +package rand + +import "crypto/rand" + +// Reader is the default reader. +var Reader = rand.Reader + +// Read implements io.Reader.Read. +func Read(b []byte) (int, error) { + return rand.Read(b) +} diff --git a/pkg/rand/rand_linux.go b/pkg/rand/rand_linux.go new file mode 100644 index 0000000..ee346f8 --- /dev/null +++ b/pkg/rand/rand_linux.go @@ -0,0 +1,82 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rand + +import ( + "bufio" + "crypto/rand" + "io" + + "github.com/sagernet/gvisor/pkg/sync" + "golang.org/x/sys/unix" +) + +// reader implements an io.Reader that returns pseudorandom bytes. +type reader struct { + once sync.Once + useGetrandom bool +} + +// Read implements io.Reader.Read. +func (r *reader) Read(p []byte) (int, error) { + r.once.Do(func() { + _, err := unix.Getrandom(p, 0) + if err != unix.ENOSYS { + r.useGetrandom = true + } + }) + + if r.useGetrandom { + return unix.Getrandom(p, 0) + } + return rand.Read(p) +} + +// bufferedReader implements a threadsafe buffered io.Reader. +type bufferedReader struct { + mu sync.Mutex + r *bufio.Reader +} + +// Read implements io.Reader.Read. +func (b *bufferedReader) Read(p []byte) (int, error) { + // In Linux, reads of up to page size bytes will always complete fully. + // See drivers/char/random.c:get_random_bytes_user(). + // NOTE(gvisor.dev/issue/9445): Some applications rely on this behavior. + const pageSize = 4096 + min := len(p) + if min > pageSize { + min = pageSize + } + b.mu.Lock() + defer b.mu.Unlock() + return io.ReadAtLeast(b.r, p, min) +} + +// Reader is the default reader. +var Reader io.Reader = &bufferedReader{r: bufio.NewReader(&reader{})} + +// Read reads from the default reader. +func Read(b []byte) (int, error) { + return io.ReadFull(Reader, b) +} + +// Init can be called to make sure /dev/urandom is pre-opened on kernels that +// do not support getrandom(2). +func Init() error { + p := make([]byte, 1) + _, err := Read(p) + return err +} diff --git a/pkg/rand/rand_linux_state_autogen.go b/pkg/rand/rand_linux_state_autogen.go new file mode 100644 index 0000000..f727c93 --- /dev/null +++ b/pkg/rand/rand_linux_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package rand diff --git a/pkg/rand/rand_state_autogen.go b/pkg/rand/rand_state_autogen.go new file mode 100644 index 0000000..4320837 --- /dev/null +++ b/pkg/rand/rand_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build !linux +// +build !linux + +package rand diff --git a/pkg/rand/rng.go b/pkg/rand/rng.go new file mode 100644 index 0000000..ac2d0f8 --- /dev/null +++ b/pkg/rand/rng.go @@ -0,0 +1,131 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rand implements a cryptographically secure pseudorandom number +// generator. +package rand + +import ( + "encoding/binary" + "fmt" + "io" +) + +// RNG exposes convenience functions based on a cryptographically secure +// io.Reader. +type RNG struct { + Reader io.Reader +} + +// RNGFrom returns a new RNG. r must be a cryptographically secure io.Reader. +func RNGFrom(r io.Reader) RNG { + return RNG{Reader: r} +} + +// Uint16 is analogous to the standard library's math/rand.Uint16. +func (rg *RNG) Uint16() uint16 { + var data [2]byte + if _, err := rg.Reader.Read(data[:]); err != nil { + panic(fmt.Sprintf("Read() failed: %v", err)) + } + return binary.NativeEndian.Uint16(data[:]) +} + +// Uint32 is analogous to the standard library's math/rand.Uint32. +func (rg *RNG) Uint32() uint32 { + var data [4]byte + if _, err := rg.Reader.Read(data[:]); err != nil { + panic(fmt.Sprintf("Read() failed: %v", err)) + } + return binary.NativeEndian.Uint32(data[:]) +} + +// Int63n is analogous to the standard library's math/rand.Int63n. +func (rg *RNG) Int63n(n int64) int64 { + // Based on Go's rand package implementation, but using + // cryptographically secure random numbers. + if n <= 0 { + panic(fmt.Sprintf("n must be positive, but got %d", n)) + } + + // This can be done quickly when n is a power of 2. + if n&(n-1) == 0 { + return int64(rg.Uint64()) & (n - 1) + } + + // The naive approach would be to return rg.Int63()%n, but we need the + // random number to be fair. It shouldn't be biased towards certain + // results, but simple modular math can be very biased. For example, if + // n is 40% of the maximum int64, then the output values of rg.Int63 + // map to return values as follows: + // + // - The first 40% of values map to themselves. + // - The second 40% map to themselves - maximum int64. + // - The remaining 20% map to the themselves - 2 * (maximum int64), + // i.e. the first half of possible output values. + // + // And thus 60% of results map the first half of possible output + // values, and 40% map the second half. Oops! + // + // We use the same trick as Go to deal with this: shave off the last + // segment (the 20% in our example) to make the RNG more fair. + // + // In the worst case, n is just over half of maximum int64, meaning + // that the upper half of rg.Int63 return values are bad. So each call + // to rg.Int63 has, at worst, a 50% chance of needing a retry. + maximum := int64((1 << 63) - 1 - (1<<63)%uint64(n)) + ret := rg.Int63() + for ret > maximum { + ret = rg.Int63() + } + return ret % n +} + +// Int63 is analogous to the standard library's math/rand.Int63. +func (rg *RNG) Int63() int64 { + return ((1 << 63) - 1) & int64(rg.Uint64()) +} + +// Uint64 is analogous to the standard library's math/rand.Uint64. +func (rg *RNG) Uint64() uint64 { + var data [8]byte + if _, err := rg.Reader.Read(data[:]); err != nil { + panic(fmt.Sprintf("Read() failed: %v", err)) + } + return binary.NativeEndian.Uint64(data[:]) +} + +// Uint32 is analogous to the standard library's math/rand.Uint32. +func Uint32() uint32 { + rng := RNG{Reader: Reader} + return rng.Uint32() +} + +// Int63n is analogous to the standard library's math/rand.Int63n. +func Int63n(n int64) int64 { + rng := RNG{Reader: Reader} + return rng.Int63n(n) +} + +// Int63 is analogous to the standard library's math/rand.Int63. +func Int63() int64 { + rng := RNG{Reader: Reader} + return rng.Int63() +} + +// Uint64 is analogous to the standard library's math/rand.Uint64. +func Uint64() uint64 { + rng := RNG{Reader: Reader} + return rng.Uint64() +} diff --git a/pkg/rawfile/blockingpoll_amd64.s b/pkg/rawfile/blockingpoll_amd64.s new file mode 100644 index 0000000..3e192f4 --- /dev/null +++ b/pkg/rawfile/blockingpoll_amd64.s @@ -0,0 +1,41 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// BlockingPoll makes the ppoll() syscall while calling the version of +// entersyscall that relinquishes the P so that other Gs can run. This is meant +// to be called in cases when the syscall is expected to block. +// +// func BlockingPoll(fds *PollEvent, nfds int, timeout *syscall.Timespec) (n int, err syscall.Errno) +TEXT ·BlockingPoll(SB),NOSPLIT|NOFRAME,$0-40 + CALL ·callEntersyscallblock(SB) + MOVQ fds+0(FP), DI + MOVQ nfds+8(FP), SI + MOVQ timeout+16(FP), DX + MOVQ $0x0, R10 // sigmask parameter which isn't used here + MOVQ $0x10f, AX // SYS_PPOLL + SYSCALL + CMPQ AX, $0xfffffffffffff002 + JLS ok + MOVQ $-1, ret+24(FP) + NEGQ AX + MOVQ AX, ret1+32(FP) + CALL ·callExitsyscall(SB) + RET +ok: + MOVQ AX, ret+24(FP) + MOVQ $0, ret1+32(FP) + CALL ·callExitsyscall(SB) + RET diff --git a/pkg/rawfile/blockingpoll_arm64.s b/pkg/rawfile/blockingpoll_arm64.s new file mode 100644 index 0000000..99c65f8 --- /dev/null +++ b/pkg/rawfile/blockingpoll_arm64.s @@ -0,0 +1,42 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// BlockingPoll makes the ppoll() syscall while calling the version of +// entersyscall that relinquishes the P so that other Gs can run. This is meant +// to be called in cases when the syscall is expected to block. +// +// func BlockingPoll(fds *PollEvent, nfds int, timeout *syscall.Timespec) (n int, err syscall.Errno) +TEXT ·BlockingPoll(SB),NOSPLIT,$0-40 + BL ·callEntersyscallblock(SB) + MOVD fds+0(FP), R0 + MOVD nfds+8(FP), R1 + MOVD timeout+16(FP), R2 + MOVD $0x0, R3 // sigmask parameter which isn't used here + MOVD $0x49, R8 // SYS_PPOLL + SVC + CMP $0xfffffffffffff002, R0 + BLS ok + MOVD $-1, R1 + MOVD R1, ret+24(FP) + NEG R0, R0 + MOVD R0, ret1+32(FP) + BL ·callExitsyscall(SB) + RET +ok: + MOVD R0, ret+24(FP) + MOVD $0, ret1+32(FP) + BL ·callExitsyscall(SB) + RET diff --git a/pkg/rawfile/blockingpoll_noyield_unsafe.go b/pkg/rawfile/blockingpoll_noyield_unsafe.go new file mode 100644 index 0000000..c1438da --- /dev/null +++ b/pkg/rawfile/blockingpoll_noyield_unsafe.go @@ -0,0 +1,33 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !amd64 && !arm64 +// +build linux,!amd64,!arm64 + +package rawfile + +import ( + "unsafe" + + "golang.org/x/sys/unix" +) + +// BlockingPoll is just a stub function that forwards to the ppoll() system call +// on non-amd64 and non-arm64 platforms. +func BlockingPoll(fds *PollEvent, nfds int, timeout *unix.Timespec) (int, unix.Errno) { + n, _, e := unix.Syscall6(unix.SYS_PPOLL, uintptr(unsafe.Pointer(fds)), + uintptr(nfds), uintptr(unsafe.Pointer(timeout)), 0, 0, 0) + + return int(n), e +} diff --git a/pkg/rawfile/blockingpoll_yield_unsafe.go b/pkg/rawfile/blockingpoll_yield_unsafe.go new file mode 100644 index 0000000..9662411 --- /dev/null +++ b/pkg/rawfile/blockingpoll_yield_unsafe.go @@ -0,0 +1,69 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build ((linux && amd64) || (linux && arm64)) && go1.18 +// +build linux,amd64 linux,arm64 +// +build go1.18 + +// //go:linkname directives type-checked by checklinkname. Any other +// non-linkname assumptions outside the Go 1 compatibility guarantee should +// have an accompanied vet check or version guard build tag. + +package rawfile + +import ( + _ "unsafe" // for go:linkname + + "golang.org/x/sys/unix" +) + +// BlockingPoll on amd64/arm64 makes the ppoll() syscall while calling the +// version of entersyscall that relinquishes the P so that other Gs can +// run. This is meant to be called in cases when the syscall is expected to +// block. On non amd64/arm64 platforms it just forwards to the ppoll() system +// call. +// +//go:noescape +func BlockingPoll(fds *PollEvent, nfds int, timeout *unix.Timespec) (int, unix.Errno) + +// Use go:linkname to call into the runtime. As of Go 1.13 this has to +// be done from Go code so that we make an ABIInternal call to an +// ABIInternal function; see https://golang.org/issue/27539. + +// We need to call both entersyscallblock and exitsyscall this way so +// that the runtime's check on the stack pointer lines up. + +// Note that calling an unexported function in the runtime package is +// unsafe and this hack is likely to break in future Go releases. + +//go:linkname entersyscallblock runtime.entersyscallblock +func entersyscallblock() + +//go:linkname exitsyscall runtime.exitsyscall +func exitsyscall() + +// These forwarding functions must be nosplit because 1) we must +// disallow preemption between entersyscallblock and exitsyscall, and +// 2) we have an untyped assembly frame on the stack which can not be +// grown or moved. + +//go:nosplit +func callEntersyscallblock() { + entersyscallblock() +} + +//go:nosplit +func callExitsyscall() { + exitsyscall() +} diff --git a/pkg/rawfile/rawfile_unsafe.go b/pkg/rawfile/rawfile_unsafe.go new file mode 100644 index 0000000..4ec671a --- /dev/null +++ b/pkg/rawfile/rawfile_unsafe.go @@ -0,0 +1,233 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package rawfile contains utilities for using raw host files on Linux hosts. +package rawfile + +import ( + "reflect" + "unsafe" + + "golang.org/x/sys/unix" +) + +// SizeofIovec is the size of a unix.Iovec in bytes. +const SizeofIovec = unsafe.Sizeof(unix.Iovec{}) + +// MaxIovs is UIO_MAXIOV, the maximum number of iovecs that may be passed to a +// host system call in a single array. +const MaxIovs = 1024 + +// IovecFromBytes returns a unix.Iovec representing bs. +// +// Preconditions: len(bs) > 0. +func IovecFromBytes(bs []byte) unix.Iovec { + iov := unix.Iovec{ + Base: &bs[0], + } + iov.SetLen(len(bs)) + return iov +} + +func bytesFromIovec(iov unix.Iovec) (bs []byte) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) + sh.Data = uintptr(unsafe.Pointer(iov.Base)) + sh.Len = int(iov.Len) + sh.Cap = int(iov.Len) + return +} + +// AppendIovecFromBytes returns append(iovs, IovecFromBytes(bs)). If len(bs) == +// 0, AppendIovecFromBytes returns iovs without modification. If len(iovs) >= +// max, AppendIovecFromBytes replaces the final iovec in iovs with one that +// also includes the contents of bs. Note that this implies that +// AppendIovecFromBytes is only usable when the returned iovec slice is used as +// the source of a write. +func AppendIovecFromBytes(iovs []unix.Iovec, bs []byte, max int) []unix.Iovec { + if len(bs) == 0 { + return iovs + } + if len(iovs) < max { + return append(iovs, IovecFromBytes(bs)) + } + iovs[len(iovs)-1] = IovecFromBytes(append(bytesFromIovec(iovs[len(iovs)-1]), bs...)) + return iovs +} + +// MMsgHdr represents the mmsg_hdr structure required by recvmmsg() on linux. +type MMsgHdr struct { + Msg unix.Msghdr + Len uint32 + _ [4]byte +} + +// SizeofMMsgHdr is the size of a MMsgHdr in bytes. +const SizeofMMsgHdr = unsafe.Sizeof(MMsgHdr{}) + +// GetMTU determines the MTU of a network interface device. +func GetMTU(name string) (uint32, error) { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + if err != nil { + return 0, err + } + + defer unix.Close(fd) + + var ifreq struct { + name [16]byte + mtu int32 + _ [20]byte + } + + copy(ifreq.name[:], name) + _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), unix.SIOCGIFMTU, uintptr(unsafe.Pointer(&ifreq))) + if errno != 0 { + return 0, errno + } + + return uint32(ifreq.mtu), nil +} + +// NonBlockingWrite writes the given buffer to a file descriptor. It fails if +// partial data is written. +func NonBlockingWrite(fd int, buf []byte) unix.Errno { + var ptr unsafe.Pointer + if len(buf) > 0 { + ptr = unsafe.Pointer(&buf[0]) + } + + _, _, e := unix.RawSyscall(unix.SYS_WRITE, uintptr(fd), uintptr(ptr), uintptr(len(buf))) + return e +} + +// NonBlockingWriteIovec writes iovec to a file descriptor in a single unix. +// It fails if partial data is written. +func NonBlockingWriteIovec(fd int, iovec []unix.Iovec) unix.Errno { + iovecLen := uintptr(len(iovec)) + _, _, e := unix.RawSyscall(unix.SYS_WRITEV, uintptr(fd), uintptr(unsafe.Pointer(&iovec[0])), iovecLen) + return e +} + +// NonBlockingSendMMsg sends multiple messages on a socket. +func NonBlockingSendMMsg(fd int, msgHdrs []MMsgHdr) (int, unix.Errno) { + n, _, e := unix.RawSyscall6(unix.SYS_SENDMMSG, uintptr(fd), uintptr(unsafe.Pointer(&msgHdrs[0])), uintptr(len(msgHdrs)), unix.MSG_DONTWAIT, 0, 0) + return int(n), e +} + +// PollEvent represents the pollfd structure passed to a poll() system call. +type PollEvent struct { + FD int32 + Events int16 + Revents int16 +} + +// BlockingRead reads from a file descriptor that is set up as non-blocking. +// If no data is available, it will block in a poll() syscall until the file +// descriptor becomes readable. +func BlockingRead(fd int, b []byte) (int, unix.Errno) { + for { + n, _, e := unix.RawSyscall(unix.SYS_READ, uintptr(fd), uintptr(unsafe.Pointer(&b[0])), uintptr(len(b))) + if e == 0 { + return int(n), 0 + } + + event := PollEvent{ + FD: int32(fd), + Events: 1, // POLLIN + } + + _, e = BlockingPoll(&event, 1, nil) + if e != 0 && e != unix.EINTR { + return 0, e + } + } +} + +// BlockingReadvUntilStopped reads from a file descriptor that is set up as +// non-blocking and stores the data in a list of iovecs buffers. If no data is +// available, it will block in a poll() syscall until the file descriptor +// becomes readable or stop is signalled (efd becomes readable). Returns -1 in +// the latter case. +func BlockingReadvUntilStopped(efd int, fd int, iovecs []unix.Iovec) (int, unix.Errno) { + for { + n, _, e := unix.RawSyscall(unix.SYS_READV, uintptr(fd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs))) + if e == 0 { + return int(n), 0 + } + if e != 0 && e != unix.EWOULDBLOCK { + return 0, e + } + stopped, e := BlockingPollUntilStopped(efd, fd, unix.POLLIN) + if stopped { + return -1, e + } + if e != 0 && e != unix.EINTR { + return 0, e + } + } +} + +// BlockingRecvMMsgUntilStopped reads from a file descriptor that is set up as +// non-blocking and stores the received messages in a slice of MMsgHdr +// structures. If no data is available, it will block in a poll() syscall until +// the file descriptor becomes readable or stop is signalled (efd becomes +// readable). Returns -1 in the latter case. +func BlockingRecvMMsgUntilStopped(efd int, fd int, msgHdrs []MMsgHdr) (int, unix.Errno) { + for { + n, _, e := unix.RawSyscall6(unix.SYS_RECVMMSG, uintptr(fd), uintptr(unsafe.Pointer(&msgHdrs[0])), uintptr(len(msgHdrs)), unix.MSG_DONTWAIT, 0, 0) + if e == 0 { + return int(n), e + } + + if e != 0 && e != unix.EWOULDBLOCK { + return 0, e + } + + stopped, e := BlockingPollUntilStopped(efd, fd, unix.POLLIN) + if stopped { + return -1, e + } + if e != 0 && e != unix.EINTR { + return 0, e + } + } +} + +// BlockingPollUntilStopped polls for events on fd or until a stop is signalled +// on the event fd efd. Returns true if stopped, i.e., efd has event POLLIN. +func BlockingPollUntilStopped(efd int, fd int, events int16) (bool, unix.Errno) { + pevents := [...]PollEvent{ + { + FD: int32(efd), + Events: unix.POLLIN, + }, + { + FD: int32(fd), + Events: events, + }, + } + _, _, errno := unix.Syscall6(unix.SYS_PPOLL, uintptr(unsafe.Pointer(&pevents[0])), uintptr(len(pevents)), 0, 0, 0, 0) + if errno != 0 { + return pevents[0].Revents&unix.POLLIN != 0, errno + } + + if pevents[1].Revents&unix.POLLHUP != 0 || pevents[1].Revents&unix.POLLERR != 0 { + errno = unix.ECONNRESET + } + + return pevents[0].Revents&unix.POLLIN != 0, errno +} diff --git a/pkg/rawfile/rawfile_unsafe_state_autogen.go b/pkg/rawfile/rawfile_unsafe_state_autogen.go new file mode 100644 index 0000000..c446cda --- /dev/null +++ b/pkg/rawfile/rawfile_unsafe_state_autogen.go @@ -0,0 +1,11 @@ +// automatically generated by stateify. + +//go:build linux && !amd64 && !arm64 && ((linux && amd64) || (linux && arm64)) && go1.18 && linux +// +build linux +// +build !amd64 +// +build !arm64 +// +build linux,amd64 linux,arm64 +// +build go1.18 +// +build linux + +package rawfile diff --git a/pkg/refs/refcounter.go b/pkg/refs/refcounter.go new file mode 100644 index 0000000..7cc2934 --- /dev/null +++ b/pkg/refs/refcounter.go @@ -0,0 +1,196 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package refs defines an interface for reference counted objects. +package refs + +import ( + "bytes" + "fmt" + "runtime" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/context" + "github.com/sagernet/gvisor/pkg/sync" +) + +// RefCounter is the interface to be implemented by objects that are reference +// counted. +type RefCounter interface { + // IncRef increments the reference counter on the object. + IncRef() + + // DecRef decrements the object's reference count. Users of refs_template.Refs + // may specify a destructor to be called once the reference count reaches zero. + DecRef(ctx context.Context) +} + +// TryRefCounter is like RefCounter but allow the ref increment to be tried. +type TryRefCounter interface { + RefCounter + + // TryIncRef attempts to increment the reference count, but may fail if all + // references have already been dropped, in which case it returns false. If + // true is returned, then a valid reference is now held on the object. + TryIncRef() bool +} + +// LeakMode configures the leak checker. +type LeakMode uint32 + +const ( + // NoLeakChecking indicates that no effort should be made to check for + // leaks. + NoLeakChecking LeakMode = iota + + // LeaksLogWarning indicates that a warning should be logged when leaks + // are found. + LeaksLogWarning + + // LeaksPanic indidcates that a panic should be issued when leaks are found. + LeaksPanic +) + +// Set implements flag.Value. +func (l *LeakMode) Set(v string) error { + switch v { + case "disabled": + *l = NoLeakChecking + case "log-names": + *l = LeaksLogWarning + case "panic": + *l = LeaksPanic + default: + return fmt.Errorf("invalid ref leak mode %q", v) + } + return nil +} + +// Get implements flag.Value. +func (l *LeakMode) Get() any { + return *l +} + +// String implements flag.Value. +func (l LeakMode) String() string { + switch l { + case NoLeakChecking: + return "disabled" + case LeaksLogWarning: + return "log-names" + case LeaksPanic: + return "panic" + default: + panic(fmt.Sprintf("invalid ref leak mode %d", l)) + } +} + +// leakMode stores the current mode for the reference leak checker. +// +// Values must be one of the LeakMode values. +// +// leakMode must be accessed atomically. +var leakMode atomicbitops.Uint32 + +// SetLeakMode configures the reference leak checker. +func SetLeakMode(mode LeakMode) { + leakMode.Store(uint32(mode)) +} + +// GetLeakMode returns the current leak mode. +func GetLeakMode() LeakMode { + return LeakMode(leakMode.Load()) +} + +const maxStackFrames = 40 + +type fileLine struct { + file string + line int +} + +// A stackKey is a representation of a stack frame for use as a map key. +// +// The fileLine type is used as PC values seem to vary across collections, even +// for the same call stack. +type stackKey [maxStackFrames]fileLine + +var stackCache = struct { + sync.Mutex + entries map[stackKey][]uintptr +}{entries: map[stackKey][]uintptr{}} + +func makeStackKey(pcs []uintptr) stackKey { + frames := runtime.CallersFrames(pcs) + var key stackKey + keySlice := key[:0] + for { + frame, more := frames.Next() + keySlice = append(keySlice, fileLine{frame.File, frame.Line}) + + if !more || len(keySlice) == len(key) { + break + } + } + return key +} + +// RecordStack constructs and returns the PCs on the current stack. +func RecordStack() []uintptr { + pcs := make([]uintptr, maxStackFrames) + n := runtime.Callers(1, pcs) + if n == 0 { + // No pcs available. Stop now. + // + // This can happen if the first argument to runtime.Callers + // is large. + return nil + } + pcs = pcs[:n] + key := makeStackKey(pcs) + stackCache.Lock() + v, ok := stackCache.entries[key] + if !ok { + // Reallocate to prevent pcs from escaping. + v = append([]uintptr(nil), pcs...) + stackCache.entries[key] = v + } + stackCache.Unlock() + return v +} + +// FormatStack converts the given stack into a readable format. +func FormatStack(pcs []uintptr) string { + frames := runtime.CallersFrames(pcs) + var trace bytes.Buffer + for { + frame, more := frames.Next() + fmt.Fprintf(&trace, "%s:%d: %s\n", frame.File, frame.Line, frame.Function) + + if !more { + break + } + } + return trace.String() +} + +// OnExit is called on sandbox exit. It runs GC to enqueue refcount finalizers, +// which check for reference leaks. There is no way to guarantee that every +// finalizer will run before exiting, but this at least ensures that they will +// be discovered/enqueued by GC. +func OnExit() { + if LeakMode(leakMode.Load()) != NoLeakChecking { + runtime.GC() + } +} diff --git a/pkg/refs/refs_map.go b/pkg/refs/refs_map.go new file mode 100644 index 0000000..2c15fd5 --- /dev/null +++ b/pkg/refs/refs_map.go @@ -0,0 +1,179 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package refs + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/sync" +) + +var ( + // liveObjects is a global map of reference-counted objects. Objects are + // inserted when leak check is enabled, and they are removed when they are + // destroyed. It is protected by liveObjectsMu. + liveObjects map[CheckedObject]struct{} + liveObjectsMu sync.Mutex +) + +// CheckedObject represents a reference-counted object with an informative +// leak detection message. +type CheckedObject interface { + // RefType is the type of the reference-counted object. + RefType() string + + // LeakMessage supplies a warning to be printed upon leak detection. + LeakMessage() string + + // LogRefs indicates whether reference-related events should be logged. + LogRefs() bool +} + +func init() { + liveObjects = make(map[CheckedObject]struct{}) +} + +// LeakCheckEnabled returns whether leak checking is enabled. The following +// functions should only be called if it returns true. +func LeakCheckEnabled() bool { + mode := GetLeakMode() + return mode != NoLeakChecking +} + +// leakCheckPanicEnabled returns whether DoLeakCheck() should panic when leaks +// are detected. +func leakCheckPanicEnabled() bool { + return GetLeakMode() == LeaksPanic +} + +// Register adds obj to the live object map. +func Register(obj CheckedObject) { + if LeakCheckEnabled() { + liveObjectsMu.Lock() + if _, ok := liveObjects[obj]; ok { + panic(fmt.Sprintf("Unexpected entry in leak checking map: reference %p already added", obj)) + } + liveObjects[obj] = struct{}{} + liveObjectsMu.Unlock() + if LeakCheckEnabled() && obj.LogRefs() { + logEvent(obj, "registered") + } + } +} + +// Unregister removes obj from the live object map. +func Unregister(obj CheckedObject) { + if LeakCheckEnabled() { + liveObjectsMu.Lock() + defer liveObjectsMu.Unlock() + if _, ok := liveObjects[obj]; !ok { + panic(fmt.Sprintf("Expected to find entry in leak checking map for reference %p", obj)) + } + delete(liveObjects, obj) + if LeakCheckEnabled() && obj.LogRefs() { + logEvent(obj, "unregistered") + } + } +} + +// LogIncRef logs a reference increment. +func LogIncRef(obj CheckedObject, refs int64) { + if LeakCheckEnabled() && obj.LogRefs() { + logEvent(obj, fmt.Sprintf("IncRef to %d", refs)) + } +} + +// LogTryIncRef logs a successful TryIncRef call. +func LogTryIncRef(obj CheckedObject, refs int64) { + if LeakCheckEnabled() && obj.LogRefs() { + logEvent(obj, fmt.Sprintf("TryIncRef to %d", refs)) + } +} + +// LogDecRef logs a reference decrement. +func LogDecRef(obj CheckedObject, refs int64) { + if LeakCheckEnabled() && obj.LogRefs() { + logEvent(obj, fmt.Sprintf("DecRef to %d", refs)) + } +} + +// logEvent logs a message for the given reference-counted object. +// +// obj.LogRefs() should be checked before calling logEvent, in order to avoid +// calling any text processing needed to evaluate msg. +func logEvent(obj CheckedObject, msg string) { + log.Infof("[%s %p] %s:\n%s", obj.RefType(), obj, msg, FormatStack(RecordStack())) +} + +// checkOnce makes sure that leak checking is only done once. DoLeakCheck is +// called from multiple places (which may overlap) to cover different sandbox +// exit scenarios. +var checkOnce sync.Once + +// DoLeakCheck iterates through the live object map and logs a message for each +// object. It should be called when no reference-counted objects are reachable +// anymore, at which point anything left in the map is considered a leak. On +// multiple calls, only the first call will perform the leak check. +func DoLeakCheck() { + if LeakCheckEnabled() { + checkOnce.Do(doLeakCheck) + } +} + +// DoRepeatedLeakCheck is the same as DoLeakCheck except that it can be called +// multiple times by the caller to incrementally perform leak checking. +func DoRepeatedLeakCheck() { + if LeakCheckEnabled() { + doLeakCheck() + } +} + +type leakCheckDisabled interface { + LeakCheckDisabled() bool +} + +// CleanupSync is used to wait for async cleanup actions. +var CleanupSync sync.WaitGroup + +func doLeakCheck() { + CleanupSync.Wait() + liveObjectsMu.Lock() + defer liveObjectsMu.Unlock() + leaked := len(liveObjects) + if leaked > 0 { + n := 0 + msg := fmt.Sprintf("Leak checking detected %d leaked objects:\n", leaked) + for obj := range liveObjects { + skip := false + if o, ok := obj.(leakCheckDisabled); ok { + skip = o.LeakCheckDisabled() + } + if skip { + log.Debugf(obj.LeakMessage()) + continue + } + msg += obj.LeakMessage() + "\n" + n++ + } + if n == 0 { + return + } + if leakCheckPanicEnabled() { + panic(msg) + } + log.Warningf(msg) + } +} diff --git a/pkg/refs/refs_state_autogen.go b/pkg/refs/refs_state_autogen.go new file mode 100644 index 0000000..dfa2c1b --- /dev/null +++ b/pkg/refs/refs_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package refs diff --git a/pkg/ringdeque/ringdeque.go b/pkg/ringdeque/ringdeque.go new file mode 100644 index 0000000..9590158 --- /dev/null +++ b/pkg/ringdeque/ringdeque.go @@ -0,0 +1,154 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ringdeque provides the Deque type. +package ringdeque + +// A Deque implements a double-ended queue of values of type T using a +// resizable ring buffer. +// +// Deque is not safe to use concurrently from multiple goroutines. +type Deque[T any] struct { + // Items [off, off+len) modulo len(buf) in buf are valid. + // Invariants: + // - len(buf) is 0 or a power of 2. + // - If len(buf) == 0, off == 0; otherwise off < len(buf). + off uint64 + len uint64 + buf []T +} + +// Preconditions: d.len == len(d.buf). +func (d *Deque[T]) expand() { + newLen := 2 // arbitrary minimum + if d.len != 0 { + newLen = len(d.buf) * 2 + } + newBuf := make([]T, newLen) + // By precondition, we need to copy every element in d.buf. + n := copy(newBuf, d.buf[d.off:]) + copy(newBuf[n:], d.buf[:d.off]) + d.off = 0 + d.buf = newBuf +} + +func (d *Deque[T]) mask() uint64 { + return uint64(len(d.buf)) - 1 +} + +// Empty returns true if r contains no values. +func (d *Deque[T]) Empty() bool { + return d.len == 0 +} + +// Len returns the number of values in d. +func (d *Deque[T]) Len() int { + return int(d.len) +} + +// Clear removes all values from d. +func (d *Deque[T]) Clear() { + d.len = 0 +} + +// PushFront inserts x at the front of d. +func (d *Deque[T]) PushFront(x T) { + if int(d.len) == len(d.buf) { + d.expand() + } + newOff := (d.off - 1) & d.mask() + d.buf[newOff] = x + d.off = newOff + d.len++ +} + +// PushBack inserts x at the back of d. +func (d *Deque[T]) PushBack(x T) { + if int(d.len) == len(d.buf) { + d.expand() + } + i := (d.off + d.len) & d.mask() + d.buf[i] = x + d.len++ +} + +// PeekFront returns the value at the front of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) PeekFront() T { + return *d.PeekFrontPtr() +} + +// PeekFrontPtr returns a pointer to the value at the front of d. The pointer +// is only valid until the next mutation of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) PeekFrontPtr() *T { + if d.Empty() { + panic("peek of empty Deque") + } + return &d.buf[d.off] +} + +// PeekBack returns the value at the back of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) PeekBack() T { + return *d.PeekBackPtr() +} + +// PeekBackPtr returns a pointer to the value at the back of d. The pointer is +// only valid until the next mutation of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) PeekBackPtr() *T { + if d.Empty() { + panic("peek of empty Deque") + } + i := (d.off + d.len - 1) & d.mask() + return &d.buf[i] +} + +// RemoveFront removes the value at the front of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) RemoveFront() { + d.off = (d.off + 1) & d.mask() + d.len-- +} + +// RemoveBack removes the value at the back of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) RemoveBack() { + d.len-- +} + +// PopFront removes and returns the value at the front of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) PopFront() (x T) { + x = d.PeekFront() + d.RemoveFront() + return +} + +// PopBack removes and returns the value at the back of d. +// +// Preconditions: !d.Empty(). +func (d *Deque[T]) PopBack() (x T) { + x = d.PeekBack() + d.RemoveBack() + return +} diff --git a/pkg/safecopy/atomic_amd64.s b/pkg/safecopy/atomic_amd64.s new file mode 100644 index 0000000..c2e1872 --- /dev/null +++ b/pkg/safecopy/atomic_amd64.s @@ -0,0 +1,160 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// handleSwapUint32Fault returns the value stored in DI. Control is transferred +// to it when swapUint32 below receives SIGSEGV or SIGBUS, with the signal +// number stored in DI. +// +// It must have the same frame configuration as swapUint32 so that it can undo +// any potential call frame set up by the assembler. +TEXT handleSwapUint32Fault(SB), NOSPLIT|NOFRAME, $0-24 + MOVL DI, sig+20(FP) + RET + +// swapUint32 atomically stores new into *ptr and returns (the previous ptr* +// value, 0). If a SIGSEGV or SIGBUS signal is received during the swap, the +// value of old is unspecified, and sig is the number of the signal that was +// received. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//func swapUint32(ptr unsafe.Pointer, new uint32) (old uint32, sig int32) +TEXT ·swapUint32(SB), NOSPLIT|NOFRAME, $0-24 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleSwapUint32Fault will store a different value in this address. + MOVL $0, sig+20(FP) + + MOVQ ptr+0(FP), DI + MOVL new+8(FP), AX + XCHGL AX, 0(DI) + MOVL AX, old+16(FP) + RET + +// func addrOfSwapUint32() uintptr +TEXT ·addrOfSwapUint32(SB), $0-8 + MOVQ $·swapUint32(SB), AX + MOVQ AX, ret+0(FP) + RET + +// handleSwapUint64Fault returns the value stored in DI. Control is transferred +// to it when swapUint64 below receives SIGSEGV or SIGBUS, with the signal +// number stored in DI. +// +// It must have the same frame configuration as swapUint64 so that it can undo +// any potential call frame set up by the assembler. +TEXT handleSwapUint64Fault(SB), NOSPLIT|NOFRAME, $0-28 + MOVL DI, sig+24(FP) + RET + +// swapUint64 atomically stores new into *ptr and returns (the previous *ptr +// value, 0). If a SIGSEGV or SIGBUS signal is received during the swap, the +// value of old is unspecified, and sig is the number of the signal that was +// received. +// +// Preconditions: ptr must be aligned to a 8-byte boundary. +// +//func swapUint64(ptr unsafe.Pointer, new uint64) (old uint64, sig int32) +TEXT ·swapUint64(SB), NOSPLIT|NOFRAME, $0-28 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleSwapUint64Fault will store a different value in this address. + MOVL $0, sig+24(FP) + + MOVQ ptr+0(FP), DI + MOVQ new+8(FP), AX + XCHGQ AX, 0(DI) + MOVQ AX, old+16(FP) + RET + +// func addrOfSwapUint64() uintptr +TEXT ·addrOfSwapUint64(SB), NOSPLIT|NOFRAME, $0-8 + MOVQ $·swapUint64(SB), AX + MOVQ AX, ret+0(FP) + RET + +// handleCompareAndSwapUint32Fault returns the value stored in DI. Control is +// transferred to it when swapUint64 below receives SIGSEGV or SIGBUS, with the +// signal number stored in DI. +// +// It must have the same frame configuration as compareAndSwapUint32 so that it +// can undo any potential call frame set up by the assembler. +TEXT handleCompareAndSwapUint32Fault(SB), NOSPLIT|NOFRAME, $0-24 + MOVL DI, sig+20(FP) + RET + +// compareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns +// (the value previously stored at ptr, 0). If a SIGSEGV or SIGBUS signal is +// received during the operation, the value of prev is unspecified, and sig is +// the number of the signal that was received. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//func compareAndSwapUint32(ptr unsafe.Pointer, old, new uint32) (prev uint32, sig int32) +TEXT ·compareAndSwapUint32(SB), NOSPLIT|NOFRAME, $0-24 + // Store 0 as the returned signal number. If we run to completion, this is + // the value the caller will see; if a signal is received, + // handleCompareAndSwapUint32Fault will store a different value in this + // address. + MOVL $0, sig+20(FP) + + MOVQ ptr+0(FP), DI + MOVL old+8(FP), AX + MOVL new+12(FP), DX + LOCK + CMPXCHGL DX, 0(DI) + MOVL AX, prev+16(FP) + RET + +// func addrOfCompareAndSwapUint32() uintptr +TEXT ·addrOfCompareAndSwapUint32(SB), NOSPLIT|NOFRAME, $0-8 + MOVQ $·compareAndSwapUint32(SB), AX + MOVQ AX, ret+0(FP) + RET + +// handleLoadUint32Fault returns the value stored in DI. Control is transferred +// to it when LoadUint32 below receives SIGSEGV or SIGBUS, with the signal +// number stored in DI. +// +// It must have the same frame configuration as loadUint32 so that it can undo +// any potential call frame set up by the assembler. +TEXT handleLoadUint32Fault(SB), NOSPLIT|NOFRAME, $0-16 + MOVL DI, sig+12(FP) + RET + +// loadUint32 atomically loads *ptr and returns it. If a SIGSEGV or SIGBUS +// signal is received, the value returned is unspecified, and sig is the number +// of the signal that was received. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//func loadUint32(ptr unsafe.Pointer) (val uint32, sig int32) +TEXT ·loadUint32(SB), NOSPLIT|NOFRAME, $0-16 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleLoadUint32Fault will store a different value in this address. + MOVL $0, sig+12(FP) + + MOVQ ptr+0(FP), AX + MOVL (AX), BX + MOVL BX, val+8(FP) + RET + +// func addrOfLoadUint32() uintptr +TEXT ·addrOfLoadUint32(SB), NOSPLIT|NOFRAME, $0-8 + MOVQ $·loadUint32(SB), AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/safecopy/atomic_arm64.s b/pkg/safecopy/atomic_arm64.s new file mode 100644 index 0000000..6cc452a --- /dev/null +++ b/pkg/safecopy/atomic_arm64.s @@ -0,0 +1,152 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +#include "textflag.h" + +// handleSwapUint32Fault returns the value stored in R1. Control is transferred +// to it when swapUint32 below receives SIGSEGV or SIGBUS, with the signal +// number stored in R1. +// +// It must have the same frame configuration as swapUint32 so that it can undo +// any potential call frame set up by the assembler. +TEXT handleSwapUint32Fault(SB), NOSPLIT, $0-24 + MOVW R1, sig+20(FP) + RET + +// See the corresponding doc in safecopy_unsafe.go +// +// The code is derived from Go source runtime/internal/atomic.Xchg. +// +//func swapUint32(ptr unsafe.Pointer, new uint32) (old uint32, sig int32) +TEXT ·swapUint32(SB), NOSPLIT, $0-24 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleSwapUint32Fault will store a different value in this address. + MOVW $0, sig+20(FP) +again: + MOVD ptr+0(FP), R0 + MOVW new+8(FP), R1 + LDAXRW (R0), R2 + STLXRW R1, (R0), R3 + CBNZ R3, again + MOVW R2, old+16(FP) + RET + +// func addrOfSwapUint32() uintptr +TEXT ·addrOfSwapUint32(SB), $0-8 + MOVD $·swapUint32(SB), R0 + MOVD R0, ret+0(FP) + RET + +// handleSwapUint64Fault returns the value stored in R1. Control is transferred +// to it when swapUint64 below receives SIGSEGV or SIGBUS, with the signal +// number stored in R1. +// +// It must have the same frame configuration as swapUint64 so that it can undo +// any potential call frame set up by the assembler. +TEXT handleSwapUint64Fault(SB), NOSPLIT, $0-28 + MOVW R1, sig+24(FP) + RET + +// See the corresponding doc in safecopy_unsafe.go +// +// The code is derived from Go source runtime/internal/atomic.Xchg64. +// +//func swapUint64(ptr unsafe.Pointer, new uint64) (old uint64, sig int32) +TEXT ·swapUint64(SB), NOSPLIT, $0-28 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleSwapUint64Fault will store a different value in this address. + MOVW $0, sig+24(FP) +again: + MOVD ptr+0(FP), R0 + MOVD new+8(FP), R1 + LDAXR (R0), R2 + STLXR R1, (R0), R3 + CBNZ R3, again + MOVD R2, old+16(FP) + RET + +// func addrOfSwapUint64() uintptr +TEXT ·addrOfSwapUint64(SB), $0-8 + MOVD $·swapUint64(SB), R0 + MOVD R0, ret+0(FP) + RET + +// handleCompareAndSwapUint32Fault returns the value stored in R1. Control is +// transferred to it when compareAndSwapUint32 below receives SIGSEGV or SIGBUS, +// with the signal number stored in R1. +// +// It must have the same frame configuration as compareAndSwapUint32 so that it +// can undo any potential call frame set up by the assembler. +TEXT handleCompareAndSwapUint32Fault(SB), NOSPLIT, $0-24 + MOVW R1, sig+20(FP) + RET + +// See the corresponding doc in safecopy_unsafe.go +// +// The code is derived from Go source runtime/internal/atomic.Cas. +// +//func compareAndSwapUint32(ptr unsafe.Pointer, old, new uint32) (prev uint32, sig int32) +TEXT ·compareAndSwapUint32(SB), NOSPLIT, $0-24 + // Store 0 as the returned signal number. If we run to completion, this is + // the value the caller will see; if a signal is received, + // handleCompareAndSwapUint32Fault will store a different value in this + // address. + MOVW $0, sig+20(FP) + + MOVD ptr+0(FP), R0 + MOVW old+8(FP), R1 + MOVW new+12(FP), R2 +again: + LDAXRW (R0), R3 + CMPW R1, R3 + BNE done + STLXRW R2, (R0), R4 + CBNZ R4, again +done: + MOVW R3, prev+16(FP) + RET + +// func addrOfCompareAndSwapUint32() uintptr +TEXT ·addrOfCompareAndSwapUint32(SB), $0-8 + MOVD $·compareAndSwapUint32(SB), R0 + MOVD R0, ret+0(FP) + RET + +// handleLoadUint32Fault returns the value stored in DI. Control is transferred +// to it when LoadUint32 below receives SIGSEGV or SIGBUS, with the signal +// number stored in DI. +// +// It must have the same frame configuration as loadUint32 so that it can undo +// any potential call frame set up by the assembler. +TEXT handleLoadUint32Fault(SB), NOSPLIT, $0-16 + MOVW R1, sig+12(FP) + RET + +// loadUint32 atomically loads *ptr and returns it. If a SIGSEGV or SIGBUS +// signal is received, the value returned is unspecified, and sig is the number +// of the signal that was received. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//func loadUint32(ptr unsafe.Pointer) (val uint32, sig int32) +TEXT ·loadUint32(SB), NOSPLIT, $0-16 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleLoadUint32Fault will store a different value in this address. + MOVW $0, sig+12(FP) + + MOVD ptr+0(FP), R0 + LDARW (R0), R1 + MOVW R1, val+8(FP) + RET + +// func addrOfLoadUint32() uintptr +TEXT ·addrOfLoadUint32(SB), $0-8 + MOVD $·loadUint32(SB), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/safecopy/memclr_amd64.s b/pkg/safecopy/memclr_amd64.s new file mode 100644 index 0000000..815b60a --- /dev/null +++ b/pkg/safecopy/memclr_amd64.s @@ -0,0 +1,155 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +#include "textflag.h" + +// handleMemclrFault returns (the value stored in AX, the value stored in DI). +// Control is transferred to it when memclr below receives SIGSEGV or SIGBUS, +// with the faulting address stored in AX and the signal number stored in DI. +// +// It must have the same frame configuration as memclr so that it can undo any +// potential call frame set up by the assembler. +TEXT handleMemclrFault(SB), NOSPLIT|NOFRAME, $0-28 + MOVQ AX, addr+16(FP) + MOVL DI, sig+24(FP) + RET + +// memclr sets the n bytes following ptr to zeroes. If a SIGSEGV or SIGBUS +// signal is received during the write, it returns the address that caused the +// fault and the number of the signal that was received. Otherwise, it returns +// an unspecified address and a signal number of 0. +// +// Data is written in order, such that if a fault happens at address p, it is +// safe to assume that all data before p-maxRegisterSize has already been +// successfully written. +// +// The code is derived from runtime.memclrNoHeapPointers. +// +// func memclr(ptr unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32) +TEXT ·memclr(SB), NOSPLIT|NOFRAME, $0-28 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleMemclrFault will store a different value in this address. + MOVL $0, sig+24(FP) + + MOVQ ptr+0(FP), DI + MOVQ n+8(FP), BX + XORQ AX, AX + + // MOVOU seems always faster than REP STOSQ. +tail: + TESTQ BX, BX + JEQ _0 + CMPQ BX, $2 + JBE _1or2 + CMPQ BX, $4 + JBE _3or4 + CMPQ BX, $8 + JB _5through7 + JE _8 + CMPQ BX, $16 + JBE _9through16 + PXOR X0, X0 + CMPQ BX, $32 + JBE _17through32 + CMPQ BX, $64 + JBE _33through64 + CMPQ BX, $128 + JBE _65through128 + CMPQ BX, $256 + JBE _129through256 + // TODO: use branch table and BSR to make this just a single dispatch + // TODO: for really big clears, use MOVNTDQ, even without AVX2. + +loop: + MOVOU X0, 0(DI) + MOVOU X0, 16(DI) + MOVOU X0, 32(DI) + MOVOU X0, 48(DI) + MOVOU X0, 64(DI) + MOVOU X0, 80(DI) + MOVOU X0, 96(DI) + MOVOU X0, 112(DI) + MOVOU X0, 128(DI) + MOVOU X0, 144(DI) + MOVOU X0, 160(DI) + MOVOU X0, 176(DI) + MOVOU X0, 192(DI) + MOVOU X0, 208(DI) + MOVOU X0, 224(DI) + MOVOU X0, 240(DI) + SUBQ $256, BX + ADDQ $256, DI + CMPQ BX, $256 + JAE loop + JMP tail + +_1or2: + MOVB AX, (DI) + MOVB AX, -1(DI)(BX*1) + RET +_0: + RET +_3or4: + MOVW AX, (DI) + MOVW AX, -2(DI)(BX*1) + RET +_5through7: + MOVL AX, (DI) + MOVL AX, -4(DI)(BX*1) + RET +_8: + // We need a separate case for 8 to make sure we clear pointers atomically. + MOVQ AX, (DI) + RET +_9through16: + MOVQ AX, (DI) + MOVQ AX, -8(DI)(BX*1) + RET +_17through32: + MOVOU X0, (DI) + MOVOU X0, -16(DI)(BX*1) + RET +_33through64: + MOVOU X0, (DI) + MOVOU X0, 16(DI) + MOVOU X0, -32(DI)(BX*1) + MOVOU X0, -16(DI)(BX*1) + RET +_65through128: + MOVOU X0, (DI) + MOVOU X0, 16(DI) + MOVOU X0, 32(DI) + MOVOU X0, 48(DI) + MOVOU X0, -64(DI)(BX*1) + MOVOU X0, -48(DI)(BX*1) + MOVOU X0, -32(DI)(BX*1) + MOVOU X0, -16(DI)(BX*1) + RET +_129through256: + MOVOU X0, (DI) + MOVOU X0, 16(DI) + MOVOU X0, 32(DI) + MOVOU X0, 48(DI) + MOVOU X0, 64(DI) + MOVOU X0, 80(DI) + MOVOU X0, 96(DI) + MOVOU X0, 112(DI) + MOVOU X0, -128(DI)(BX*1) + MOVOU X0, -112(DI)(BX*1) + MOVOU X0, -96(DI)(BX*1) + MOVOU X0, -80(DI)(BX*1) + MOVOU X0, -64(DI)(BX*1) + MOVOU X0, -48(DI)(BX*1) + MOVOU X0, -32(DI)(BX*1) + MOVOU X0, -16(DI)(BX*1) + RET + +// func addrOfMemclr() uintptr +TEXT ·addrOfMemclr(SB), $0-8 + MOVQ $·memclr(SB), AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/safecopy/memclr_arm64.s b/pkg/safecopy/memclr_arm64.s new file mode 100644 index 0000000..0fc73bb --- /dev/null +++ b/pkg/safecopy/memclr_arm64.s @@ -0,0 +1,82 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +#include "textflag.h" + +// handleMemclrFault returns (the value stored in R0, the value stored in R1). +// Control is transferred to it when memclr below receives SIGSEGV or SIGBUS, +// with the faulting address stored in R0 and the signal number stored in R1. +// +// It must have the same frame configuration as memclr so that it can undo any +// potential call frame set up by the assembler. +TEXT handleMemclrFault(SB), NOSPLIT, $0-28 + MOVD R0, addr+16(FP) + MOVW R1, sig+24(FP) + RET + +// See the corresponding doc in safecopy_unsafe.go +// +// The code is derived from runtime.memclrNoHeapPointers. +// +// func memclr(ptr unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32) +TEXT ·memclr(SB), NOSPLIT, $0-28 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleMemclrFault will store a different value in this address. + MOVW $0, sig+24(FP) + MOVD ptr+0(FP), R0 + MOVD n+8(FP), R1 + + // If size is less than 16 bytes, use tail_zero to zero what remains + CMP $16, R1 + BLT tail_zero + // Get buffer offset into 16 byte aligned address for better performance + ANDS $15, R0, ZR + BNE unaligned_to_16 +aligned_to_16: + LSR $4, R1, R2 +zero_by_16: + STP.P (ZR, ZR), 16(R0) // Store pair with post index. + SUBS $1, R2, R2 + BNE zero_by_16 + ANDS $15, R1, R1 + BEQ end + + // Zero buffer with size=R1 < 16 +tail_zero: + TBZ $3, R1, tail_zero_4 + MOVD.P ZR, 8(R0) +tail_zero_4: + TBZ $2, R1, tail_zero_2 + MOVW.P ZR, 4(R0) +tail_zero_2: + TBZ $1, R1, tail_zero_1 + MOVH.P ZR, 2(R0) +tail_zero_1: + TBZ $0, R1, end + MOVB ZR, (R0) +end: + RET + +unaligned_to_16: + MOVD R0, R2 +head_loop: + MOVBU.P ZR, 1(R0) + ANDS $15, R0, ZR + BNE head_loop + // Adjust length for what remains + SUB R2, R0, R3 + SUB R3, R1 + // If size is less than 16 bytes, use tail_zero to zero what remains + CMP $16, R1 + BLT tail_zero + B aligned_to_16 + +// func addrOfMemclr() uintptr +TEXT ·addrOfMemclr(SB), $0-8 + MOVD $·memclr(SB), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/safecopy/memcpy_amd64.s b/pkg/safecopy/memcpy_amd64.s new file mode 100644 index 0000000..d1d759a --- /dev/null +++ b/pkg/safecopy/memcpy_amd64.s @@ -0,0 +1,225 @@ +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. +// Portions Copyright 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "textflag.h" + +// handleMemcpyFault returns (the value stored in AX, the value stored in DI). +// Control is transferred to it when memcpy below receives SIGSEGV or SIGBUS, +// with the faulting address stored in AX and the signal number stored in DI. +// +// It must have the same frame configuration as memcpy so that it can undo any +// potential call frame set up by the assembler. +TEXT handleMemcpyFault(SB), NOSPLIT|NOFRAME, $0-36 + MOVQ AX, addr+24(FP) + MOVL DI, sig+32(FP) + RET + +// memcpy copies data from src to dst. If a SIGSEGV or SIGBUS signal is received +// during the copy, it returns the address that caused the fault and the number +// of the signal that was received. Otherwise, it returns an unspecified address +// and a signal number of 0. +// +// Data is copied in order, such that if a fault happens at address p, it is +// safe to assume that all data before p-maxRegisterSize has already been +// successfully copied. +// +// The code is derived from the forward copying part of runtime.memmove. +// +// func memcpy(dst, src unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32) +TEXT ·memcpy(SB), NOSPLIT|NOFRAME, $0-36 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleMemcpyFault will store a different value in this address. + MOVL $0, sig+32(FP) + + MOVQ dst+0(FP), DI + MOVQ src+8(FP), SI + MOVQ n+16(FP), BX + +tail: + // BSR+branch table make almost all memmove/memclr benchmarks worse. Not + // worth doing. + TESTQ BX, BX + JEQ move_0 + CMPQ BX, $2 + JBE move_1or2 + CMPQ BX, $4 + JBE move_3or4 + CMPQ BX, $8 + JB move_5through7 + JE move_8 + CMPQ BX, $16 + JBE move_9through16 + CMPQ BX, $32 + JBE move_17through32 + CMPQ BX, $64 + JBE move_33through64 + CMPQ BX, $128 + JBE move_65through128 + CMPQ BX, $256 + JBE move_129through256 + +move_257plus: + SUBQ $256, BX + MOVOU (SI), X0 + MOVOU X0, (DI) + MOVOU 16(SI), X1 + MOVOU X1, 16(DI) + MOVOU 32(SI), X2 + MOVOU X2, 32(DI) + MOVOU 48(SI), X3 + MOVOU X3, 48(DI) + MOVOU 64(SI), X4 + MOVOU X4, 64(DI) + MOVOU 80(SI), X5 + MOVOU X5, 80(DI) + MOVOU 96(SI), X6 + MOVOU X6, 96(DI) + MOVOU 112(SI), X7 + MOVOU X7, 112(DI) + MOVOU 128(SI), X8 + MOVOU X8, 128(DI) + MOVOU 144(SI), X9 + MOVOU X9, 144(DI) + MOVOU 160(SI), X10 + MOVOU X10, 160(DI) + MOVOU 176(SI), X11 + MOVOU X11, 176(DI) + MOVOU 192(SI), X12 + MOVOU X12, 192(DI) + MOVOU 208(SI), X13 + MOVOU X13, 208(DI) + MOVOU 224(SI), X14 + MOVOU X14, 224(DI) + MOVOU 240(SI), X15 + MOVOU X15, 240(DI) + CMPQ BX, $256 + LEAQ 256(SI), SI + LEAQ 256(DI), DI + JGE move_257plus + JMP tail + +move_1or2: + MOVB (SI), AX + MOVB AX, (DI) + MOVB -1(SI)(BX*1), CX + MOVB CX, -1(DI)(BX*1) + RET +move_0: + RET +move_3or4: + MOVW (SI), AX + MOVW AX, (DI) + MOVW -2(SI)(BX*1), CX + MOVW CX, -2(DI)(BX*1) + RET +move_5through7: + MOVL (SI), AX + MOVL AX, (DI) + MOVL -4(SI)(BX*1), CX + MOVL CX, -4(DI)(BX*1) + RET +move_8: + // We need a separate case for 8 to make sure we write pointers atomically. + MOVQ (SI), AX + MOVQ AX, (DI) + RET +move_9through16: + MOVQ (SI), AX + MOVQ AX, (DI) + MOVQ -8(SI)(BX*1), CX + MOVQ CX, -8(DI)(BX*1) + RET +move_17through32: + MOVOU (SI), X0 + MOVOU X0, (DI) + MOVOU -16(SI)(BX*1), X1 + MOVOU X1, -16(DI)(BX*1) + RET +move_33through64: + MOVOU (SI), X0 + MOVOU X0, (DI) + MOVOU 16(SI), X1 + MOVOU X1, 16(DI) + MOVOU -32(SI)(BX*1), X2 + MOVOU X2, -32(DI)(BX*1) + MOVOU -16(SI)(BX*1), X3 + MOVOU X3, -16(DI)(BX*1) + RET +move_65through128: + MOVOU (SI), X0 + MOVOU X0, (DI) + MOVOU 16(SI), X1 + MOVOU X1, 16(DI) + MOVOU 32(SI), X2 + MOVOU X2, 32(DI) + MOVOU 48(SI), X3 + MOVOU X3, 48(DI) + MOVOU -64(SI)(BX*1), X4 + MOVOU X4, -64(DI)(BX*1) + MOVOU -48(SI)(BX*1), X5 + MOVOU X5, -48(DI)(BX*1) + MOVOU -32(SI)(BX*1), X6 + MOVOU X6, -32(DI)(BX*1) + MOVOU -16(SI)(BX*1), X7 + MOVOU X7, -16(DI)(BX*1) + RET +move_129through256: + MOVOU (SI), X0 + MOVOU X0, (DI) + MOVOU 16(SI), X1 + MOVOU X1, 16(DI) + MOVOU 32(SI), X2 + MOVOU X2, 32(DI) + MOVOU 48(SI), X3 + MOVOU X3, 48(DI) + MOVOU 64(SI), X4 + MOVOU X4, 64(DI) + MOVOU 80(SI), X5 + MOVOU X5, 80(DI) + MOVOU 96(SI), X6 + MOVOU X6, 96(DI) + MOVOU 112(SI), X7 + MOVOU X7, 112(DI) + MOVOU -128(SI)(BX*1), X8 + MOVOU X8, -128(DI)(BX*1) + MOVOU -112(SI)(BX*1), X9 + MOVOU X9, -112(DI)(BX*1) + MOVOU -96(SI)(BX*1), X10 + MOVOU X10, -96(DI)(BX*1) + MOVOU -80(SI)(BX*1), X11 + MOVOU X11, -80(DI)(BX*1) + MOVOU -64(SI)(BX*1), X12 + MOVOU X12, -64(DI)(BX*1) + MOVOU -48(SI)(BX*1), X13 + MOVOU X13, -48(DI)(BX*1) + MOVOU -32(SI)(BX*1), X14 + MOVOU X14, -32(DI)(BX*1) + MOVOU -16(SI)(BX*1), X15 + MOVOU X15, -16(DI)(BX*1) + RET + +// func addrOfMemcpy() uintptr +TEXT ·addrOfMemcpy(SB), $0-8 + MOVQ $·memcpy(SB), AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/safecopy/memcpy_arm64.s b/pkg/safecopy/memcpy_arm64.s new file mode 100644 index 0000000..1acddc7 --- /dev/null +++ b/pkg/safecopy/memcpy_arm64.s @@ -0,0 +1,86 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +#include "textflag.h" + +// handleMemcpyFault returns (the value stored in R0, the value stored in R1). +// Control is transferred to it when memcpy below receives SIGSEGV or SIGBUS, +// with the faulting address stored in R0 and the signal number stored in R1. +// +// It must have the same frame configuration as memcpy so that it can undo any +// potential call frame set up by the assembler. +TEXT handleMemcpyFault(SB), NOSPLIT, $0-36 + MOVD R0, addr+24(FP) + MOVW R1, sig+32(FP) + RET + +// memcpy copies data from src to dst. If a SIGSEGV or SIGBUS signal is received +// during the copy, it returns the address that caused the fault and the number +// of the signal that was received. Otherwise, it returns an unspecified address +// and a signal number of 0. +// +// Data is copied in order, such that if a fault happens at address p, it is +// safe to assume that all data before p-maxRegisterSize has already been +// successfully copied. +// +// The code is derived from the Go source runtime.memmove. +// +// func memcpy(dst, src unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32) +TEXT ·memcpy(SB), NOSPLIT, $-8-36 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleMemcpyFault will store a different value in this address. + MOVW $0, sig+32(FP) + + MOVD dst+0(FP), R3 + MOVD src+8(FP), R4 + MOVD n+16(FP), R5 + CMP $0, R5 + BNE check + RET + +check: + AND $~7, R5, R7 // R7 is N&~7. + SUB R7, R5, R6 // R6 is N&7. + + // Copying forward proceeds by copying R7/8 words then copying R6 bytes. + // R3 and R4 are advanced as we copy. + + // (There may be implementations of armv8 where copying by bytes until + // at least one of source or dest is word aligned is a worthwhile + // optimization, but the on the one tested so far (xgene) it did not + // make a significance difference.) + + CMP $0, R7 // Do we need to do any word-by-word copying? + BEQ noforwardlarge + ADD R3, R7, R9 // R9 points just past where we copy by word. + +forwardlargeloop: + MOVD.P 8(R4), R8 // R8 is just a scratch register. + MOVD.P R8, 8(R3) + CMP R3, R9 + BNE forwardlargeloop + +noforwardlarge: + CMP $0, R6 // Do we need to do any byte-by-byte copying? + BNE forwardtail + RET + +forwardtail: + ADD R3, R6, R9 // R9 points just past the destination memory. + +forwardtailloop: + MOVBU.P 1(R4), R8 + MOVBU.P R8, 1(R3) + CMP R3, R9 + BNE forwardtailloop + RET + +// func addrOfMemcpy() uintptr +TEXT ·addrOfMemcpy(SB), $0-8 + MOVD $·memcpy(SB), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/safecopy/safecopy.go b/pkg/safecopy/safecopy.go new file mode 100644 index 0000000..6189adf --- /dev/null +++ b/pkg/safecopy/safecopy.go @@ -0,0 +1,151 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package safecopy provides an efficient implementation of functions to access +// memory that may result in SIGSEGV or SIGBUS being sent to the accessor. +package safecopy + +import ( + "fmt" + "runtime" + + "github.com/sagernet/gvisor/pkg/errors" + "github.com/sagernet/gvisor/pkg/errors/linuxerr" + "github.com/sagernet/gvisor/pkg/sighandling" + "golang.org/x/sys/unix" +) + +// SegvError is returned when a safecopy function receives SIGSEGV. +type SegvError struct { + // Addr is the address at which the SIGSEGV occurred. + Addr uintptr +} + +// Error implements error.Error. +func (e SegvError) Error() string { + return fmt.Sprintf("SIGSEGV at %#x", e.Addr) +} + +// BusError is returned when a safecopy function receives SIGBUS. +type BusError struct { + // Addr is the address at which the SIGBUS occurred. + Addr uintptr +} + +// Error implements error.Error. +func (e BusError) Error() string { + return fmt.Sprintf("SIGBUS at %#x", e.Addr) +} + +// AlignmentError is returned when a safecopy function is passed an address +// that does not meet alignment requirements. +type AlignmentError struct { + // Addr is the invalid address. + Addr uintptr + + // Alignment is the required alignment. + Alignment uintptr +} + +// Error implements error.Error. +func (e AlignmentError) Error() string { + return fmt.Sprintf("address %#x is not aligned to a %d-byte boundary", e.Addr, e.Alignment) +} + +var ( + // The begin and end addresses below are for the functions that are + // checked by the signal handler. + memcpyBegin uintptr + memcpyEnd uintptr + memclrBegin uintptr + memclrEnd uintptr + swapUint32Begin uintptr + swapUint32End uintptr + swapUint64Begin uintptr + swapUint64End uintptr + compareAndSwapUint32Begin uintptr + compareAndSwapUint32End uintptr + loadUint32Begin uintptr + loadUint32End uintptr + + // savedSigSegVHandler is a pointer to the SIGSEGV handler that was + // configured before we replaced it with our own. We still call into it + // when we get a SIGSEGV that is not interesting to us. + savedSigSegVHandler uintptr + + // Same as above, but for SIGBUS signals. + savedSigBusHandler uintptr +) + +// signalHandler is our replacement signal handler for SIGSEGV and SIGBUS +// signals. +func signalHandler() + +// addrOfSignalHandler returns the start address of signalHandler. +// +// See comment on addrOfMemcpy for more details. +func addrOfSignalHandler() uintptr + +// FindEndAddress returns the end address (one byte beyond the last) of the +// function that contains the specified address (begin). +func FindEndAddress(begin uintptr) uintptr { + f := runtime.FuncForPC(begin) + if f != nil { + for p := begin; ; p++ { + g := runtime.FuncForPC(p) + if f != g { + return p + } + } + } + return begin +} + +// initializeAddresses initializes the addresses used by the signal handler. +func initializeAddresses() { + // The following functions are written in assembly language, so they won't + // be inlined by the existing compiler/linker. Tests will fail if this + // assumption is violated. + memcpyBegin = addrOfMemcpy() + memcpyEnd = FindEndAddress(memcpyBegin) + memclrBegin = addrOfMemclr() + memclrEnd = FindEndAddress(memclrBegin) + swapUint32Begin = addrOfSwapUint32() + swapUint32End = FindEndAddress(swapUint32Begin) + swapUint64Begin = addrOfSwapUint64() + swapUint64End = FindEndAddress(swapUint64Begin) + compareAndSwapUint32Begin = addrOfCompareAndSwapUint32() + compareAndSwapUint32End = FindEndAddress(compareAndSwapUint32Begin) + loadUint32Begin = addrOfLoadUint32() + loadUint32End = FindEndAddress(loadUint32Begin) + initializeArchAddresses() +} + +func init() { + initializeAddresses() + if err := sighandling.ReplaceSignalHandler(unix.SIGSEGV, addrOfSignalHandler(), &savedSigSegVHandler); err != nil { + panic(fmt.Sprintf("Unable to set handler for SIGSEGV: %v", err)) + } + if err := sighandling.ReplaceSignalHandler(unix.SIGBUS, addrOfSignalHandler(), &savedSigBusHandler); err != nil { + panic(fmt.Sprintf("Unable to set handler for SIGBUS: %v", err)) + } + linuxerr.AddErrorUnwrapper(func(e error) (*errors.Error, bool) { + switch e.(type) { + case SegvError, BusError, AlignmentError: + return linuxerr.EFAULT, true + default: + return nil, false + } + }) +} diff --git a/pkg/safecopy/safecopy_amd64_unsafe.go b/pkg/safecopy/safecopy_amd64_unsafe.go new file mode 100644 index 0000000..b1e864a --- /dev/null +++ b/pkg/safecopy/safecopy_amd64_unsafe.go @@ -0,0 +1,42 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || i386 +// +build amd64 i386 + +package safecopy + +import ( + "unsafe" +) + +var ( + checkXstateBegin uintptr + checkXstateEnd uintptr +) + +func initializeArchAddresses() { + checkXstateBegin = addrOfCheckXstate() + checkXstateEnd = FindEndAddress(checkXstateBegin) +} + +//go:noescape +func checkXstate(addr uintptr) (fault uintptr, sig int32, mxcsr uint32, cw uint16) +func addrOfCheckXstate() uintptr + +// CheckXstate verifies that xstate can be restored by the xrstor instruction. +func CheckXstate(state *byte) error { + _, sig, _, _ := checkXstate(uintptr(unsafe.Pointer(state))) + return errorFromFaultSignal(uintptr(unsafe.Pointer(state)), sig) +} diff --git a/pkg/safecopy/safecopy_amd64_unsafe_state_autogen.go b/pkg/safecopy/safecopy_amd64_unsafe_state_autogen.go new file mode 100644 index 0000000..b4552df --- /dev/null +++ b/pkg/safecopy/safecopy_amd64_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build amd64 || i386 +// +build amd64 i386 + +package safecopy diff --git a/pkg/safecopy/safecopy_arm64.go b/pkg/safecopy/safecopy_arm64.go new file mode 100644 index 0000000..5d941eb --- /dev/null +++ b/pkg/safecopy/safecopy_arm64.go @@ -0,0 +1,21 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package safecopy + +func initializeArchAddresses() { +} diff --git a/pkg/safecopy/safecopy_arm64_state_autogen.go b/pkg/safecopy/safecopy_arm64_state_autogen.go new file mode 100644 index 0000000..46e56e6 --- /dev/null +++ b/pkg/safecopy/safecopy_arm64_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build arm64 +// +build arm64 + +package safecopy diff --git a/pkg/safecopy/safecopy_state_autogen.go b/pkg/safecopy/safecopy_state_autogen.go new file mode 100644 index 0000000..791eef9 --- /dev/null +++ b/pkg/safecopy/safecopy_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package safecopy diff --git a/pkg/safecopy/safecopy_unsafe.go b/pkg/safecopy/safecopy_unsafe.go new file mode 100644 index 0000000..15f84ab --- /dev/null +++ b/pkg/safecopy/safecopy_unsafe.go @@ -0,0 +1,333 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package safecopy + +import ( + "fmt" + "runtime" + "unsafe" + + "golang.org/x/sys/unix" +) + +// maxRegisterSize is the maximum register size used in memcpy and memclr. It +// is used to decide by how much to rewind the copy (for memcpy) or zeroing +// (for memclr) before proceeding. +const maxRegisterSize = 16 + +// memcpy copies data from src to dst. If a SIGSEGV or SIGBUS signal is received +// during the copy, it returns the address that caused the fault and the number +// of the signal that was received. Otherwise, it returns an unspecified address +// and a signal number of 0. +// +// Data is copied in order, such that if a fault happens at address p, it is +// safe to assume that all data before p-maxRegisterSize has already been +// successfully copied. +// +//go:noescape +func memcpy(dst, src uintptr, n uintptr) (fault uintptr, sig int32) + +// memclr sets the n bytes following ptr to zeroes. If a SIGSEGV or SIGBUS +// signal is received during the write, it returns the address that caused the +// fault and the number of the signal that was received. Otherwise, it returns +// an unspecified address and a signal number of 0. +// +// Data is written in order, such that if a fault happens at address p, it is +// safe to assume that all data before p-maxRegisterSize has already been +// successfully written. +// +//go:noescape +func memclr(ptr uintptr, n uintptr) (fault uintptr, sig int32) + +// swapUint32 atomically stores new into *ptr and returns (the previous *ptr +// value, 0). If a SIGSEGV or SIGBUS signal is received during the swap, the +// value of old is unspecified, and sig is the number of the signal that was +// received. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//go:noescape +func swapUint32(ptr unsafe.Pointer, new uint32) (old uint32, sig int32) + +// swapUint64 atomically stores new into *ptr and returns (the previous *ptr +// value, 0). If a SIGSEGV or SIGBUS signal is received during the swap, the +// value of old is unspecified, and sig is the number of the signal that was +// received. +// +// Preconditions: ptr must be aligned to a 8-byte boundary. +// +//go:noescape +func swapUint64(ptr unsafe.Pointer, new uint64) (old uint64, sig int32) + +// compareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns +// (the value previously stored at ptr, 0). If a SIGSEGV or SIGBUS signal is +// received during the operation, the value of prev is unspecified, and sig is +// the number of the signal that was received. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//go:noescape +func compareAndSwapUint32(ptr unsafe.Pointer, old, new uint32) (prev uint32, sig int32) + +// LoadUint32 is like sync/atomic.LoadUint32, but operates with user memory. It +// may fail with SIGSEGV or SIGBUS if it is received while reading from ptr. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +// +//go:noescape +func loadUint32(ptr unsafe.Pointer) (val uint32, sig int32) + +// Return the start address of the functions above. +// +// In Go 1.17+, Go references to assembly functions resolve to an ABIInternal +// wrapper function rather than the function itself. We must reference from +// assembly to get the ABI0 (i.e., primary) address. +func addrOfMemcpy() uintptr +func addrOfMemclr() uintptr +func addrOfSwapUint32() uintptr +func addrOfSwapUint64() uintptr +func addrOfCompareAndSwapUint32() uintptr +func addrOfLoadUint32() uintptr + +// CopyIn copies len(dst) bytes from src to dst. It returns the number of bytes +// copied and an error if SIGSEGV or SIGBUS is received while reading from src. +func CopyIn(dst []byte, src unsafe.Pointer) (int, error) { + n, err := copyIn(dst, uintptr(src)) + runtime.KeepAlive(src) + return n, err +} + +// copyIn is the underlying definition for CopyIn. +func copyIn(dst []byte, src uintptr) (int, error) { + toCopy := uintptr(len(dst)) + if len(dst) == 0 { + return 0, nil + } + + fault, sig := memcpy(uintptr(unsafe.Pointer(&dst[0])), src, toCopy) + if sig == 0 { + return len(dst), nil + } + + if fault < src || fault >= src+toCopy { + panic(fmt.Sprintf("CopyIn raised signal %d at %#x, which is outside source [%#x, %#x)", sig, fault, src, src+toCopy)) + } + + // memcpy might have ended the copy up to maxRegisterSize bytes before + // fault, if an instruction caused a memory access that straddled two + // pages, and the second one faulted. Try to copy up to the fault. + var done int + if fault-src > maxRegisterSize { + done = int(fault - src - maxRegisterSize) + } + n, err := copyIn(dst[done:int(fault-src)], src+uintptr(done)) + done += n + if err != nil { + return done, err + } + return done, errorFromFaultSignal(fault, sig) +} + +// CopyOut copies len(src) bytes from src to dst. If returns the number of +// bytes done and an error if SIGSEGV or SIGBUS is received while writing to +// dst. +func CopyOut(dst unsafe.Pointer, src []byte) (int, error) { + n, err := copyOut(uintptr(dst), src) + runtime.KeepAlive(dst) + return n, err +} + +// copyOut is the underlying definition for CopyOut. +func copyOut(dst uintptr, src []byte) (int, error) { + toCopy := uintptr(len(src)) + if toCopy == 0 { + return 0, nil + } + + fault, sig := memcpy(dst, uintptr(unsafe.Pointer(&src[0])), toCopy) + if sig == 0 { + return len(src), nil + } + + if fault < dst || fault >= dst+toCopy { + panic(fmt.Sprintf("CopyOut raised signal %d at %#x, which is outside destination [%#x, %#x)", sig, fault, dst, dst+toCopy)) + } + + // memcpy might have ended the copy up to maxRegisterSize bytes before + // fault, if an instruction caused a memory access that straddled two + // pages, and the second one faulted. Try to copy up to the fault. + var done int + if fault-dst > maxRegisterSize { + done = int(fault - dst - maxRegisterSize) + } + n, err := copyOut(dst+uintptr(done), src[done:int(fault-dst)]) + done += n + if err != nil { + return done, err + } + return done, errorFromFaultSignal(fault, sig) +} + +// Copy copies toCopy bytes from src to dst. It returns the number of bytes +// copied and an error if SIGSEGV or SIGBUS is received while reading from src +// or writing to dst. +// +// Data is copied in order; if [src, src+toCopy) and [dst, dst+toCopy) overlap, +// the resulting contents of dst are unspecified. +func Copy(dst, src unsafe.Pointer, toCopy uintptr) (uintptr, error) { + n, err := copyN(uintptr(dst), uintptr(src), toCopy) + runtime.KeepAlive(dst) + runtime.KeepAlive(src) + return n, err +} + +// copyN is the underlying definition for Copy. +func copyN(dst, src uintptr, toCopy uintptr) (uintptr, error) { + if toCopy == 0 { + return 0, nil + } + + fault, sig := memcpy(dst, src, toCopy) + if sig == 0 { + return toCopy, nil + } + + // Did the fault occur while reading from src or writing to dst? + faultAfterSrc := ^uintptr(0) + if fault >= src { + faultAfterSrc = fault - src + } + faultAfterDst := ^uintptr(0) + if fault >= dst { + faultAfterDst = fault - dst + } + if faultAfterSrc >= toCopy && faultAfterDst >= toCopy { + panic(fmt.Sprintf("Copy raised signal %d at %#x, which is outside source [%#x, %#x) and destination [%#x, %#x)", sig, fault, src, src+toCopy, dst, dst+toCopy)) + } + faultedAfter := faultAfterSrc + if faultedAfter > faultAfterDst { + faultedAfter = faultAfterDst + } + + // memcpy might have ended the copy up to maxRegisterSize bytes before + // fault, if an instruction caused a memory access that straddled two + // pages, and the second one faulted. Try to copy up to the fault. + var done uintptr + if faultedAfter > maxRegisterSize { + done = faultedAfter - maxRegisterSize + } + n, err := copyN(dst+done, src+done, faultedAfter-done) + done += n + if err != nil { + return done, err + } + return done, errorFromFaultSignal(fault, sig) +} + +// ZeroOut writes toZero zero bytes to dst. It returns the number of bytes +// written and an error if SIGSEGV or SIGBUS is received while writing to dst. +func ZeroOut(dst unsafe.Pointer, toZero uintptr) (uintptr, error) { + n, err := zeroOut(uintptr(dst), toZero) + runtime.KeepAlive(dst) + return n, err +} + +// zeroOut is the underlying definition for ZeroOut. +func zeroOut(dst uintptr, toZero uintptr) (uintptr, error) { + if toZero == 0 { + return 0, nil + } + + fault, sig := memclr(dst, toZero) + if sig == 0 { + return toZero, nil + } + + if fault < dst || fault >= dst+toZero { + panic(fmt.Sprintf("ZeroOut raised signal %d at %#x, which is outside destination [%#x, %#x)", sig, fault, dst, dst+toZero)) + } + + // memclr might have ended the write up to maxRegisterSize bytes before + // fault, if an instruction caused a memory access that straddled two + // pages, and the second one faulted. Try to write up to the fault. + var done uintptr + if fault-dst > maxRegisterSize { + done = fault - dst - maxRegisterSize + } + n, err := zeroOut(dst+done, fault-dst-done) + done += n + if err != nil { + return done, err + } + return done, errorFromFaultSignal(fault, sig) +} + +// SwapUint32 is equivalent to sync/atomic.SwapUint32, except that it returns +// an error if SIGSEGV or SIGBUS is received while accessing ptr, or if ptr is +// not aligned to a 4-byte boundary. +func SwapUint32(ptr unsafe.Pointer, new uint32) (uint32, error) { + if addr := uintptr(ptr); addr&3 != 0 { + return 0, AlignmentError{addr, 4} + } + old, sig := swapUint32(ptr, new) + return old, errorFromFaultSignal(uintptr(ptr), sig) +} + +// SwapUint64 is equivalent to sync/atomic.SwapUint64, except that it returns +// an error if SIGSEGV or SIGBUS is received while accessing ptr, or if ptr is +// not aligned to an 8-byte boundary. +func SwapUint64(ptr unsafe.Pointer, new uint64) (uint64, error) { + if addr := uintptr(ptr); addr&7 != 0 { + return 0, AlignmentError{addr, 8} + } + old, sig := swapUint64(ptr, new) + return old, errorFromFaultSignal(uintptr(ptr), sig) +} + +// CompareAndSwapUint32 is equivalent to atomicbitops.CompareAndSwapUint32, +// except that it returns an error if SIGSEGV or SIGBUS is received while +// accessing ptr, or if ptr is not aligned to a 4-byte boundary. +func CompareAndSwapUint32(ptr unsafe.Pointer, old, new uint32) (uint32, error) { + if addr := uintptr(ptr); addr&3 != 0 { + return 0, AlignmentError{addr, 4} + } + prev, sig := compareAndSwapUint32(ptr, old, new) + return prev, errorFromFaultSignal(uintptr(ptr), sig) +} + +// LoadUint32 is like sync/atomic.LoadUint32, but operates with user memory. It +// may fail with SIGSEGV or SIGBUS if it is received while reading from ptr. +// +// Preconditions: ptr must be aligned to a 4-byte boundary. +func LoadUint32(ptr unsafe.Pointer) (uint32, error) { + if addr := uintptr(ptr); addr&3 != 0 { + return 0, AlignmentError{addr, 4} + } + val, sig := loadUint32(ptr) + return val, errorFromFaultSignal(uintptr(ptr), sig) +} + +func errorFromFaultSignal(addr uintptr, sig int32) error { + switch sig { + case 0: + return nil + case int32(unix.SIGSEGV): + return SegvError{addr} + case int32(unix.SIGBUS): + return BusError{addr} + default: + panic(fmt.Sprintf("safecopy got unexpected signal %d at address %#x", sig, addr)) + } +} diff --git a/pkg/safecopy/safecopy_unsafe_state_autogen.go b/pkg/safecopy/safecopy_unsafe_state_autogen.go new file mode 100644 index 0000000..791eef9 --- /dev/null +++ b/pkg/safecopy/safecopy_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package safecopy diff --git a/pkg/safecopy/sighandler_amd64.s b/pkg/safecopy/sighandler_amd64.s new file mode 100644 index 0000000..f705011 --- /dev/null +++ b/pkg/safecopy/sighandler_amd64.s @@ -0,0 +1,147 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// The signals handled by sigHandler. +#define SIGBUS 7 +#define SIGSEGV 11 + +// Offsets to the registers in context->uc_mcontext.gregs[]. +#define REG_RDI 0x68 +#define REG_RAX 0x90 +#define REG_IP 0xa8 + +// Offset to the si_addr field of siginfo. +#define SI_CODE 0x08 +#define SI_ADDR 0x10 + +// signalHandler is the signal handler for SIGSEGV and SIGBUS signals. It must +// not be set up as a handler to any other signals. +// +// If the instruction causing the signal is within a safecopy-protected +// function, the signal is handled such that execution resumes in the +// appropriate fault handling stub with AX containing the faulting address and +// DI containing the signal number. Otherwise control is transferred to the +// previously configured signal handler (savedSigSegvHandler or +// savedSigBusHandler). +// +// This function cannot be written in go because it runs whenever a signal is +// received by the thread (preempting whatever was running), which includes when +// garbage collector has stopped or isn't expecting any interactions (like +// barriers). +// +// The arguments are the following: +// DI - The signal number. +// SI - Pointer to siginfo_t structure. +// DX - Pointer to ucontext structure. +TEXT ·signalHandler(SB),NOSPLIT|NOFRAME,$0 + // Check if the signal is from the kernel. + MOVQ $0x0, CX + CMPL CX, SI_CODE(SI) + JGE original_handler + + // Check if RIP is within the area we care about. + MOVQ REG_IP(DX), CX + CMPQ CX, ·memcpyBegin(SB) + JB not_memcpy + CMPQ CX, ·memcpyEnd(SB) + JAE not_memcpy + + // Modify the context such that execution will resume in the fault + // handler. + LEAQ handleMemcpyFault(SB), CX + JMP handle_fault + +not_memcpy: + CMPQ CX, ·memclrBegin(SB) + JB not_memclr + CMPQ CX, ·memclrEnd(SB) + JAE not_memclr + + LEAQ handleMemclrFault(SB), CX + JMP handle_fault + +not_memclr: + CMPQ CX, ·swapUint32Begin(SB) + JB not_swapuint32 + CMPQ CX, ·swapUint32End(SB) + JAE not_swapuint32 + + LEAQ handleSwapUint32Fault(SB), CX + JMP handle_fault + +not_swapuint32: + CMPQ CX, ·swapUint64Begin(SB) + JB not_swapuint64 + CMPQ CX, ·swapUint64End(SB) + JAE not_swapuint64 + + LEAQ handleSwapUint64Fault(SB), CX + JMP handle_fault + +not_swapuint64: + CMPQ CX, ·compareAndSwapUint32Begin(SB) + JB not_casuint32 + CMPQ CX, ·compareAndSwapUint32End(SB) + JAE not_casuint32 + + LEAQ handleCompareAndSwapUint32Fault(SB), CX + JMP handle_fault + +not_casuint32: + CMPQ CX, ·loadUint32Begin(SB) + JB not_loaduint32 + CMPQ CX, ·loadUint32End(SB) + JAE not_loaduint32 + + LEAQ handleLoadUint32Fault(SB), CX + JMP handle_fault + +not_loaduint32: + CMPQ CX, ·checkXstateBegin(SB) + JB not_checkXstate + CMPQ CX, ·checkXstateEnd(SB) + JAE not_checkXstate + + LEAQ handleCheckXstateFault(SB), CX + JMP handle_fault +not_checkXstate: +original_handler: + // Jump to the previous signal handler, which is likely the golang one. + XORQ CX, CX + MOVQ ·savedSigBusHandler(SB), AX + CMPL DI, $SIGSEGV + CMOVQEQ ·savedSigSegVHandler(SB), AX + JMP AX + +handle_fault: + // Entered with the address of the fault handler in RCX; store it in + // RIP. + MOVQ CX, REG_IP(DX) + + // Store the faulting address in RAX. + MOVQ SI_ADDR(SI), CX + MOVQ CX, REG_RAX(DX) + + // Store the signal number in EDI. + MOVL DI, REG_RDI(DX) + + RET + +// func addrOfSignalHandler() uintptr +TEXT ·addrOfSignalHandler(SB), $0-8 + MOVQ $·signalHandler(SB), AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/safecopy/sighandler_arm64.s b/pkg/safecopy/sighandler_arm64.s new file mode 100644 index 0000000..41ed70f --- /dev/null +++ b/pkg/safecopy/sighandler_arm64.s @@ -0,0 +1,149 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// The signals handled by sigHandler. +#define SIGBUS 7 +#define SIGSEGV 11 + +// Offsets to the registers in context->uc_mcontext.gregs[]. +#define REG_R0 0xB8 +#define REG_R1 0xC0 +#define REG_PC 0x1B8 + +// Offset to the si_addr field of siginfo. +#define SI_CODE 0x08 +#define SI_ADDR 0x10 + +// signalHandler is the signal handler for SIGSEGV and SIGBUS signals. It must +// not be set up as a handler to any other signals. +// +// If the instruction causing the signal is within a safecopy-protected +// function, the signal is handled such that execution resumes in the +// appropriate fault handling stub with R0 containing the faulting address and +// R1 containing the signal number. Otherwise control is transferred to the +// previously configured signal handler (savedSigSegvHandler or +// savedSigBusHandler). +// +// This function cannot be written in go because it runs whenever a signal is +// received by the thread (preempting whatever was running), which includes when +// garbage collector has stopped or isn't expecting any interactions (like +// barriers). +// +// The arguments are the following: +// R0 - The signal number. +// R1 - Pointer to siginfo_t structure. +// R2 - Pointer to ucontext structure. +TEXT ·signalHandler(SB),NOSPLIT,$0 + // Check if the signal is from the kernel, si_code > 0 means a kernel signal. + MOVD SI_CODE(R1), R7 + CMPW $0x0, R7 + BLE original_handler + + // Check if PC is within the area we care about. + MOVD REG_PC(R2), R7 + MOVD ·memcpyBegin(SB), R8 + CMP R8, R7 + BLO not_memcpy + MOVD ·memcpyEnd(SB), R8 + CMP R8, R7 + BHS not_memcpy + + // Modify the context such that execution will resume in the fault handler. + MOVD $handleMemcpyFault(SB), R7 + B handle_fault + +not_memcpy: + MOVD ·memclrBegin(SB), R8 + CMP R8, R7 + BLO not_memclr + MOVD ·memclrEnd(SB), R8 + CMP R8, R7 + BHS not_memclr + + MOVD $handleMemclrFault(SB), R7 + B handle_fault + +not_memclr: + MOVD ·swapUint32Begin(SB), R8 + CMP R8, R7 + BLO not_swapuint32 + MOVD ·swapUint32End(SB), R8 + CMP R8, R7 + BHS not_swapuint32 + + MOVD $handleSwapUint32Fault(SB), R7 + B handle_fault + +not_swapuint32: + MOVD ·swapUint64Begin(SB), R8 + CMP R8, R7 + BLO not_swapuint64 + MOVD ·swapUint64End(SB), R8 + CMP R8, R7 + BHS not_swapuint64 + + MOVD $handleSwapUint64Fault(SB), R7 + B handle_fault + +not_swapuint64: + MOVD ·compareAndSwapUint32Begin(SB), R8 + CMP R8, R7 + BLO not_casuint32 + MOVD ·compareAndSwapUint32End(SB), R8 + CMP R8, R7 + BHS not_casuint32 + + MOVD $handleCompareAndSwapUint32Fault(SB), R7 + B handle_fault + +not_casuint32: + MOVD ·loadUint32Begin(SB), R8 + CMP R8, R7 + BLO not_loaduint32 + MOVD ·loadUint32End(SB), R8 + CMP R8, R7 + BHS not_loaduint32 + + MOVD $handleLoadUint32Fault(SB), R7 + B handle_fault + +not_loaduint32: +original_handler: + // Jump to the previous signal handler, which is likely the golang one. + MOVD ·savedSigBusHandler(SB), R7 + MOVD ·savedSigSegVHandler(SB), R8 + CMPW $SIGSEGV, R0 + CSEL EQ, R8, R7, R7 + B (R7) + +handle_fault: + // Entered with the address of the fault handler in R7; store it in PC. + MOVD R7, REG_PC(R2) + + // Store the faulting address in R0. + MOVD SI_ADDR(R1), R7 + MOVD R7, REG_R0(R2) + + // Store the signal number in R1. + MOVW R0, REG_R1(R2) + + RET + +// func addrOfSignalHandler() uintptr +TEXT ·addrOfSignalHandler(SB), $0-8 + MOVD $·signalHandler(SB), R0 + MOVD R0, ret+0(FP) + RET diff --git a/pkg/safecopy/xrstor_amd64.s b/pkg/safecopy/xrstor_amd64.s new file mode 100644 index 0000000..6c81d1b --- /dev/null +++ b/pkg/safecopy/xrstor_amd64.s @@ -0,0 +1,59 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// handleCheckXstateFault returns (the value stored in AX, the value stored in DI). +// Control is transferred to it when checkXstate below receives SIGSEGV or SIGBUS, +// with the faulting address stored in AX and the signal number stored in DI. +// +// It must have the same frame configuration as memcpy so that it can undo any +// potential call frame set up by the assembler. +TEXT handleCheckXstateFault(SB), NOSPLIT|NOFRAME, $0-26 + MOVQ AX, addr+8(FP) + MOVL DI, sig+16(FP) + + LDMXCSR mxcsr+20(FP) + BYTE $0xDB; BYTE $0xE2; // FNCLEX + FLDCW cw+24(FP) + RET + + +// ·checkXstate verifies that the specified floating point state can be loaded. +TEXT ·checkXstate(SB),NOSPLIT|NOFRAME,$0-26 + // Store 0 as the returned signal number. If we run to completion, + // this is the value the caller will see; if a signal is received, + // handleMemcpyFault will store a different value in this address. + MOVL $0, sig+16(FP) + // MXCSR and the x87 control word are the only floating point state + // that is callee-save and thus we must save. + STMXCSR mxcsr+20(FP) + FSTCW cw+24(FP) + + MOVQ addr+0(FP), DI + MOVL $0xffffffff, AX + MOVL $0xffffffff, DX + XRSTOR64 (DI) + + // Restore MXCSR and the x87 control word. + LDMXCSR mxcsr+20(FP) + BYTE $0xDB; BYTE $0xE2; // FNCLEX + FLDCW cw+24(FP) + RET + +// func addrOfCheckXstate() uintptr +TEXT ·addrOfCheckXstate(SB), $0-8 + MOVQ $·checkXstate(SB), AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/secio/full_reader.go b/pkg/secio/full_reader.go new file mode 100644 index 0000000..aed2564 --- /dev/null +++ b/pkg/secio/full_reader.go @@ -0,0 +1,34 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secio + +import ( + "io" +) + +// FullReader adapts an io.Reader to never return partial reads with a nil +// error. +type FullReader struct { + Reader io.Reader +} + +// Read implements io.Reader.Read. +func (r FullReader) Read(dst []byte) (int, error) { + n, err := io.ReadFull(r.Reader, dst) + if err == io.ErrUnexpectedEOF { + return n, io.EOF + } + return n, err +} diff --git a/pkg/secio/secio.go b/pkg/secio/secio.go new file mode 100644 index 0000000..29e9671 --- /dev/null +++ b/pkg/secio/secio.go @@ -0,0 +1,105 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package secio provides support for sectioned I/O. +package secio + +import ( + "errors" + "io" +) + +// ErrReachedLimit is returned when SectionReader.Read or SectionWriter.Write +// reaches its limit. +var ErrReachedLimit = errors.New("reached limit") + +// SectionReader implements io.Reader on a section of an underlying io.ReaderAt. +// It is similar to io.SectionReader, but: +// +// - Reading beyond the limit returns ErrReachedLimit, not io.EOF. +// +// - Limit overflow is handled correctly. +type SectionReader struct { + r io.ReaderAt + off int64 + limit int64 +} + +// Read implements io.Reader.Read. +func (r *SectionReader) Read(dst []byte) (int, error) { + if r.limit >= 0 { + if max := r.limit - r.off; max < int64(len(dst)) { + dst = dst[:max] + } + } + n, err := r.r.ReadAt(dst, r.off) + r.off += int64(n) + if err == nil && r.off == r.limit { + err = ErrReachedLimit + } + return n, err +} + +// NewOffsetReader returns an io.Reader that reads from r starting at offset +// off. +func NewOffsetReader(r io.ReaderAt, off int64) *SectionReader { + return &SectionReader{r, off, -1} +} + +// NewSectionReader returns an io.Reader that reads from r starting at offset +// off and stops with ErrReachedLimit after n bytes. +func NewSectionReader(r io.ReaderAt, off int64, n int64) *SectionReader { + // If off + n overflows, it will be < 0 such that no limit applies, but + // this is the correct behavior as long as r prohibits reading at offsets + // beyond MaxInt64. + return &SectionReader{r, off, off + n} +} + +// SectionWriter implements io.Writer on a section of an underlying +// io.WriterAt. Writing beyond the limit returns ErrReachedLimit. +type SectionWriter struct { + w io.WriterAt + off int64 + limit int64 +} + +// Write implements io.Writer.Write. +func (w *SectionWriter) Write(src []byte) (int, error) { + if w.limit >= 0 { + if max := w.limit - w.off; max < int64(len(src)) { + src = src[:max] + } + } + n, err := w.w.WriteAt(src, w.off) + w.off += int64(n) + if err == nil && w.off == w.limit { + err = ErrReachedLimit + } + return n, err +} + +// NewOffsetWriter returns an io.Writer that writes to w starting at offset +// off. +func NewOffsetWriter(w io.WriterAt, off int64) *SectionWriter { + return &SectionWriter{w, off, -1} +} + +// NewSectionWriter returns an io.Writer that writes to w starting at offset +// off and stops with ErrReachedLimit after n bytes. +func NewSectionWriter(w io.WriterAt, off int64, n int64) *SectionWriter { + // If off + n overflows, it will be < 0 such that no limit applies, but + // this is the correct behavior as long as w prohibits writing at offsets + // beyond MaxInt64. + return &SectionWriter{w, off, off + n} +} diff --git a/pkg/secio/secio_state_autogen.go b/pkg/secio/secio_state_autogen.go new file mode 100644 index 0000000..372ac4b --- /dev/null +++ b/pkg/secio/secio_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package secio diff --git a/pkg/sighandling/sighandling.go b/pkg/sighandling/sighandling.go new file mode 100644 index 0000000..7350e6d --- /dev/null +++ b/pkg/sighandling/sighandling.go @@ -0,0 +1,113 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sighandling contains helpers for handling signals to applications. +package sighandling + +import ( + "os" + "os/signal" + "reflect" + + "github.com/sagernet/gvisor/pkg/abi/linux" + "golang.org/x/sys/unix" +) + +// numSignals is the number of normal (non-realtime) signals on Linux. +const numSignals = 32 + +// handleSignals listens for incoming signals and calls the given handler +// function. +// +// It stops when the stop channel is closed. The done channel is closed once it +// will no longer deliver signals to k. +func handleSignals(sigchans []chan os.Signal, handler func(linux.Signal), stop, done chan struct{}) { + // Build a select case. + sc := []reflect.SelectCase{{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(stop)}} + for _, sigchan := range sigchans { + sc = append(sc, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(sigchan)}) + } + + for { + // Wait for a notification. + index, _, ok := reflect.Select(sc) + + // Was it the stop channel? + if index == 0 { + if !ok { + // Stop forwarding and notify that it's done. + close(done) + return + } + continue + } + + // How about a different close? + if !ok { + panic("signal channel closed unexpectedly") + } + + // Otherwise, it was a signal on channel N. Index 0 represents the stop + // channel, so index N represents the channel for signal N. + handler(linux.Signal(index)) + } +} + +// StartSignalForwarding ensures that synchronous signals are passed to the +// given handler function and returns a callback that stops signal delivery. +// +// Note that this function permanently takes over signal handling. After the +// stop callback, signals revert to the default Go runtime behavior, which +// cannot be overridden with external calls to signal.Notify. +func StartSignalForwarding(handler func(linux.Signal)) func() { + stop := make(chan struct{}) + done := make(chan struct{}) + + // Register individual channels. One channel per standard signal is + // required as os.Notify() is non-blocking and may drop signals. To avoid + // this, standard signals have to be queued separately. Channel size 1 is + // enough for standard signals as their semantics allow de-duplication. + // + // External real-time signals are not supported. We rely on the go-runtime + // for their handling. + // + // We do not forward some signals that are likely induced by the behavior + // of the forwarding process. + var sigchans []chan os.Signal + for sig := 1; sig <= numSignals+1; sig++ { + sigchan := make(chan os.Signal, 1) + sigchans = append(sigchans, sigchan) + + // SIGURG is used by Go's runtime scheduler. + if sig == int(linux.SIGURG) { + continue + } + // SIGPIPE is received when sending to disconnected host pipes/sockets. + if sig == int(linux.SIGPIPE) { + continue + } + // SIGCHLD is received when a child of the forwarding process exits. + if sig == int(linux.SIGCHLD) { + continue + } + signal.Notify(sigchan, unix.Signal(sig)) + } + // Start up our listener. + go handleSignals(sigchans, handler, stop, done) // S/R-SAFE: synchronized by Kernel.extMu. + + return func() { + close(stop) + <-done + } +} diff --git a/pkg/sighandling/sighandling_darwin.go b/pkg/sighandling/sighandling_darwin.go new file mode 100644 index 0000000..e7f5ce5 --- /dev/null +++ b/pkg/sighandling/sighandling_darwin.go @@ -0,0 +1,40 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin +// +build darwin + +package sighandling + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +// IgnoreChildStop sets the SA_NOCLDSTOP flag, causing child processes to not +// generate SIGCHLD when they stop. +func IgnoreChildStop() error { + return errors.New("IgnoreChildStop not supported on Darwin") +} + +// ReplaceSignalHandler replaces the existing signal handler for the provided +// signal with the function pointer at `handler`. This bypasses the Go runtime +// signal handlers, and should only be used for low-level signal handlers where +// use of signal.Notify is not appropriate. +// +// It stores the value of the previously set handler in previous. +func ReplaceSignalHandler(sig unix.Signal, handler uintptr, previous *uintptr) error { + return errors.New("ReplaceSignalHandler not supported on Darwin") +} diff --git a/pkg/sighandling/sighandling_linux_unsafe.go b/pkg/sighandling/sighandling_linux_unsafe.go new file mode 100644 index 0000000..4f2e4ff --- /dev/null +++ b/pkg/sighandling/sighandling_linux_unsafe.go @@ -0,0 +1,103 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sighandling + +import ( + "fmt" + "os" + "unsafe" + + "github.com/sagernet/gvisor/pkg/abi/linux" + "golang.org/x/sys/unix" +) + +// IgnoreChildStop sets the SA_NOCLDSTOP flag, causing child processes to not +// generate SIGCHLD when they stop. +func IgnoreChildStop() error { + var sa linux.SigAction + + // Get the existing signal handler information, and set the flag. + if _, _, e := unix.RawSyscall6(unix.SYS_RT_SIGACTION, uintptr(unix.SIGCHLD), 0, uintptr(unsafe.Pointer(&sa)), linux.SignalSetSize, 0, 0); e != 0 { + return e + } + sa.Flags |= linux.SA_NOCLDSTOP + if _, _, e := unix.RawSyscall6(unix.SYS_RT_SIGACTION, uintptr(unix.SIGCHLD), uintptr(unsafe.Pointer(&sa)), 0, linux.SignalSetSize, 0, 0); e != 0 { + return e + } + + return nil +} + +// ReplaceSignalHandler replaces the existing signal handler for the provided +// signal with the function pointer at `handler`. This bypasses the Go runtime +// signal handlers, and should only be used for low-level signal handlers where +// use of signal.Notify is not appropriate. +// +// It stores the value of the previously set handler in previous. +func ReplaceSignalHandler(sig unix.Signal, handler uintptr, previous *uintptr) error { + var sa linux.SigAction + const maskLen = 8 + + // Get the existing signal handler information, and save the current + // handler. Once we replace it, we will use this pointer to fall back to + // it when we receive other signals. + if _, _, e := unix.RawSyscall6(unix.SYS_RT_SIGACTION, uintptr(sig), 0, uintptr(unsafe.Pointer(&sa)), maskLen, 0, 0); e != 0 { + return e + } + + // Fail if there isn't a previous handler. + if sa.Handler == 0 { + return fmt.Errorf("previous handler for signal %x isn't set", sig) + } + + *previous = uintptr(sa.Handler) + + // Install our own handler. + sa.Handler = uint64(handler) + if _, _, e := unix.RawSyscall6(unix.SYS_RT_SIGACTION, uintptr(sig), uintptr(unsafe.Pointer(&sa)), 0, maskLen, 0, 0); e != 0 { + return e + } + + return nil +} + +// KillItself sends SIGKILL to the current process, bypassing the init process +// restriction. +// +// The standard `kill(getpid(), SIGKILL)` syscall doesn't work when the current +// process is the init process within its PID namespace. This is a "known" +// Linux feature. +// +// This function uses the rt_tgqueueinfo syscall to send a "kernel-generated" +// SIGKILL. +func KillItself() error { + pid := os.Getpid() + tid, _, _ := unix.RawSyscall(unix.SYS_GETTID, 0, 0, 0) + info := linux.SignalInfo{Code: linux.SI_KERNEL} + // The current thread can send a fake kernel siginfo to itself. + if _, _, e := unix.RawSyscall6( + unix.SYS_RT_TGSIGQUEUEINFO, + uintptr(pid), uintptr(tid), + uintptr(linux.SIGKILL), + uintptr(unsafe.Pointer(&info)), + 0, 0, + ); e != 0 { + return e + } + panic("unreachable") +} diff --git a/pkg/sighandling/sighandling_linux_unsafe_state_autogen.go b/pkg/sighandling/sighandling_linux_unsafe_state_autogen.go new file mode 100644 index 0000000..c44c162 --- /dev/null +++ b/pkg/sighandling/sighandling_linux_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package sighandling diff --git a/pkg/sighandling/sighandling_state_autogen.go b/pkg/sighandling/sighandling_state_autogen.go new file mode 100644 index 0000000..383cc6a --- /dev/null +++ b/pkg/sighandling/sighandling_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build darwin +// +build darwin + +package sighandling diff --git a/pkg/sleep/sleep_unsafe.go b/pkg/sleep/sleep_unsafe.go new file mode 100644 index 0000000..b9a84b2 --- /dev/null +++ b/pkg/sleep/sleep_unsafe.go @@ -0,0 +1,476 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sleep allows goroutines to efficiently sleep on multiple sources of +// notifications (wakers). It offers O(1) complexity, which is different from +// multi-channel selects which have O(n) complexity (where n is the number of +// channels) and a considerable constant factor. +// +// It is similar to edge-triggered epoll waits, where the user registers each +// object of interest once, and then can repeatedly wait on all of them. +// +// A Waker object is used to wake a sleeping goroutine (G) up, or prevent it +// from going to sleep next. A Sleeper object is used to receive notifications +// from wakers, and if no notifications are available, to optionally sleep until +// one becomes available. +// +// A Waker can be associated with at most one Sleeper, but a Sleeper can be +// associated with multiple Wakers. A Sleeper has a list of asserted (ready) +// wakers; when Fetch() is called repeatedly, elements from this list are +// returned until the list becomes empty in which case the goroutine goes to +// sleep. When Assert() is called on a Waker, it adds itself to the Sleeper's +// asserted list and wakes the G up from its sleep if needed. +// +// Sleeper objects are expected to be used as follows, with just one goroutine +// executing this code: +// +// // One time set-up. +// s := sleep.Sleeper{} +// s.AddWaker(&w1) +// s.AddWaker(&w2) +// +// // Called repeatedly. +// for { +// switch s.Fetch(true) { +// case &w1: +// // Do work triggered by w1 being asserted. +// case &w2: +// // Do work triggered by w2 being asserted. +// } +// } +// +// And Waker objects are expected to call w.Assert() when they want the sleeper +// to wake up and perform work. +// +// The notifications are edge-triggered, which means that if a Waker calls +// Assert() several times before the sleeper has the chance to wake up, it will +// only be notified once and should perform all pending work (alternatively, it +// can also call Assert() on the waker, to ensure that it will wake up again). +// +// The "unsafeness" here is in the casts to/from unsafe.Pointer, which is safe +// when only one type is used for each unsafe.Pointer (which is the case here), +// we should just make sure that this remains the case in the future. The usage +// of unsafe package could be confined to sharedWaker and sharedSleeper types +// that would hold pointers in atomic.Pointers, but the go compiler currently +// can't optimize these as well (it won't inline their method calls), which +// reduces performance. +package sleep + +import ( + "context" + "sync/atomic" + "unsafe" + + "github.com/sagernet/gvisor/pkg/sync" +) + +const ( + // preparingG is stored in sleepers to indicate that they're preparing + // to sleep. + preparingG = 1 +) + +// assertedSleeper is a sentinel sleeper. A pointer to it is stored in +// wakers that are asserted. +var assertedSleeper Sleeper + +// Sleeper allows a goroutine to sleep and receive wake up notifications from +// Wakers in an efficient way. +// +// This is similar to edge-triggered epoll in that wakers are added to the +// sleeper once and the sleeper can then repeatedly sleep in O(1) time while +// waiting on all wakers. +// +// None of the methods in a Sleeper can be called concurrently. Wakers that have +// been added to a sleeper A can only be added to another sleeper after A.Done() +// returns. These restrictions allow this to be implemented lock-free. +// +// This struct is thread-compatible. +// +// +stateify savable +type Sleeper struct { + _ sync.NoCopy + + // sharedList is a "stack" of asserted wakers. They atomically add + // themselves to the front of this list as they become asserted. + sharedList unsafe.Pointer `state:".(*Waker)"` + + // localList is a list of asserted wakers that is only accessible to the + // waiter, and thus doesn't have to be accessed atomically. When + // fetching more wakers, the waiter will first go through this list, and + // only when it's empty will it atomically fetch wakers from + // sharedList. + localList *Waker + + // allWakers is a list with all wakers that have been added to this + // sleeper. It is used during cleanup to remove associations. + allWakers *Waker + + // waitingG holds the G that is sleeping, if any. It is used by wakers + // to determine which G, if any, they should wake. + waitingG uintptr `state:"zero"` +} + +// saveSharedList is invoked by stateify. +func (s *Sleeper) saveSharedList() *Waker { + return (*Waker)(atomic.LoadPointer(&s.sharedList)) +} + +// loadSharedList is invoked by stateify. +func (s *Sleeper) loadSharedList(_ context.Context, w *Waker) { + atomic.StorePointer(&s.sharedList, unsafe.Pointer(w)) +} + +// AddWaker associates the given waker to the sleeper. +func (s *Sleeper) AddWaker(w *Waker) { + if w.allWakersNext != nil { + panic("waker has non-nil allWakersNext; owned by another sleeper?") + } + if w.next != nil { + panic("waker has non-nil next; queued in another sleeper?") + } + + // Add the waker to the list of all wakers. + w.allWakersNext = s.allWakers + s.allWakers = w + + // Try to associate the waker with the sleeper. If it's already + // asserted, we simply enqueue it in the "ready" list. + for { + p := (*Sleeper)(atomic.LoadPointer(&w.s)) + if p == &assertedSleeper { + s.enqueueAssertedWaker(w, true /* wakep */) + return + } + + if atomic.CompareAndSwapPointer(&w.s, usleeper(p), usleeper(s)) { + return + } + } +} + +// nextWaker returns the next waker in the notification list, blocking if +// needed. The parameter wakepOrSleep indicates that if the operation does not +// block, then we will need to explicitly wake a runtime P. +// +// Precondition: wakepOrSleep may be true iff block is true. +// +//go:nosplit +func (s *Sleeper) nextWaker(block, wakepOrSleep bool) *Waker { + // Attempt to replenish the local list if it's currently empty. + if s.localList == nil { + for atomic.LoadPointer(&s.sharedList) == nil { + // Fail request if caller requested that we + // don't block. + if !block { + return nil + } + + // Indicate to wakers that we're about to sleep, + // this allows them to abort the wait by setting + // waitingG back to zero (which we'll notice + // before committing the sleep). + atomic.StoreUintptr(&s.waitingG, preparingG) + + // Check if something was queued while we were + // preparing to sleep. We need this interleaving + // to avoid missing wake ups. + if atomic.LoadPointer(&s.sharedList) != nil { + atomic.StoreUintptr(&s.waitingG, 0) + break + } + + // Since we are sleeping for sure, we no longer + // need to wakep once we get a value. + wakepOrSleep = false + + // Try to commit the sleep and report it to the + // tracer as a select. + // + // gopark puts the caller to sleep and calls + // commitSleep to decide whether to immediately + // wake the caller up or to leave it sleeping. + const traceEvGoBlockSelect = 24 + // See:runtime2.go in the go runtime package for + // the values to pass as the waitReason here. + const waitReasonSelect = 9 + sync.Gopark(commitSleep, unsafe.Pointer(&s.waitingG), sync.WaitReasonSelect, sync.TraceBlockSelect, 0) + } + + // Pull the shared list out and reverse it in the local + // list. Given that wakers push themselves in reverse + // order, we fix things here. + v := (*Waker)(atomic.SwapPointer(&s.sharedList, nil)) + for v != nil { + cur := v + v = v.next + + cur.next = s.localList + s.localList = cur + } + } + + // Remove the waker in the front of the list. + w := s.localList + s.localList = w.next + + // Do we need to wake a P? + if wakepOrSleep { + sync.Wakep() + } + + return w +} + +// commitSleep signals to wakers that the given g is now sleeping. Wakers can +// then fetch it and wake it. +// +// The commit may fail if wakers have been asserted after our last check, in +// which case they will have set s.waitingG to zero. +// +//go:norace +//go:nosplit +func commitSleep(g uintptr, waitingG unsafe.Pointer) bool { + return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(waitingG), preparingG, g) +} + +// fetch is the backing implementation for Fetch and AssertAndFetch. +// +// Preconditions are the same as nextWaker. +// +//go:nosplit +func (s *Sleeper) fetch(block, wakepOrSleep bool) *Waker { + for { + w := s.nextWaker(block, wakepOrSleep) + if w == nil { + return nil + } + + // Reassociate the waker with the sleeper. If the waker was + // still asserted we can return it, otherwise try the next one. + old := (*Sleeper)(atomic.SwapPointer(&w.s, usleeper(s))) + if old == &assertedSleeper { + return w + } + } +} + +// Fetch fetches the next wake-up notification. If a notification is +// immediately available, the asserted waker is returned immediately. +// Otherwise, the behavior depends on the value of 'block': if true, the +// current goroutine blocks until a notification arrives and returns the +// asserted waker; if false, nil will be returned. +// +// N.B. This method is *not* thread-safe. Only one goroutine at a time is +// allowed to call this method. +func (s *Sleeper) Fetch(block bool) *Waker { + return s.fetch(block, false /* wakepOrSleep */) +} + +// AssertAndFetch asserts the given waker and fetches the next wake-up notification. +// Note that this will always be blocking, since there is no value in joining a +// non-blocking operation. +// +// N.B. Like Fetch, this method is *not* thread-safe. This will also yield the current +// P to the next goroutine, avoiding associated scheduled overhead. +// +// +checkescape:all +// +//go:nosplit +func (s *Sleeper) AssertAndFetch(n *Waker) *Waker { + n.assert(false /* wakep */) + return s.fetch(true /* block */, true /* wakepOrSleep*/) +} + +// Done is used to indicate that the caller won't use this Sleeper anymore. It +// removes the association with all wakers so that they can be safely reused +// by another sleeper after Done() returns. +func (s *Sleeper) Done() { + // Remove all associations that we can, and build a list of the ones we + // could not. An association can be removed right away from waker w if + // w.s has a pointer to the sleeper, that is, the waker is not asserted + // yet. By atomically switching w.s to nil, we guarantee that + // subsequent calls to Assert() on the waker will not result in it + // being queued. + for w := s.allWakers; w != nil; w = s.allWakers { + next := w.allWakersNext // Before zapping. + if atomic.CompareAndSwapPointer(&w.s, usleeper(s), nil) { + w.allWakersNext = nil + w.next = nil + s.allWakers = next // Move ahead. + continue + } + + // Dequeue exactly one waiter from the list, it may not be + // this one but we know this one is in the process. We must + // leave it in the asserted state but drop it from our lists. + if w := s.nextWaker(true, false); w != nil { + prev := &s.allWakers + for *prev != w { + prev = &((*prev).allWakersNext) + } + *prev = (*prev).allWakersNext + w.allWakersNext = nil + w.next = nil + } + } +} + +// enqueueAssertedWaker enqueues an asserted waker to the "ready" circular list +// of wakers that want to notify the sleeper. +// +//go:nosplit +func (s *Sleeper) enqueueAssertedWaker(w *Waker, wakep bool) { + // Add the new waker to the front of the list. + for { + v := (*Waker)(atomic.LoadPointer(&s.sharedList)) + w.next = v + if atomic.CompareAndSwapPointer(&s.sharedList, uwaker(v), uwaker(w)) { + break + } + } + + // Nothing to do if there isn't a G waiting. + if atomic.LoadUintptr(&s.waitingG) == 0 { + return + } + + // Signal to the sleeper that a waker has been asserted. + switch g := atomic.SwapUintptr(&s.waitingG, 0); g { + case 0, preparingG: + default: + // We managed to get a G. Wake it up. + sync.Goready(g, 0, wakep) + } +} + +// Waker represents a source of wake-up notifications to be sent to sleepers. A +// waker can be associated with at most one sleeper at a time, and at any given +// time is either in asserted or non-asserted state. +// +// Once asserted, the waker remains so until it is manually cleared or a sleeper +// consumes its assertion (i.e., a sleeper wakes up or is prevented from going +// to sleep due to the waker). +// +// This struct is thread-safe, that is, its methods can be called concurrently +// by multiple goroutines. +// +// Note, it is not safe to copy a Waker as its fields are modified by value +// (the pointer fields are individually modified with atomic operations). +// +// +stateify savable +type Waker struct { + _ sync.NoCopy + + // s is the sleeper that this waker can wake up. Only one sleeper at a + // time is allowed. This field can have three classes of values: + // nil -- the waker is not asserted: it either is not associated with + // a sleeper, or is queued to a sleeper due to being previously + // asserted. This is the zero value. + // &assertedSleeper -- the waker is asserted. + // otherwise -- the waker is not asserted, and is associated with the + // given sleeper. Once it transitions to asserted state, the + // associated sleeper will be woken. + s unsafe.Pointer `state:".(wakerState)"` + + // next is used to form a linked list of asserted wakers in a sleeper. + next *Waker + + // allWakersNext is used to form a linked list of all wakers associated + // to a given sleeper. + allWakersNext *Waker +} + +// +stateify savable +type wakerState struct { + asserted bool + other *Sleeper +} + +// saveS is invoked by stateify. +func (w *Waker) saveS() wakerState { + s := (*Sleeper)(atomic.LoadPointer(&w.s)) + if s == &assertedSleeper { + return wakerState{asserted: true} + } + return wakerState{other: s} +} + +// loadS is invoked by stateify. +func (w *Waker) loadS(_ context.Context, ws wakerState) { + if ws.asserted { + atomic.StorePointer(&w.s, unsafe.Pointer(&assertedSleeper)) + } else { + atomic.StorePointer(&w.s, unsafe.Pointer(ws.other)) + } +} + +// assert is the implementation for Assert. +// +//go:nosplit +func (w *Waker) assert(wakep bool) { + // Nothing to do if the waker is already asserted. This check allows us + // to complete this case (already asserted) without any interlocked + // operations on x86. + if atomic.LoadPointer(&w.s) == usleeper(&assertedSleeper) { + return + } + + // Mark the waker as asserted, and wake up a sleeper if there is one. + switch s := (*Sleeper)(atomic.SwapPointer(&w.s, usleeper(&assertedSleeper))); s { + case nil: + case &assertedSleeper: + default: + s.enqueueAssertedWaker(w, wakep) + } +} + +// Assert moves the waker to an asserted state, if it isn't asserted yet. When +// asserted, the waker will cause its matching sleeper to wake up. +func (w *Waker) Assert() { + w.assert(true /* wakep */) +} + +// Clear moves the waker to then non-asserted state and returns whether it was +// asserted before being cleared. +// +// N.B. The waker isn't removed from the "ready" list of a sleeper (if it +// happens to be in one), but the sleeper will notice that it is not asserted +// anymore and won't return it to the caller. +func (w *Waker) Clear() bool { + // Nothing to do if the waker is not asserted. This check allows us to + // complete this case (already not asserted) without any interlocked + // operations on x86. + if atomic.LoadPointer(&w.s) != usleeper(&assertedSleeper) { + return false + } + + // Try to store nil in the sleeper, which indicates that the waker is + // not asserted. + return atomic.CompareAndSwapPointer(&w.s, usleeper(&assertedSleeper), nil) +} + +// IsAsserted returns whether the waker is currently asserted (i.e., if it's +// currently in a state that would cause its matching sleeper to wake up). +func (w *Waker) IsAsserted() bool { + return (*Sleeper)(atomic.LoadPointer(&w.s)) == &assertedSleeper +} + +func usleeper(s *Sleeper) unsafe.Pointer { + return unsafe.Pointer(s) +} + +func uwaker(w *Waker) unsafe.Pointer { + return unsafe.Pointer(w) +} diff --git a/pkg/sleep/sleep_unsafe_state_autogen.go b/pkg/sleep/sleep_unsafe_state_autogen.go new file mode 100644 index 0000000..5ca2094 --- /dev/null +++ b/pkg/sleep/sleep_unsafe_state_autogen.go @@ -0,0 +1,109 @@ +// automatically generated by stateify. + +package sleep + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (s *Sleeper) StateTypeName() string { + return "pkg/sleep.Sleeper" +} + +func (s *Sleeper) StateFields() []string { + return []string{ + "sharedList", + "localList", + "allWakers", + } +} + +func (s *Sleeper) beforeSave() {} + +// +checklocksignore +func (s *Sleeper) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + var sharedListValue *Waker + sharedListValue = s.saveSharedList() + stateSinkObject.SaveValue(0, sharedListValue) + stateSinkObject.Save(1, &s.localList) + stateSinkObject.Save(2, &s.allWakers) +} + +func (s *Sleeper) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Sleeper) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(1, &s.localList) + stateSourceObject.Load(2, &s.allWakers) + stateSourceObject.LoadValue(0, new(*Waker), func(y any) { s.loadSharedList(ctx, y.(*Waker)) }) +} + +func (w *Waker) StateTypeName() string { + return "pkg/sleep.Waker" +} + +func (w *Waker) StateFields() []string { + return []string{ + "s", + "next", + "allWakersNext", + } +} + +func (w *Waker) beforeSave() {} + +// +checklocksignore +func (w *Waker) StateSave(stateSinkObject state.Sink) { + w.beforeSave() + var sValue wakerState + sValue = w.saveS() + stateSinkObject.SaveValue(0, sValue) + stateSinkObject.Save(1, &w.next) + stateSinkObject.Save(2, &w.allWakersNext) +} + +func (w *Waker) afterLoad(context.Context) {} + +// +checklocksignore +func (w *Waker) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(1, &w.next) + stateSourceObject.Load(2, &w.allWakersNext) + stateSourceObject.LoadValue(0, new(wakerState), func(y any) { w.loadS(ctx, y.(wakerState)) }) +} + +func (w *wakerState) StateTypeName() string { + return "pkg/sleep.wakerState" +} + +func (w *wakerState) StateFields() []string { + return []string{ + "asserted", + "other", + } +} + +func (w *wakerState) beforeSave() {} + +// +checklocksignore +func (w *wakerState) StateSave(stateSinkObject state.Sink) { + w.beforeSave() + stateSinkObject.Save(0, &w.asserted) + stateSinkObject.Save(1, &w.other) +} + +func (w *wakerState) afterLoad(context.Context) {} + +// +checklocksignore +func (w *wakerState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &w.asserted) + stateSourceObject.Load(1, &w.other) +} + +func init() { + state.Register((*Sleeper)(nil)) + state.Register((*Waker)(nil)) + state.Register((*wakerState)(nil)) +} diff --git a/pkg/state/addr_range.go b/pkg/state/addr_range.go new file mode 100644 index 0000000..0b7346e --- /dev/null +++ b/pkg/state/addr_range.go @@ -0,0 +1,76 @@ +package state + +// A Range represents a contiguous range of T. +// +// +stateify savable +type addrRange struct { + // Start is the inclusive start of the range. + Start uintptr + + // End is the exclusive end of the range. + End uintptr +} + +// WellFormed returns true if r.Start <= r.End. All other methods on a Range +// require that the Range is well-formed. +// +//go:nosplit +func (r addrRange) WellFormed() bool { + return r.Start <= r.End +} + +// Length returns the length of the range. +// +//go:nosplit +func (r addrRange) Length() uintptr { + return r.End - r.Start +} + +// Contains returns true if r contains x. +// +//go:nosplit +func (r addrRange) Contains(x uintptr) bool { + return r.Start <= x && x < r.End +} + +// Overlaps returns true if r and r2 overlap. +// +//go:nosplit +func (r addrRange) Overlaps(r2 addrRange) bool { + return r.Start < r2.End && r2.Start < r.End +} + +// IsSupersetOf returns true if r is a superset of r2; that is, the range r2 is +// contained within r. +// +//go:nosplit +func (r addrRange) IsSupersetOf(r2 addrRange) bool { + return r.Start <= r2.Start && r.End >= r2.End +} + +// Intersect returns a range consisting of the intersection between r and r2. +// If r and r2 do not overlap, Intersect returns a range with unspecified +// bounds, but for which Length() == 0. +// +//go:nosplit +func (r addrRange) Intersect(r2 addrRange) addrRange { + if r.Start < r2.Start { + r.Start = r2.Start + } + if r.End > r2.End { + r.End = r2.End + } + if r.End < r.Start { + r.End = r.Start + } + return r +} + +// CanSplitAt returns true if it is legal to split a segment spanning the range +// r at x; that is, splitting at x would produce two ranges, both of which have +// non-zero length. +// +//go:nosplit +func (r addrRange) CanSplitAt(x uintptr) bool { + return r.Contains(x) && r.Start < x +} diff --git a/pkg/state/addr_set.go b/pkg/state/addr_set.go new file mode 100644 index 0000000..1069baa --- /dev/null +++ b/pkg/state/addr_set.go @@ -0,0 +1,2017 @@ +package state + +import ( + "bytes" + "context" + "fmt" +) + +// trackGaps is an optional parameter. +// +// If trackGaps is 1, the Set will track maximum gap size recursively, +// enabling the GapIterator.{Prev,Next}LargeEnoughGap functions. In this +// case, Key must be an unsigned integer. +// +// trackGaps must be 0 or 1. +const addrtrackGaps = 0 + +var _ = uint8(addrtrackGaps << 7) // Will fail if not zero or one. + +// dynamicGap is a type that disappears if trackGaps is 0. +type addrdynamicGap [addrtrackGaps]uintptr + +// Get returns the value of the gap. +// +// Precondition: trackGaps must be non-zero. +func (d *addrdynamicGap) Get() uintptr { + return d[:][0] +} + +// Set sets the value of the gap. +// +// Precondition: trackGaps must be non-zero. +func (d *addrdynamicGap) Set(v uintptr) { + d[:][0] = v +} + +const ( + // minDegree is the minimum degree of an internal node in a Set B-tree. + // + // - Any non-root node has at least minDegree-1 segments. + // + // - Any non-root internal (non-leaf) node has at least minDegree children. + // + // - The root node may have fewer than minDegree-1 segments, but it may + // only have 0 segments if the tree is empty. + // + // Our implementation requires minDegree >= 3. Higher values of minDegree + // usually improve performance, but increase memory usage for small sets. + addrminDegree = 10 + + addrmaxDegree = 2 * addrminDegree +) + +// A Set is a mapping of segments with non-overlapping Range keys. The zero +// value for a Set is an empty set. Set values are not safely movable nor +// copyable. Set is thread-compatible. +// +// +stateify savable +type addrSet struct { + root addrnode `state:".([]addrFlatSegment)"` +} + +// IsEmpty returns true if the set contains no segments. +func (s *addrSet) IsEmpty() bool { + return s.root.nrSegments == 0 +} + +// IsEmptyRange returns true iff no segments in the set overlap the given +// range. This is semantically equivalent to s.SpanRange(r) == 0, but may be +// more efficient. +func (s *addrSet) IsEmptyRange(r addrRange) bool { + switch { + case r.Length() < 0: + panic(fmt.Sprintf("invalid range %v", r)) + case r.Length() == 0: + return true + } + _, gap := s.Find(r.Start) + if !gap.Ok() { + return false + } + return r.End <= gap.End() +} + +// Span returns the total size of all segments in the set. +func (s *addrSet) Span() uintptr { + var sz uintptr + for seg := s.FirstSegment(); seg.Ok(); seg = seg.NextSegment() { + sz += seg.Range().Length() + } + return sz +} + +// SpanRange returns the total size of the intersection of segments in the set +// with the given range. +func (s *addrSet) SpanRange(r addrRange) uintptr { + switch { + case r.Length() < 0: + panic(fmt.Sprintf("invalid range %v", r)) + case r.Length() == 0: + return 0 + } + var sz uintptr + for seg := s.LowerBoundSegment(r.Start); seg.Ok() && seg.Start() < r.End; seg = seg.NextSegment() { + sz += seg.Range().Intersect(r).Length() + } + return sz +} + +// FirstSegment returns the first segment in the set. If the set is empty, +// FirstSegment returns a terminal iterator. +func (s *addrSet) FirstSegment() addrIterator { + if s.root.nrSegments == 0 { + return addrIterator{} + } + return s.root.firstSegment() +} + +// LastSegment returns the last segment in the set. If the set is empty, +// LastSegment returns a terminal iterator. +func (s *addrSet) LastSegment() addrIterator { + if s.root.nrSegments == 0 { + return addrIterator{} + } + return s.root.lastSegment() +} + +// FirstGap returns the first gap in the set. +func (s *addrSet) FirstGap() addrGapIterator { + n := &s.root + for n.hasChildren { + n = n.children[0] + } + return addrGapIterator{n, 0} +} + +// LastGap returns the last gap in the set. +func (s *addrSet) LastGap() addrGapIterator { + n := &s.root + for n.hasChildren { + n = n.children[n.nrSegments] + } + return addrGapIterator{n, n.nrSegments} +} + +// Find returns the segment or gap whose range contains the given key. If a +// segment is found, the returned Iterator is non-terminal and the +// returned GapIterator is terminal. Otherwise, the returned Iterator is +// terminal and the returned GapIterator is non-terminal. +func (s *addrSet) Find(key uintptr) (addrIterator, addrGapIterator) { + n := &s.root + for { + + lower := 0 + upper := n.nrSegments + for lower < upper { + i := lower + (upper-lower)/2 + if r := n.keys[i]; key < r.End { + if key >= r.Start { + return addrIterator{n, i}, addrGapIterator{} + } + upper = i + } else { + lower = i + 1 + } + } + i := lower + if !n.hasChildren { + return addrIterator{}, addrGapIterator{n, i} + } + n = n.children[i] + } +} + +// FindSegment returns the segment whose range contains the given key. If no +// such segment exists, FindSegment returns a terminal iterator. +func (s *addrSet) FindSegment(key uintptr) addrIterator { + seg, _ := s.Find(key) + return seg +} + +// LowerBoundSegment returns the segment with the lowest range that contains a +// key greater than or equal to min. If no such segment exists, +// LowerBoundSegment returns a terminal iterator. +func (s *addrSet) LowerBoundSegment(min uintptr) addrIterator { + seg, gap := s.Find(min) + if seg.Ok() { + return seg + } + return gap.NextSegment() +} + +// UpperBoundSegment returns the segment with the highest range that contains a +// key less than or equal to max. If no such segment exists, UpperBoundSegment +// returns a terminal iterator. +func (s *addrSet) UpperBoundSegment(max uintptr) addrIterator { + seg, gap := s.Find(max) + if seg.Ok() { + return seg + } + return gap.PrevSegment() +} + +// FindGap returns the gap containing the given key. If no such gap exists +// (i.e. the set contains a segment containing that key), FindGap returns a +// terminal iterator. +func (s *addrSet) FindGap(key uintptr) addrGapIterator { + _, gap := s.Find(key) + return gap +} + +// LowerBoundGap returns the gap with the lowest range that is greater than or +// equal to min. +func (s *addrSet) LowerBoundGap(min uintptr) addrGapIterator { + seg, gap := s.Find(min) + if gap.Ok() { + return gap + } + return seg.NextGap() +} + +// UpperBoundGap returns the gap with the highest range that is less than or +// equal to max. +func (s *addrSet) UpperBoundGap(max uintptr) addrGapIterator { + seg, gap := s.Find(max) + if gap.Ok() { + return gap + } + return seg.PrevGap() +} + +// FirstLargeEnoughGap returns the first gap in the set with at least the given +// length. If no such gap exists, FirstLargeEnoughGap returns a terminal +// iterator. +// +// Precondition: trackGaps must be 1. +func (s *addrSet) FirstLargeEnoughGap(minSize uintptr) addrGapIterator { + if addrtrackGaps != 1 { + panic("set is not tracking gaps") + } + gap := s.FirstGap() + if gap.Range().Length() >= minSize { + return gap + } + return gap.NextLargeEnoughGap(minSize) +} + +// LastLargeEnoughGap returns the last gap in the set with at least the given +// length. If no such gap exists, LastLargeEnoughGap returns a terminal +// iterator. +// +// Precondition: trackGaps must be 1. +func (s *addrSet) LastLargeEnoughGap(minSize uintptr) addrGapIterator { + if addrtrackGaps != 1 { + panic("set is not tracking gaps") + } + gap := s.LastGap() + if gap.Range().Length() >= minSize { + return gap + } + return gap.PrevLargeEnoughGap(minSize) +} + +// LowerBoundLargeEnoughGap returns the first gap in the set with at least the +// given length and whose range contains a key greater than or equal to min. If +// no such gap exists, LowerBoundLargeEnoughGap returns a terminal iterator. +// +// Precondition: trackGaps must be 1. +func (s *addrSet) LowerBoundLargeEnoughGap(min, minSize uintptr) addrGapIterator { + if addrtrackGaps != 1 { + panic("set is not tracking gaps") + } + gap := s.LowerBoundGap(min) + if gap.Range().Length() >= minSize { + return gap + } + return gap.NextLargeEnoughGap(minSize) +} + +// UpperBoundLargeEnoughGap returns the last gap in the set with at least the +// given length and whose range contains a key less than or equal to max. If no +// such gap exists, UpperBoundLargeEnoughGap returns a terminal iterator. +// +// Precondition: trackGaps must be 1. +func (s *addrSet) UpperBoundLargeEnoughGap(max, minSize uintptr) addrGapIterator { + if addrtrackGaps != 1 { + panic("set is not tracking gaps") + } + gap := s.UpperBoundGap(max) + if gap.Range().Length() >= minSize { + return gap + } + return gap.PrevLargeEnoughGap(minSize) +} + +// Insert inserts the given segment into the given gap. If the new segment can +// be merged with adjacent segments, Insert will do so. Insert returns an +// iterator to the segment containing the inserted value (which may have been +// merged with other values). All existing iterators (including gap, but not +// including the returned iterator) are invalidated. +// +// If the gap cannot accommodate the segment, or if r is invalid, Insert panics. +// +// Insert is semantically equivalent to a InsertWithoutMerging followed by a +// Merge, but may be more efficient. Note that there is no unchecked variant of +// Insert since Insert must retrieve and inspect gap's predecessor and +// successor segments regardless. +func (s *addrSet) Insert(gap addrGapIterator, r addrRange, val *objectEncodeState) addrIterator { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + prev, next := gap.PrevSegment(), gap.NextSegment() + if prev.Ok() && prev.End() > r.Start { + panic(fmt.Sprintf("new segment %v overlaps predecessor %v", r, prev.Range())) + } + if next.Ok() && next.Start() < r.End { + panic(fmt.Sprintf("new segment %v overlaps successor %v", r, next.Range())) + } + if prev.Ok() && prev.End() == r.Start { + if mval, ok := (addrSetFunctions{}).Merge(prev.Range(), prev.Value(), r, val); ok { + shrinkMaxGap := addrtrackGaps != 0 && gap.Range().Length() == gap.node.maxGap.Get() + prev.SetEndUnchecked(r.End) + prev.SetValue(mval) + if shrinkMaxGap { + gap.node.updateMaxGapLeaf() + } + if next.Ok() && next.Start() == r.End { + val = mval + if mval, ok := (addrSetFunctions{}).Merge(prev.Range(), val, next.Range(), next.Value()); ok { + prev.SetEndUnchecked(next.End()) + prev.SetValue(mval) + return s.Remove(next).PrevSegment() + } + } + return prev + } + } + if next.Ok() && next.Start() == r.End { + if mval, ok := (addrSetFunctions{}).Merge(r, val, next.Range(), next.Value()); ok { + shrinkMaxGap := addrtrackGaps != 0 && gap.Range().Length() == gap.node.maxGap.Get() + next.SetStartUnchecked(r.Start) + next.SetValue(mval) + if shrinkMaxGap { + gap.node.updateMaxGapLeaf() + } + return next + } + } + + return s.InsertWithoutMergingUnchecked(gap, r, val) +} + +// InsertWithoutMerging inserts the given segment into the given gap and +// returns an iterator to the inserted segment. All existing iterators +// (including gap, but not including the returned iterator) are invalidated. +// +// If the gap cannot accommodate the segment, or if r is invalid, +// InsertWithoutMerging panics. +func (s *addrSet) InsertWithoutMerging(gap addrGapIterator, r addrRange, val *objectEncodeState) addrIterator { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + if gr := gap.Range(); !gr.IsSupersetOf(r) { + panic(fmt.Sprintf("cannot insert segment range %v into gap range %v", r, gr)) + } + return s.InsertWithoutMergingUnchecked(gap, r, val) +} + +// InsertWithoutMergingUnchecked inserts the given segment into the given gap +// and returns an iterator to the inserted segment. All existing iterators +// (including gap, but not including the returned iterator) are invalidated. +// +// Preconditions: +// - r.Start >= gap.Start(). +// - r.End <= gap.End(). +func (s *addrSet) InsertWithoutMergingUnchecked(gap addrGapIterator, r addrRange, val *objectEncodeState) addrIterator { + gap = gap.node.rebalanceBeforeInsert(gap) + splitMaxGap := addrtrackGaps != 0 && (gap.node.nrSegments == 0 || gap.Range().Length() == gap.node.maxGap.Get()) + copy(gap.node.keys[gap.index+1:], gap.node.keys[gap.index:gap.node.nrSegments]) + copy(gap.node.values[gap.index+1:], gap.node.values[gap.index:gap.node.nrSegments]) + gap.node.keys[gap.index] = r + gap.node.values[gap.index] = val + gap.node.nrSegments++ + if splitMaxGap { + gap.node.updateMaxGapLeaf() + } + return addrIterator{gap.node, gap.index} +} + +// InsertRange inserts the given segment into the set. If the new segment can +// be merged with adjacent segments, InsertRange will do so. InsertRange +// returns an iterator to the segment containing the inserted value (which may +// have been merged with other values). All existing iterators (excluding the +// returned iterator) are invalidated. +// +// If the new segment would overlap an existing segment, or if r is invalid, +// InsertRange panics. +// +// InsertRange searches the set to find the gap to insert into. If the caller +// already has the appropriate GapIterator, or if the caller needs to do +// additional work between finding the gap and insertion, use Insert instead. +func (s *addrSet) InsertRange(r addrRange, val *objectEncodeState) addrIterator { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + seg, gap := s.Find(r.Start) + if seg.Ok() { + panic(fmt.Sprintf("new segment %v overlaps existing segment %v", r, seg.Range())) + } + if gap.End() < r.End { + panic(fmt.Sprintf("new segment %v overlaps existing segment %v", r, gap.NextSegment().Range())) + } + return s.Insert(gap, r, val) +} + +// InsertWithoutMergingRange inserts the given segment into the set and returns +// an iterator to the inserted segment. All existing iterators (excluding the +// returned iterator) are invalidated. +// +// If the new segment would overlap an existing segment, or if r is invalid, +// InsertWithoutMergingRange panics. +// +// InsertWithoutMergingRange searches the set to find the gap to insert into. +// If the caller already has the appropriate GapIterator, or if the caller +// needs to do additional work between finding the gap and insertion, use +// InsertWithoutMerging instead. +func (s *addrSet) InsertWithoutMergingRange(r addrRange, val *objectEncodeState) addrIterator { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + seg, gap := s.Find(r.Start) + if seg.Ok() { + panic(fmt.Sprintf("new segment %v overlaps existing segment %v", r, seg.Range())) + } + if gap.End() < r.End { + panic(fmt.Sprintf("new segment %v overlaps existing segment %v", r, gap.NextSegment().Range())) + } + return s.InsertWithoutMerging(gap, r, val) +} + +// TryInsertRange attempts to insert the given segment into the set. If the new +// segment can be merged with adjacent segments, TryInsertRange will do so. +// TryInsertRange returns an iterator to the segment containing the inserted +// value (which may have been merged with other values). All existing iterators +// (excluding the returned iterator) are invalidated. +// +// If the new segment would overlap an existing segment, TryInsertRange does +// nothing and returns a terminal iterator. +// +// TryInsertRange searches the set to find the gap to insert into. If the +// caller already has the appropriate GapIterator, or if the caller needs to do +// additional work between finding the gap and insertion, use Insert instead. +func (s *addrSet) TryInsertRange(r addrRange, val *objectEncodeState) addrIterator { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + seg, gap := s.Find(r.Start) + if seg.Ok() { + return addrIterator{} + } + if gap.End() < r.End { + return addrIterator{} + } + return s.Insert(gap, r, val) +} + +// TryInsertWithoutMergingRange attempts to insert the given segment into the +// set. If successful, it returns an iterator to the inserted segment; all +// existing iterators (excluding the returned iterator) are invalidated. If the +// new segment would overlap an existing segment, TryInsertWithoutMergingRange +// does nothing and returns a terminal iterator. +// +// TryInsertWithoutMergingRange searches the set to find the gap to insert +// into. If the caller already has the appropriate GapIterator, or if the +// caller needs to do additional work between finding the gap and insertion, +// use InsertWithoutMerging instead. +func (s *addrSet) TryInsertWithoutMergingRange(r addrRange, val *objectEncodeState) addrIterator { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + seg, gap := s.Find(r.Start) + if seg.Ok() { + return addrIterator{} + } + if gap.End() < r.End { + return addrIterator{} + } + return s.InsertWithoutMerging(gap, r, val) +} + +// Remove removes the given segment and returns an iterator to the vacated gap. +// All existing iterators (including seg, but not including the returned +// iterator) are invalidated. +func (s *addrSet) Remove(seg addrIterator) addrGapIterator { + if seg.node.hasChildren { + + victim := seg.PrevSegment() + + seg.SetRangeUnchecked(victim.Range()) + seg.SetValue(victim.Value()) + + nextAdjacentNode := seg.NextSegment().node + if addrtrackGaps != 0 { + nextAdjacentNode.updateMaxGapLeaf() + } + return s.Remove(victim).NextGap() + } + copy(seg.node.keys[seg.index:], seg.node.keys[seg.index+1:seg.node.nrSegments]) + copy(seg.node.values[seg.index:], seg.node.values[seg.index+1:seg.node.nrSegments]) + addrSetFunctions{}.ClearValue(&seg.node.values[seg.node.nrSegments-1]) + seg.node.nrSegments-- + if addrtrackGaps != 0 { + seg.node.updateMaxGapLeaf() + } + return seg.node.rebalanceAfterRemove(addrGapIterator{seg.node, seg.index}) +} + +// RemoveAll removes all segments from the set. All existing iterators are +// invalidated. +func (s *addrSet) RemoveAll() { + s.root = addrnode{} +} + +// RemoveRange removes all segments in the given range. An iterator to the +// newly formed gap is returned, and all existing iterators are invalidated. +// +// RemoveRange searches the set to find segments to remove. If the caller +// already has an iterator to either end of the range of segments to remove, or +// if the caller needs to do additional work before removing each segment, +// iterate segments and call Remove in a loop instead. +func (s *addrSet) RemoveRange(r addrRange) addrGapIterator { + return s.RemoveRangeWith(r, nil) +} + +// RemoveFullRange is equivalent to RemoveRange, except that if any key in the +// given range does not correspond to a segment, RemoveFullRange panics. +func (s *addrSet) RemoveFullRange(r addrRange) addrGapIterator { + return s.RemoveFullRangeWith(r, nil) +} + +// RemoveRangeWith removes all segments in the given range. An iterator to the +// newly formed gap is returned, and all existing iterators are invalidated. +// +// The function f is applied to each segment immediately before it is removed, +// in order of ascending keys. Segments that lie partially outside r are split +// before f is called, such that f only observes segments entirely within r. +// Non-empty gaps between segments are skipped. +// +// RemoveRangeWith searches the set to find segments to remove. If the caller +// already has an iterator to either end of the range of segments to remove, or +// if the caller needs to do additional work before removing each segment, +// iterate segments and call Remove in a loop instead. +// +// N.B. f must not invalidate iterators into s. +func (s *addrSet) RemoveRangeWith(r addrRange, f func(seg addrIterator)) addrGapIterator { + seg, gap := s.Find(r.Start) + if seg.Ok() { + seg = s.Isolate(seg, r) + if f != nil { + f(seg) + } + gap = s.Remove(seg) + } + for seg = gap.NextSegment(); seg.Ok() && seg.Start() < r.End; seg = gap.NextSegment() { + seg = s.SplitAfter(seg, r.End) + if f != nil { + f(seg) + } + gap = s.Remove(seg) + } + return gap +} + +// RemoveFullRangeWith is equivalent to RemoveRangeWith, except that if any key +// in the given range does not correspond to a segment, RemoveFullRangeWith +// panics. +func (s *addrSet) RemoveFullRangeWith(r addrRange, f func(seg addrIterator)) addrGapIterator { + seg := s.FindSegment(r.Start) + if !seg.Ok() { + panic(fmt.Sprintf("missing segment at %v", r.Start)) + } + seg = s.SplitBefore(seg, r.Start) + for { + seg = s.SplitAfter(seg, r.End) + if f != nil { + f(seg) + } + end := seg.End() + gap := s.Remove(seg) + if r.End <= end { + return gap + } + seg = gap.NextSegment() + if !seg.Ok() || seg.Start() != end { + panic(fmt.Sprintf("missing segment at %v", end)) + } + } +} + +// Merge attempts to merge two neighboring segments. If successful, Merge +// returns an iterator to the merged segment, and all existing iterators are +// invalidated. Otherwise, Merge returns a terminal iterator. +// +// If first is not the predecessor of second, Merge panics. +func (s *addrSet) Merge(first, second addrIterator) addrIterator { + if first.NextSegment() != second { + panic(fmt.Sprintf("attempt to merge non-neighboring segments %v, %v", first.Range(), second.Range())) + } + return s.MergeUnchecked(first, second) +} + +// MergeUnchecked attempts to merge two neighboring segments. If successful, +// MergeUnchecked returns an iterator to the merged segment, and all existing +// iterators are invalidated. Otherwise, MergeUnchecked returns a terminal +// iterator. +// +// Precondition: first is the predecessor of second: first.NextSegment() == +// second, first == second.PrevSegment(). +func (s *addrSet) MergeUnchecked(first, second addrIterator) addrIterator { + if first.End() == second.Start() { + if mval, ok := (addrSetFunctions{}).Merge(first.Range(), first.Value(), second.Range(), second.Value()); ok { + + first.SetEndUnchecked(second.End()) + first.SetValue(mval) + + return s.Remove(second).PrevSegment() + } + } + return addrIterator{} +} + +// MergePrev attempts to merge the given segment with its predecessor if +// possible, and returns an updated iterator to the extended segment. All +// existing iterators (including seg, but not including the returned iterator) +// are invalidated. +// +// MergePrev is usually used when mutating segments while iterating them in +// order of increasing keys, to attempt merging of each mutated segment with +// its previously-mutated predecessor. In such cases, merging a mutated segment +// with its unmutated successor would incorrectly cause the latter to be +// skipped. +func (s *addrSet) MergePrev(seg addrIterator) addrIterator { + if prev := seg.PrevSegment(); prev.Ok() { + if mseg := s.MergeUnchecked(prev, seg); mseg.Ok() { + seg = mseg + } + } + return seg +} + +// MergeNext attempts to merge the given segment with its successor if +// possible, and returns an updated iterator to the extended segment. All +// existing iterators (including seg, but not including the returned iterator) +// are invalidated. +// +// MergeNext is usually used when mutating segments while iterating them in +// order of decreasing keys, to attempt merging of each mutated segment with +// its previously-mutated successor. In such cases, merging a mutated segment +// with its unmutated predecessor would incorrectly cause the latter to be +// skipped. +func (s *addrSet) MergeNext(seg addrIterator) addrIterator { + if next := seg.NextSegment(); next.Ok() { + if mseg := s.MergeUnchecked(seg, next); mseg.Ok() { + seg = mseg + } + } + return seg +} + +// Unisolate attempts to merge the given segment with its predecessor and +// successor if possible, and returns an updated iterator to the extended +// segment. All existing iterators (including seg, but not including the +// returned iterator) are invalidated. +// +// Unisolate is usually used in conjunction with Isolate when mutating part of +// a single segment in a way that may affect its mergeability. For the reasons +// described by MergePrev and MergeNext, it is usually incorrect to use the +// return value of Unisolate in a loop variable. +func (s *addrSet) Unisolate(seg addrIterator) addrIterator { + if prev := seg.PrevSegment(); prev.Ok() { + if mseg := s.MergeUnchecked(prev, seg); mseg.Ok() { + seg = mseg + } + } + if next := seg.NextSegment(); next.Ok() { + if mseg := s.MergeUnchecked(seg, next); mseg.Ok() { + seg = mseg + } + } + return seg +} + +// MergeAll merges all mergeable adjacent segments in the set. All existing +// iterators are invalidated. +func (s *addrSet) MergeAll() { + seg := s.FirstSegment() + if !seg.Ok() { + return + } + next := seg.NextSegment() + for next.Ok() { + if mseg := s.MergeUnchecked(seg, next); mseg.Ok() { + seg, next = mseg, mseg.NextSegment() + } else { + seg, next = next, next.NextSegment() + } + } +} + +// MergeInsideRange attempts to merge all adjacent segments that contain a key +// in the specific range. All existing iterators are invalidated. +// +// MergeInsideRange only makes sense after mutating the set in a way that may +// change the mergeability of modified segments; callers should prefer to use +// MergePrev or MergeNext during the mutating loop instead (depending on the +// direction of iteration), in order to avoid a redundant search. +func (s *addrSet) MergeInsideRange(r addrRange) { + seg := s.LowerBoundSegment(r.Start) + if !seg.Ok() { + return + } + next := seg.NextSegment() + for next.Ok() && next.Start() < r.End { + if mseg := s.MergeUnchecked(seg, next); mseg.Ok() { + seg, next = mseg, mseg.NextSegment() + } else { + seg, next = next, next.NextSegment() + } + } +} + +// MergeOutsideRange attempts to merge the segment containing r.Start with its +// predecessor, and the segment containing r.End-1 with its successor. +// +// MergeOutsideRange only makes sense after mutating the set in a way that may +// change the mergeability of modified segments; callers should prefer to use +// MergePrev or MergeNext during the mutating loop instead (depending on the +// direction of iteration), in order to avoid two redundant searches. +func (s *addrSet) MergeOutsideRange(r addrRange) { + first := s.FindSegment(r.Start) + if first.Ok() { + if prev := first.PrevSegment(); prev.Ok() { + s.Merge(prev, first) + } + } + last := s.FindSegment(r.End - 1) + if last.Ok() { + if next := last.NextSegment(); next.Ok() { + s.Merge(last, next) + } + } +} + +// Split splits the given segment at the given key and returns iterators to the +// two resulting segments. All existing iterators (including seg, but not +// including the returned iterators) are invalidated. +// +// If the segment cannot be split at split (because split is at the start or +// end of the segment's range, so splitting would produce a segment with zero +// length, or because split falls outside the segment's range altogether), +// Split panics. +func (s *addrSet) Split(seg addrIterator, split uintptr) (addrIterator, addrIterator) { + if !seg.Range().CanSplitAt(split) { + panic(fmt.Sprintf("can't split %v at %v", seg.Range(), split)) + } + return s.SplitUnchecked(seg, split) +} + +// SplitUnchecked splits the given segment at the given key and returns +// iterators to the two resulting segments. All existing iterators (including +// seg, but not including the returned iterators) are invalidated. +// +// Preconditions: seg.Start() < key < seg.End(). +func (s *addrSet) SplitUnchecked(seg addrIterator, split uintptr) (addrIterator, addrIterator) { + val1, val2 := (addrSetFunctions{}).Split(seg.Range(), seg.Value(), split) + end2 := seg.End() + seg.SetEndUnchecked(split) + seg.SetValue(val1) + seg2 := s.InsertWithoutMergingUnchecked(seg.NextGap(), addrRange{split, end2}, val2) + + return seg2.PrevSegment(), seg2 +} + +// SplitBefore ensures that the given segment's start is at least start by +// splitting at start if necessary, and returns an updated iterator to the +// bounded segment. All existing iterators (including seg, but not including +// the returned iterator) are invalidated. +// +// SplitBefore is usually when mutating segments in a range. In such cases, +// when iterating segments in order of increasing keys, the first segment may +// extend beyond the start of the range to be mutated, and needs to be +// SplitBefore to ensure that only the part of the segment within the range is +// mutated. When iterating segments in order of decreasing keys, SplitBefore +// and SplitAfter; i.e. SplitBefore needs to be invoked on each segment, while +// SplitAfter only needs to be invoked on the first. +// +// Preconditions: start < seg.End(). +func (s *addrSet) SplitBefore(seg addrIterator, start uintptr) addrIterator { + if seg.Range().CanSplitAt(start) { + _, seg = s.SplitUnchecked(seg, start) + } + return seg +} + +// SplitAfter ensures that the given segment's end is at most end by splitting +// at end if necessary, and returns an updated iterator to the bounded segment. +// All existing iterators (including seg, but not including the returned +// iterator) are invalidated. +// +// SplitAfter is usually used when mutating segments in a range. In such cases, +// when iterating segments in order of increasing keys, each iterated segment +// may extend beyond the end of the range to be mutated, and needs to be +// SplitAfter to ensure that only the part of the segment within the range is +// mutated. When iterating segments in order of decreasing keys, SplitBefore +// and SplitAfter exchange roles; i.e. SplitBefore needs to be invoked on each +// segment, while SplitAfter only needs to be invoked on the first. +// +// Preconditions: seg.Start() < end. +func (s *addrSet) SplitAfter(seg addrIterator, end uintptr) addrIterator { + if seg.Range().CanSplitAt(end) { + seg, _ = s.SplitUnchecked(seg, end) + } + return seg +} + +// Isolate ensures that the given segment's range is a subset of r by splitting +// at r.Start and r.End if necessary, and returns an updated iterator to the +// bounded segment. All existing iterators (including seg, but not including +// the returned iterators) are invalidated. +// +// Isolate is usually used when mutating part of a single segment, or when +// mutating segments in a range where the first segment is not necessarily +// split, making use of SplitBefore/SplitAfter complex. +// +// Preconditions: seg.Range().Overlaps(r). +func (s *addrSet) Isolate(seg addrIterator, r addrRange) addrIterator { + if seg.Range().CanSplitAt(r.Start) { + _, seg = s.SplitUnchecked(seg, r.Start) + } + if seg.Range().CanSplitAt(r.End) { + seg, _ = s.SplitUnchecked(seg, r.End) + } + return seg +} + +// LowerBoundSegmentSplitBefore combines LowerBoundSegment and SplitBefore. +// +// LowerBoundSegmentSplitBefore is usually used when mutating segments in a +// range while iterating them in order of increasing keys. In such cases, +// LowerBoundSegmentSplitBefore provides an iterator to the first segment to be +// mutated, suitable as the initial value for a loop variable. +func (s *addrSet) LowerBoundSegmentSplitBefore(min uintptr) addrIterator { + seg, gap := s.Find(min) + if seg.Ok() { + return s.SplitBefore(seg, min) + } + return gap.NextSegment() +} + +// UpperBoundSegmentSplitAfter combines UpperBoundSegment and SplitAfter. +// +// UpperBoundSegmentSplitAfter is usually used when mutating segments in a +// range while iterating them in order of decreasing keys. In such cases, +// UpperBoundSegmentSplitAfter provides an iterator to the first segment to be +// mutated, suitable as the initial value for a loop variable. +func (s *addrSet) UpperBoundSegmentSplitAfter(max uintptr) addrIterator { + seg, gap := s.Find(max) + if seg.Ok() { + return s.SplitAfter(seg, max) + } + return gap.PrevSegment() +} + +// VisitRange applies the function f to all segments intersecting the range r, +// in order of ascending keys. Segments will not be split, so f may be called +// on segments lying partially outside r. Non-empty gaps between segments are +// skipped. If a call to f returns false, VisitRange stops iteration +// immediately. +// +// N.B. f must not invalidate iterators into s. +func (s *addrSet) VisitRange(r addrRange, f func(seg addrIterator) bool) { + for seg := s.LowerBoundSegment(r.Start); seg.Ok() && seg.Start() < r.End; seg = seg.NextSegment() { + if !f(seg) { + return + } + } +} + +// VisitFullRange is equivalent to VisitRange, except that if any key in r that +// is visited before f returns false does not correspond to a segment, +// VisitFullRange panics. +func (s *addrSet) VisitFullRange(r addrRange, f func(seg addrIterator) bool) { + pos := r.Start + seg := s.FindSegment(r.Start) + for { + if !seg.Ok() { + panic(fmt.Sprintf("missing segment at %v", pos)) + } + if !f(seg) { + return + } + pos = seg.End() + if r.End <= pos { + return + } + seg, _ = seg.NextNonEmpty() + } +} + +// MutateRange applies the function f to all segments intersecting the range r, +// in order of ascending keys. Segments that lie partially outside r are split +// before f is called, such that f only observes segments entirely within r. +// Iterated segments are merged again after f is called. Non-empty gaps between +// segments are skipped. If a call to f returns false, MutateRange stops +// iteration immediately. +// +// MutateRange invalidates all existing iterators. +// +// N.B. f must not invalidate iterators into s. +func (s *addrSet) MutateRange(r addrRange, f func(seg addrIterator) bool) { + seg := s.LowerBoundSegmentSplitBefore(r.Start) + for seg.Ok() && seg.Start() < r.End { + seg = s.SplitAfter(seg, r.End) + cont := f(seg) + seg = s.MergePrev(seg) + if !cont { + s.MergeNext(seg) + return + } + seg = seg.NextSegment() + } + if seg.Ok() { + s.MergePrev(seg) + } +} + +// MutateFullRange is equivalent to MutateRange, except that if any key in r +// that is visited before f returns false does not correspond to a segment, +// MutateFullRange panics. +func (s *addrSet) MutateFullRange(r addrRange, f func(seg addrIterator) bool) { + seg := s.FindSegment(r.Start) + if !seg.Ok() { + panic(fmt.Sprintf("missing segment at %v", r.Start)) + } + seg = s.SplitBefore(seg, r.Start) + for { + seg = s.SplitAfter(seg, r.End) + cont := f(seg) + end := seg.End() + seg = s.MergePrev(seg) + if !cont || r.End <= end { + s.MergeNext(seg) + return + } + seg = seg.NextSegment() + if !seg.Ok() || seg.Start() != end { + panic(fmt.Sprintf("missing segment at %v", end)) + } + } +} + +// +stateify savable +type addrnode struct { + // An internal binary tree node looks like: + // + // K + // / \ + // Cl Cr + // + // where all keys in the subtree rooted by Cl (the left subtree) are less + // than K (the key of the parent node), and all keys in the subtree rooted + // by Cr (the right subtree) are greater than K. + // + // An internal B-tree node's indexes work out to look like: + // + // K0 K1 K2 ... Kn-1 + // / \/ \/ \ ... / \ + // C0 C1 C2 C3 ... Cn-1 Cn + // + // where n is nrSegments. + nrSegments int + + // parent is a pointer to this node's parent. If this node is root, parent + // is nil. + parent *addrnode + + // parentIndex is the index of this node in parent.children. + parentIndex int + + // Flag for internal nodes that is technically redundant with "children[0] + // != nil", but is stored in the first cache line. "hasChildren" rather + // than "isLeaf" because false must be the correct value for an empty root. + hasChildren bool + + // The longest gap within this node. If the node is a leaf, it's simply the + // maximum gap among all the (nrSegments+1) gaps formed by its nrSegments keys + // including the 0th and nrSegments-th gap possibly shared with its upper-level + // nodes; if it's a non-leaf node, it's the max of all children's maxGap. + maxGap addrdynamicGap + + // Nodes store keys and values in separate arrays to maximize locality in + // the common case (scanning keys for lookup). + keys [addrmaxDegree - 1]addrRange + values [addrmaxDegree - 1]*objectEncodeState + children [addrmaxDegree]*addrnode +} + +// firstSegment returns the first segment in the subtree rooted by n. +// +// Preconditions: n.nrSegments != 0. +func (n *addrnode) firstSegment() addrIterator { + for n.hasChildren { + n = n.children[0] + } + return addrIterator{n, 0} +} + +// lastSegment returns the last segment in the subtree rooted by n. +// +// Preconditions: n.nrSegments != 0. +func (n *addrnode) lastSegment() addrIterator { + for n.hasChildren { + n = n.children[n.nrSegments] + } + return addrIterator{n, n.nrSegments - 1} +} + +func (n *addrnode) prevSibling() *addrnode { + if n.parent == nil || n.parentIndex == 0 { + return nil + } + return n.parent.children[n.parentIndex-1] +} + +func (n *addrnode) nextSibling() *addrnode { + if n.parent == nil || n.parentIndex == n.parent.nrSegments { + return nil + } + return n.parent.children[n.parentIndex+1] +} + +// rebalanceBeforeInsert splits n and its ancestors if they are full, as +// required for insertion, and returns an updated iterator to the position +// represented by gap. +func (n *addrnode) rebalanceBeforeInsert(gap addrGapIterator) addrGapIterator { + if n.nrSegments < addrmaxDegree-1 { + return gap + } + if n.parent != nil { + gap = n.parent.rebalanceBeforeInsert(gap) + } + if n.parent == nil { + + left := &addrnode{ + nrSegments: addrminDegree - 1, + parent: n, + parentIndex: 0, + hasChildren: n.hasChildren, + } + right := &addrnode{ + nrSegments: addrminDegree - 1, + parent: n, + parentIndex: 1, + hasChildren: n.hasChildren, + } + copy(left.keys[:addrminDegree-1], n.keys[:addrminDegree-1]) + copy(left.values[:addrminDegree-1], n.values[:addrminDegree-1]) + copy(right.keys[:addrminDegree-1], n.keys[addrminDegree:]) + copy(right.values[:addrminDegree-1], n.values[addrminDegree:]) + n.keys[0], n.values[0] = n.keys[addrminDegree-1], n.values[addrminDegree-1] + addrzeroValueSlice(n.values[1:]) + if n.hasChildren { + copy(left.children[:addrminDegree], n.children[:addrminDegree]) + copy(right.children[:addrminDegree], n.children[addrminDegree:]) + addrzeroNodeSlice(n.children[2:]) + for i := 0; i < addrminDegree; i++ { + left.children[i].parent = left + left.children[i].parentIndex = i + right.children[i].parent = right + right.children[i].parentIndex = i + } + } + n.nrSegments = 1 + n.hasChildren = true + n.children[0] = left + n.children[1] = right + + if addrtrackGaps != 0 { + left.updateMaxGapLocal() + right.updateMaxGapLocal() + } + if gap.node != n { + return gap + } + if gap.index < addrminDegree { + return addrGapIterator{left, gap.index} + } + return addrGapIterator{right, gap.index - addrminDegree} + } + + copy(n.parent.keys[n.parentIndex+1:], n.parent.keys[n.parentIndex:n.parent.nrSegments]) + copy(n.parent.values[n.parentIndex+1:], n.parent.values[n.parentIndex:n.parent.nrSegments]) + n.parent.keys[n.parentIndex], n.parent.values[n.parentIndex] = n.keys[addrminDegree-1], n.values[addrminDegree-1] + copy(n.parent.children[n.parentIndex+2:], n.parent.children[n.parentIndex+1:n.parent.nrSegments+1]) + for i := n.parentIndex + 2; i < n.parent.nrSegments+2; i++ { + n.parent.children[i].parentIndex = i + } + sibling := &addrnode{ + nrSegments: addrminDegree - 1, + parent: n.parent, + parentIndex: n.parentIndex + 1, + hasChildren: n.hasChildren, + } + n.parent.children[n.parentIndex+1] = sibling + n.parent.nrSegments++ + copy(sibling.keys[:addrminDegree-1], n.keys[addrminDegree:]) + copy(sibling.values[:addrminDegree-1], n.values[addrminDegree:]) + addrzeroValueSlice(n.values[addrminDegree-1:]) + if n.hasChildren { + copy(sibling.children[:addrminDegree], n.children[addrminDegree:]) + addrzeroNodeSlice(n.children[addrminDegree:]) + for i := 0; i < addrminDegree; i++ { + sibling.children[i].parent = sibling + sibling.children[i].parentIndex = i + } + } + n.nrSegments = addrminDegree - 1 + + if addrtrackGaps != 0 { + n.updateMaxGapLocal() + sibling.updateMaxGapLocal() + } + + if gap.node != n { + return gap + } + if gap.index < addrminDegree { + return gap + } + return addrGapIterator{sibling, gap.index - addrminDegree} +} + +// rebalanceAfterRemove "unsplits" n and its ancestors if they are deficient +// (contain fewer segments than required by B-tree invariants), as required for +// removal, and returns an updated iterator to the position represented by gap. +// +// Precondition: n is the only node in the tree that may currently violate a +// B-tree invariant. +func (n *addrnode) rebalanceAfterRemove(gap addrGapIterator) addrGapIterator { + for { + if n.nrSegments >= addrminDegree-1 { + return gap + } + if n.parent == nil { + return gap + } + + if sibling := n.prevSibling(); sibling != nil && sibling.nrSegments >= addrminDegree { + copy(n.keys[1:], n.keys[:n.nrSegments]) + copy(n.values[1:], n.values[:n.nrSegments]) + n.keys[0] = n.parent.keys[n.parentIndex-1] + n.values[0] = n.parent.values[n.parentIndex-1] + n.parent.keys[n.parentIndex-1] = sibling.keys[sibling.nrSegments-1] + n.parent.values[n.parentIndex-1] = sibling.values[sibling.nrSegments-1] + addrSetFunctions{}.ClearValue(&sibling.values[sibling.nrSegments-1]) + if n.hasChildren { + copy(n.children[1:], n.children[:n.nrSegments+1]) + n.children[0] = sibling.children[sibling.nrSegments] + sibling.children[sibling.nrSegments] = nil + n.children[0].parent = n + n.children[0].parentIndex = 0 + for i := 1; i < n.nrSegments+2; i++ { + n.children[i].parentIndex = i + } + } + n.nrSegments++ + sibling.nrSegments-- + + if addrtrackGaps != 0 { + n.updateMaxGapLocal() + sibling.updateMaxGapLocal() + } + if gap.node == sibling && gap.index == sibling.nrSegments { + return addrGapIterator{n, 0} + } + if gap.node == n { + return addrGapIterator{n, gap.index + 1} + } + return gap + } + if sibling := n.nextSibling(); sibling != nil && sibling.nrSegments >= addrminDegree { + n.keys[n.nrSegments] = n.parent.keys[n.parentIndex] + n.values[n.nrSegments] = n.parent.values[n.parentIndex] + n.parent.keys[n.parentIndex] = sibling.keys[0] + n.parent.values[n.parentIndex] = sibling.values[0] + copy(sibling.keys[:sibling.nrSegments-1], sibling.keys[1:]) + copy(sibling.values[:sibling.nrSegments-1], sibling.values[1:]) + addrSetFunctions{}.ClearValue(&sibling.values[sibling.nrSegments-1]) + if n.hasChildren { + n.children[n.nrSegments+1] = sibling.children[0] + copy(sibling.children[:sibling.nrSegments], sibling.children[1:]) + sibling.children[sibling.nrSegments] = nil + n.children[n.nrSegments+1].parent = n + n.children[n.nrSegments+1].parentIndex = n.nrSegments + 1 + for i := 0; i < sibling.nrSegments; i++ { + sibling.children[i].parentIndex = i + } + } + n.nrSegments++ + sibling.nrSegments-- + + if addrtrackGaps != 0 { + n.updateMaxGapLocal() + sibling.updateMaxGapLocal() + } + if gap.node == sibling { + if gap.index == 0 { + return addrGapIterator{n, n.nrSegments} + } + return addrGapIterator{sibling, gap.index - 1} + } + return gap + } + + p := n.parent + if p.nrSegments == 1 { + + left, right := p.children[0], p.children[1] + p.nrSegments = left.nrSegments + right.nrSegments + 1 + p.hasChildren = left.hasChildren + p.keys[left.nrSegments] = p.keys[0] + p.values[left.nrSegments] = p.values[0] + copy(p.keys[:left.nrSegments], left.keys[:left.nrSegments]) + copy(p.values[:left.nrSegments], left.values[:left.nrSegments]) + copy(p.keys[left.nrSegments+1:], right.keys[:right.nrSegments]) + copy(p.values[left.nrSegments+1:], right.values[:right.nrSegments]) + if left.hasChildren { + copy(p.children[:left.nrSegments+1], left.children[:left.nrSegments+1]) + copy(p.children[left.nrSegments+1:], right.children[:right.nrSegments+1]) + for i := 0; i < p.nrSegments+1; i++ { + p.children[i].parent = p + p.children[i].parentIndex = i + } + } else { + p.children[0] = nil + p.children[1] = nil + } + + if gap.node == left { + return addrGapIterator{p, gap.index} + } + if gap.node == right { + return addrGapIterator{p, gap.index + left.nrSegments + 1} + } + return gap + } + // Merge n and either sibling, along with the segment separating the + // two, into whichever of the two nodes comes first. This is the + // reverse of the non-root splitting case in + // node.rebalanceBeforeInsert. + var left, right *addrnode + if n.parentIndex > 0 { + left = n.prevSibling() + right = n + } else { + left = n + right = n.nextSibling() + } + + if gap.node == right { + gap = addrGapIterator{left, gap.index + left.nrSegments + 1} + } + left.keys[left.nrSegments] = p.keys[left.parentIndex] + left.values[left.nrSegments] = p.values[left.parentIndex] + copy(left.keys[left.nrSegments+1:], right.keys[:right.nrSegments]) + copy(left.values[left.nrSegments+1:], right.values[:right.nrSegments]) + if left.hasChildren { + copy(left.children[left.nrSegments+1:], right.children[:right.nrSegments+1]) + for i := left.nrSegments + 1; i < left.nrSegments+right.nrSegments+2; i++ { + left.children[i].parent = left + left.children[i].parentIndex = i + } + } + left.nrSegments += right.nrSegments + 1 + copy(p.keys[left.parentIndex:], p.keys[left.parentIndex+1:p.nrSegments]) + copy(p.values[left.parentIndex:], p.values[left.parentIndex+1:p.nrSegments]) + addrSetFunctions{}.ClearValue(&p.values[p.nrSegments-1]) + copy(p.children[left.parentIndex+1:], p.children[left.parentIndex+2:p.nrSegments+1]) + for i := 0; i < p.nrSegments; i++ { + p.children[i].parentIndex = i + } + p.children[p.nrSegments] = nil + p.nrSegments-- + + if addrtrackGaps != 0 { + left.updateMaxGapLocal() + } + + n = p + } +} + +// updateMaxGapLeaf updates maxGap bottom-up from the calling leaf until no +// necessary update. +// +// Preconditions: n must be a leaf node, trackGaps must be 1. +func (n *addrnode) updateMaxGapLeaf() { + if n.hasChildren { + panic(fmt.Sprintf("updateMaxGapLeaf should always be called on leaf node: %v", n)) + } + max := n.calculateMaxGapLeaf() + if max == n.maxGap.Get() { + return + } + oldMax := n.maxGap.Get() + n.maxGap.Set(max) + if max > oldMax { + + for p := n.parent; p != nil; p = p.parent { + if p.maxGap.Get() >= max { + break + } + + p.maxGap.Set(max) + } + return + } + + for p := n.parent; p != nil; p = p.parent { + if p.maxGap.Get() > oldMax { + break + } + + parentNewMax := p.calculateMaxGapInternal() + if p.maxGap.Get() == parentNewMax { + break + } + + p.maxGap.Set(parentNewMax) + } +} + +// updateMaxGapLocal updates maxGap of the calling node solely with no +// propagation to ancestor nodes. +// +// Precondition: trackGaps must be 1. +func (n *addrnode) updateMaxGapLocal() { + if !n.hasChildren { + n.maxGap.Set(n.calculateMaxGapLeaf()) + } else { + n.maxGap.Set(n.calculateMaxGapInternal()) + } +} + +// calculateMaxGapLeaf iterates the gaps within a leaf node and calculate the +// max. +// +// Preconditions: n must be a leaf node. +func (n *addrnode) calculateMaxGapLeaf() uintptr { + max := addrGapIterator{n, 0}.Range().Length() + for i := 1; i <= n.nrSegments; i++ { + if current := (addrGapIterator{n, i}).Range().Length(); current > max { + max = current + } + } + return max +} + +// calculateMaxGapInternal iterates children's maxGap within an internal node n +// and calculate the max. +// +// Preconditions: n must be a non-leaf node. +func (n *addrnode) calculateMaxGapInternal() uintptr { + max := n.children[0].maxGap.Get() + for i := 1; i <= n.nrSegments; i++ { + if current := n.children[i].maxGap.Get(); current > max { + max = current + } + } + return max +} + +// searchFirstLargeEnoughGap returns the first gap having at least minSize length +// in the subtree rooted by n. If not found, return a terminal gap iterator. +func (n *addrnode) searchFirstLargeEnoughGap(minSize uintptr) addrGapIterator { + if n.maxGap.Get() < minSize { + return addrGapIterator{} + } + if n.hasChildren { + for i := 0; i <= n.nrSegments; i++ { + if largeEnoughGap := n.children[i].searchFirstLargeEnoughGap(minSize); largeEnoughGap.Ok() { + return largeEnoughGap + } + } + } else { + for i := 0; i <= n.nrSegments; i++ { + currentGap := addrGapIterator{n, i} + if currentGap.Range().Length() >= minSize { + return currentGap + } + } + } + panic(fmt.Sprintf("invalid maxGap in %v", n)) +} + +// searchLastLargeEnoughGap returns the last gap having at least minSize length +// in the subtree rooted by n. If not found, return a terminal gap iterator. +func (n *addrnode) searchLastLargeEnoughGap(minSize uintptr) addrGapIterator { + if n.maxGap.Get() < minSize { + return addrGapIterator{} + } + if n.hasChildren { + for i := n.nrSegments; i >= 0; i-- { + if largeEnoughGap := n.children[i].searchLastLargeEnoughGap(minSize); largeEnoughGap.Ok() { + return largeEnoughGap + } + } + } else { + for i := n.nrSegments; i >= 0; i-- { + currentGap := addrGapIterator{n, i} + if currentGap.Range().Length() >= minSize { + return currentGap + } + } + } + panic(fmt.Sprintf("invalid maxGap in %v", n)) +} + +// A Iterator is conceptually one of: +// +// - A pointer to a segment in a set; or +// +// - A terminal iterator, which is a sentinel indicating that the end of +// iteration has been reached. +// +// Iterators are copyable values and are meaningfully equality-comparable. The +// zero value of Iterator is a terminal iterator. +// +// Unless otherwise specified, any mutation of a set invalidates all existing +// iterators into the set. +type addrIterator struct { + // node is the node containing the iterated segment. If the iterator is + // terminal, node is nil. + node *addrnode + + // index is the index of the segment in node.keys/values. + index int +} + +// Ok returns true if the iterator is not terminal. All other methods are only +// valid for non-terminal iterators. +func (seg addrIterator) Ok() bool { + return seg.node != nil +} + +// Range returns the iterated segment's range key. +func (seg addrIterator) Range() addrRange { + return seg.node.keys[seg.index] +} + +// Start is equivalent to Range().Start, but should be preferred if only the +// start of the range is needed. +func (seg addrIterator) Start() uintptr { + return seg.node.keys[seg.index].Start +} + +// End is equivalent to Range().End, but should be preferred if only the end of +// the range is needed. +func (seg addrIterator) End() uintptr { + return seg.node.keys[seg.index].End +} + +// SetRangeUnchecked mutates the iterated segment's range key. This operation +// does not invalidate any iterators. +// +// Preconditions: +// - r.Length() > 0. +// - The new range must not overlap an existing one: +// - If seg.NextSegment().Ok(), then r.end <= seg.NextSegment().Start(). +// - If seg.PrevSegment().Ok(), then r.start >= seg.PrevSegment().End(). +func (seg addrIterator) SetRangeUnchecked(r addrRange) { + seg.node.keys[seg.index] = r +} + +// SetRange mutates the iterated segment's range key. If the new range would +// cause the iterated segment to overlap another segment, or if the new range +// is invalid, SetRange panics. This operation does not invalidate any +// iterators. +func (seg addrIterator) SetRange(r addrRange) { + if r.Length() <= 0 { + panic(fmt.Sprintf("invalid segment range %v", r)) + } + if prev := seg.PrevSegment(); prev.Ok() && r.Start < prev.End() { + panic(fmt.Sprintf("new segment range %v overlaps segment range %v", r, prev.Range())) + } + if next := seg.NextSegment(); next.Ok() && r.End > next.Start() { + panic(fmt.Sprintf("new segment range %v overlaps segment range %v", r, next.Range())) + } + seg.SetRangeUnchecked(r) +} + +// SetStartUnchecked mutates the iterated segment's start. This operation does +// not invalidate any iterators. +// +// Preconditions: The new start must be valid: +// - start < seg.End() +// - If seg.PrevSegment().Ok(), then start >= seg.PrevSegment().End(). +func (seg addrIterator) SetStartUnchecked(start uintptr) { + seg.node.keys[seg.index].Start = start +} + +// SetStart mutates the iterated segment's start. If the new start value would +// cause the iterated segment to overlap another segment, or would result in an +// invalid range, SetStart panics. This operation does not invalidate any +// iterators. +func (seg addrIterator) SetStart(start uintptr) { + if start >= seg.End() { + panic(fmt.Sprintf("new start %v would invalidate segment range %v", start, seg.Range())) + } + if prev := seg.PrevSegment(); prev.Ok() && start < prev.End() { + panic(fmt.Sprintf("new start %v would cause segment range %v to overlap segment range %v", start, seg.Range(), prev.Range())) + } + seg.SetStartUnchecked(start) +} + +// SetEndUnchecked mutates the iterated segment's end. This operation does not +// invalidate any iterators. +// +// Preconditions: The new end must be valid: +// - end > seg.Start(). +// - If seg.NextSegment().Ok(), then end <= seg.NextSegment().Start(). +func (seg addrIterator) SetEndUnchecked(end uintptr) { + seg.node.keys[seg.index].End = end +} + +// SetEnd mutates the iterated segment's end. If the new end value would cause +// the iterated segment to overlap another segment, or would result in an +// invalid range, SetEnd panics. This operation does not invalidate any +// iterators. +func (seg addrIterator) SetEnd(end uintptr) { + if end <= seg.Start() { + panic(fmt.Sprintf("new end %v would invalidate segment range %v", end, seg.Range())) + } + if next := seg.NextSegment(); next.Ok() && end > next.Start() { + panic(fmt.Sprintf("new end %v would cause segment range %v to overlap segment range %v", end, seg.Range(), next.Range())) + } + seg.SetEndUnchecked(end) +} + +// Value returns a copy of the iterated segment's value. +func (seg addrIterator) Value() *objectEncodeState { + return seg.node.values[seg.index] +} + +// ValuePtr returns a pointer to the iterated segment's value. The pointer is +// invalidated if the iterator is invalidated. This operation does not +// invalidate any iterators. +func (seg addrIterator) ValuePtr() **objectEncodeState { + return &seg.node.values[seg.index] +} + +// SetValue mutates the iterated segment's value. This operation does not +// invalidate any iterators. +func (seg addrIterator) SetValue(val *objectEncodeState) { + seg.node.values[seg.index] = val +} + +// PrevSegment returns the iterated segment's predecessor. If there is no +// preceding segment, PrevSegment returns a terminal iterator. +func (seg addrIterator) PrevSegment() addrIterator { + if seg.node.hasChildren { + return seg.node.children[seg.index].lastSegment() + } + if seg.index > 0 { + return addrIterator{seg.node, seg.index - 1} + } + if seg.node.parent == nil { + return addrIterator{} + } + return addrsegmentBeforePosition(seg.node.parent, seg.node.parentIndex) +} + +// NextSegment returns the iterated segment's successor. If there is no +// succeeding segment, NextSegment returns a terminal iterator. +func (seg addrIterator) NextSegment() addrIterator { + if seg.node.hasChildren { + return seg.node.children[seg.index+1].firstSegment() + } + if seg.index < seg.node.nrSegments-1 { + return addrIterator{seg.node, seg.index + 1} + } + if seg.node.parent == nil { + return addrIterator{} + } + return addrsegmentAfterPosition(seg.node.parent, seg.node.parentIndex) +} + +// PrevGap returns the gap immediately before the iterated segment. +func (seg addrIterator) PrevGap() addrGapIterator { + if seg.node.hasChildren { + return seg.node.children[seg.index].lastSegment().NextGap() + } + return addrGapIterator{seg.node, seg.index} +} + +// NextGap returns the gap immediately after the iterated segment. +func (seg addrIterator) NextGap() addrGapIterator { + if seg.node.hasChildren { + return seg.node.children[seg.index+1].firstSegment().PrevGap() + } + return addrGapIterator{seg.node, seg.index + 1} +} + +// PrevNonEmpty returns the iterated segment's predecessor if it is adjacent, +// or the gap before the iterated segment otherwise. If seg.Start() == +// Functions.MinKey(), PrevNonEmpty will return two terminal iterators. +// Otherwise, exactly one of the iterators returned by PrevNonEmpty will be +// non-terminal. +func (seg addrIterator) PrevNonEmpty() (addrIterator, addrGapIterator) { + if prev := seg.PrevSegment(); prev.Ok() && prev.End() == seg.Start() { + return prev, addrGapIterator{} + } + return addrIterator{}, seg.PrevGap() +} + +// NextNonEmpty returns the iterated segment's successor if it is adjacent, or +// the gap after the iterated segment otherwise. If seg.End() == +// Functions.MaxKey(), NextNonEmpty will return two terminal iterators. +// Otherwise, exactly one of the iterators returned by NextNonEmpty will be +// non-terminal. +func (seg addrIterator) NextNonEmpty() (addrIterator, addrGapIterator) { + if next := seg.NextSegment(); next.Ok() && next.Start() == seg.End() { + return next, addrGapIterator{} + } + return addrIterator{}, seg.NextGap() +} + +// A GapIterator is conceptually one of: +// +// - A pointer to a position between two segments, before the first segment, or +// after the last segment in a set, called a *gap*; or +// +// - A terminal iterator, which is a sentinel indicating that the end of +// iteration has been reached. +// +// Note that the gap between two adjacent segments exists (iterators to it are +// non-terminal), but has a length of zero. GapIterator.IsEmpty returns true +// for such gaps. An empty set contains a single gap, spanning the entire range +// of the set's keys. +// +// GapIterators are copyable values and are meaningfully equality-comparable. +// The zero value of GapIterator is a terminal iterator. +// +// Unless otherwise specified, any mutation of a set invalidates all existing +// iterators into the set. +type addrGapIterator struct { + // The representation of a GapIterator is identical to that of an Iterator, + // except that index corresponds to positions between segments in the same + // way as for node.children (see comment for node.nrSegments). + node *addrnode + index int +} + +// Ok returns true if the iterator is not terminal. All other methods are only +// valid for non-terminal iterators. +func (gap addrGapIterator) Ok() bool { + return gap.node != nil +} + +// Range returns the range spanned by the iterated gap. +func (gap addrGapIterator) Range() addrRange { + return addrRange{gap.Start(), gap.End()} +} + +// Start is equivalent to Range().Start, but should be preferred if only the +// start of the range is needed. +func (gap addrGapIterator) Start() uintptr { + if ps := gap.PrevSegment(); ps.Ok() { + return ps.End() + } + return addrSetFunctions{}.MinKey() +} + +// End is equivalent to Range().End, but should be preferred if only the end of +// the range is needed. +func (gap addrGapIterator) End() uintptr { + if ns := gap.NextSegment(); ns.Ok() { + return ns.Start() + } + return addrSetFunctions{}.MaxKey() +} + +// IsEmpty returns true if the iterated gap is empty (that is, the "gap" is +// between two adjacent segments.) +func (gap addrGapIterator) IsEmpty() bool { + return gap.Range().Length() == 0 +} + +// PrevSegment returns the segment immediately before the iterated gap. If no +// such segment exists, PrevSegment returns a terminal iterator. +func (gap addrGapIterator) PrevSegment() addrIterator { + return addrsegmentBeforePosition(gap.node, gap.index) +} + +// NextSegment returns the segment immediately after the iterated gap. If no +// such segment exists, NextSegment returns a terminal iterator. +func (gap addrGapIterator) NextSegment() addrIterator { + return addrsegmentAfterPosition(gap.node, gap.index) +} + +// PrevGap returns the iterated gap's predecessor. If no such gap exists, +// PrevGap returns a terminal iterator. +func (gap addrGapIterator) PrevGap() addrGapIterator { + seg := gap.PrevSegment() + if !seg.Ok() { + return addrGapIterator{} + } + return seg.PrevGap() +} + +// NextGap returns the iterated gap's successor. If no such gap exists, NextGap +// returns a terminal iterator. +func (gap addrGapIterator) NextGap() addrGapIterator { + seg := gap.NextSegment() + if !seg.Ok() { + return addrGapIterator{} + } + return seg.NextGap() +} + +// NextLargeEnoughGap returns the iterated gap's first next gap with larger +// length than minSize. If not found, return a terminal gap iterator (does NOT +// include this gap itself). +// +// Precondition: trackGaps must be 1. +func (gap addrGapIterator) NextLargeEnoughGap(minSize uintptr) addrGapIterator { + if addrtrackGaps != 1 { + panic("set is not tracking gaps") + } + if gap.node != nil && gap.node.hasChildren && gap.index == gap.node.nrSegments { + + gap.node = gap.NextSegment().node + gap.index = 0 + return gap.nextLargeEnoughGapHelper(minSize) + } + return gap.nextLargeEnoughGapHelper(minSize) +} + +// nextLargeEnoughGapHelper is the helper function used by NextLargeEnoughGap +// to do the real recursions. +// +// Preconditions: gap is NOT the trailing gap of a non-leaf node. +func (gap addrGapIterator) nextLargeEnoughGapHelper(minSize uintptr) addrGapIterator { + for { + + for gap.node != nil && + (gap.node.maxGap.Get() < minSize || (!gap.node.hasChildren && gap.index == gap.node.nrSegments)) { + gap.node, gap.index = gap.node.parent, gap.node.parentIndex + } + + if gap.node == nil { + return addrGapIterator{} + } + + gap.index++ + for gap.index <= gap.node.nrSegments { + if gap.node.hasChildren { + if largeEnoughGap := gap.node.children[gap.index].searchFirstLargeEnoughGap(minSize); largeEnoughGap.Ok() { + return largeEnoughGap + } + } else { + if gap.Range().Length() >= minSize { + return gap + } + } + gap.index++ + } + gap.node, gap.index = gap.node.parent, gap.node.parentIndex + if gap.node != nil && gap.index == gap.node.nrSegments { + gap.node, gap.index = gap.node.parent, gap.node.parentIndex + } + } +} + +// PrevLargeEnoughGap returns the iterated gap's first prev gap with larger or +// equal length than minSize. If not found, return a terminal gap iterator +// (does NOT include this gap itself). +// +// Precondition: trackGaps must be 1. +func (gap addrGapIterator) PrevLargeEnoughGap(minSize uintptr) addrGapIterator { + if addrtrackGaps != 1 { + panic("set is not tracking gaps") + } + if gap.node != nil && gap.node.hasChildren && gap.index == 0 { + + gap.node = gap.PrevSegment().node + gap.index = gap.node.nrSegments + return gap.prevLargeEnoughGapHelper(minSize) + } + return gap.prevLargeEnoughGapHelper(minSize) +} + +// prevLargeEnoughGapHelper is the helper function used by PrevLargeEnoughGap +// to do the real recursions. +// +// Preconditions: gap is NOT the first gap of a non-leaf node. +func (gap addrGapIterator) prevLargeEnoughGapHelper(minSize uintptr) addrGapIterator { + for { + + for gap.node != nil && + (gap.node.maxGap.Get() < minSize || (!gap.node.hasChildren && gap.index == 0)) { + gap.node, gap.index = gap.node.parent, gap.node.parentIndex + } + + if gap.node == nil { + return addrGapIterator{} + } + + gap.index-- + for gap.index >= 0 { + if gap.node.hasChildren { + if largeEnoughGap := gap.node.children[gap.index].searchLastLargeEnoughGap(minSize); largeEnoughGap.Ok() { + return largeEnoughGap + } + } else { + if gap.Range().Length() >= minSize { + return gap + } + } + gap.index-- + } + gap.node, gap.index = gap.node.parent, gap.node.parentIndex + if gap.node != nil && gap.index == 0 { + gap.node, gap.index = gap.node.parent, gap.node.parentIndex + } + } +} + +// segmentBeforePosition returns the predecessor segment of the position given +// by n.children[i], which may or may not contain a child. If no such segment +// exists, segmentBeforePosition returns a terminal iterator. +func addrsegmentBeforePosition(n *addrnode, i int) addrIterator { + for i == 0 { + if n.parent == nil { + return addrIterator{} + } + n, i = n.parent, n.parentIndex + } + return addrIterator{n, i - 1} +} + +// segmentAfterPosition returns the successor segment of the position given by +// n.children[i], which may or may not contain a child. If no such segment +// exists, segmentAfterPosition returns a terminal iterator. +func addrsegmentAfterPosition(n *addrnode, i int) addrIterator { + for i == n.nrSegments { + if n.parent == nil { + return addrIterator{} + } + n, i = n.parent, n.parentIndex + } + return addrIterator{n, i} +} + +func addrzeroValueSlice(slice []*objectEncodeState) { + for i := range slice { + addrSetFunctions{}.ClearValue(&slice[i]) + } +} + +func addrzeroNodeSlice(slice []*addrnode) { + for i := range slice { + slice[i] = nil + } +} + +// String stringifies a Set for debugging. +func (s *addrSet) String() string { + return s.root.String() +} + +// String stringifies a node (and all of its children) for debugging. +func (n *addrnode) String() string { + var buf bytes.Buffer + n.writeDebugString(&buf, "") + return buf.String() +} + +func (n *addrnode) writeDebugString(buf *bytes.Buffer, prefix string) { + if n.hasChildren != (n.nrSegments > 0 && n.children[0] != nil) { + buf.WriteString(prefix) + buf.WriteString(fmt.Sprintf("WARNING: inconsistent value of hasChildren: got %v, want %v\n", n.hasChildren, !n.hasChildren)) + } + for i := 0; i < n.nrSegments; i++ { + if child := n.children[i]; child != nil { + cprefix := fmt.Sprintf("%s- % 3d ", prefix, i) + if child.parent != n || child.parentIndex != i { + buf.WriteString(cprefix) + buf.WriteString(fmt.Sprintf("WARNING: inconsistent linkage to parent: got (%p, %d), want (%p, %d)\n", child.parent, child.parentIndex, n, i)) + } + child.writeDebugString(buf, fmt.Sprintf("%s- % 3d ", prefix, i)) + } + buf.WriteString(prefix) + if n.hasChildren { + if addrtrackGaps != 0 { + buf.WriteString(fmt.Sprintf("- % 3d: %v => %v, maxGap: %d\n", i, n.keys[i], n.values[i], n.maxGap.Get())) + } else { + buf.WriteString(fmt.Sprintf("- % 3d: %v => %v\n", i, n.keys[i], n.values[i])) + } + } else { + buf.WriteString(fmt.Sprintf("- % 3d: %v => %v\n", i, n.keys[i], n.values[i])) + } + } + if child := n.children[n.nrSegments]; child != nil { + child.writeDebugString(buf, fmt.Sprintf("%s- % 3d ", prefix, n.nrSegments)) + } +} + +// FlatSegment represents a segment as a single object. FlatSegment is used as +// an intermediate representation for save/restore and tests. +// +// +stateify savable +type addrFlatSegment struct { + Start uintptr + End uintptr + Value *objectEncodeState +} + +// ExportSlice returns a copy of all segments in the given set, in ascending +// key order. +func (s *addrSet) ExportSlice() []addrFlatSegment { + var fs []addrFlatSegment + for seg := s.FirstSegment(); seg.Ok(); seg = seg.NextSegment() { + fs = append(fs, addrFlatSegment{ + Start: seg.Start(), + End: seg.End(), + Value: seg.Value(), + }) + } + return fs +} + +// ImportSlice initializes the given set from the given slice. +// +// Preconditions: +// - s must be empty. +// - fs must represent a valid set (the segments in fs must have valid +// lengths that do not overlap). +// - The segments in fs must be sorted in ascending key order. +func (s *addrSet) ImportSlice(fs []addrFlatSegment) error { + if !s.IsEmpty() { + return fmt.Errorf("cannot import into non-empty set %v", s) + } + gap := s.FirstGap() + for i := range fs { + f := &fs[i] + r := addrRange{f.Start, f.End} + if !gap.Range().IsSupersetOf(r) { + return fmt.Errorf("segment overlaps a preceding segment or is incorrectly sorted: %v => %v", r, f.Value) + } + gap = s.InsertWithoutMerging(gap, r, f.Value).NextGap() + } + return nil +} + +// segmentTestCheck returns an error if s is incorrectly sorted, does not +// contain exactly expectedSegments segments, or contains a segment which +// fails the passed check. +// +// This should be used only for testing, and has been added to this package for +// templating convenience. +func (s *addrSet) segmentTestCheck(expectedSegments int, segFunc func(int, addrRange, *objectEncodeState) error) error { + havePrev := false + prev := uintptr(0) + nrSegments := 0 + for seg := s.FirstSegment(); seg.Ok(); seg = seg.NextSegment() { + next := seg.Start() + if havePrev && prev >= next { + return fmt.Errorf("incorrect order: key %d (segment %d) >= key %d (segment %d)", prev, nrSegments-1, next, nrSegments) + } + if segFunc != nil { + if err := segFunc(nrSegments, seg.Range(), seg.Value()); err != nil { + return err + } + } + prev = next + havePrev = true + nrSegments++ + } + if nrSegments != expectedSegments { + return fmt.Errorf("incorrect number of segments: got %d, wanted %d", nrSegments, expectedSegments) + } + return nil +} + +// countSegments counts the number of segments in the set. +// +// Similar to Check, this should only be used for testing. +func (s *addrSet) countSegments() (segments int) { + for seg := s.FirstSegment(); seg.Ok(); seg = seg.NextSegment() { + segments++ + } + return segments +} + +func (s *addrSet) saveRoot() []addrFlatSegment { + fs := s.ExportSlice() + + fs = fs[:len(fs):len(fs)] + return fs +} + +func (s *addrSet) loadRoot(_ context.Context, fs []addrFlatSegment) { + if err := s.ImportSlice(fs); err != nil { + panic(err) + } +} diff --git a/pkg/state/decode.go b/pkg/state/decode.go new file mode 100644 index 0000000..9689eb9 --- /dev/null +++ b/pkg/state/decode.go @@ -0,0 +1,742 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "bytes" + "context" + "fmt" + "math" + "reflect" + + "github.com/sagernet/gvisor/pkg/state/wire" +) + +// internalCallback is a interface called on object completion. +// +// There are two implementations: objectDecodeState & userCallback. +type internalCallback interface { + // source returns the dependent object. May be nil. + source() *objectDecodeState + + // callbackRun executes the callback. + callbackRun(ds *decodeState) +} + +// userCallback is an implementation of internalCallback. +type userCallback func() + +// source implements internalCallback.source. +func (userCallback) source() *objectDecodeState { + return nil +} + +// callbackRun implements internalCallback.callbackRun. +func (uc userCallback) callbackRun(*decodeState) { + uc() +} + +// objectDecodeState represents an object that may be in the process of being +// decoded. Specifically, it represents either a decoded object, or an an +// interest in a future object that will be decoded. When that interest is +// registered (via register), the storage for the object will be created, but +// it will not be decoded until the object is encountered in the stream. +type objectDecodeState struct { + // id is the id for this object. + id objectID + + // typ is the id for this typeID. This may be zero if this is not a + // type-registered structure. + typ typeID + + // obj is the object. This may or may not be valid yet, depending on + // whether complete returns true. However, regardless of whether the + // object is valid, obj contains a final storage location for the + // object. This is immutable. + // + // Note that this must be addressable (obj.Addr() must not panic). + // + // The obj passed to the decode methods below will equal this obj only + // in the case of decoding the top-level object. However, the passed + // obj may represent individual fields, elements of a slice, etc. that + // are effectively embedded within the reflect.Value below but with + // distinct types. + obj reflect.Value + + // blockedBy is the number of dependencies this object has. + blockedBy int + + // callbacksInline is inline storage for callbacks. + callbacksInline [2]internalCallback + + // callbacks is a set of callbacks to execute on load. + callbacks []internalCallback + + pendingEntry odsListElem + leafEntry odsListElem +} + +type odsListElem struct { + ods *objectDecodeState + odsEntry +} + +// addCallback adds a callback to the objectDecodeState. +func (ods *objectDecodeState) addCallback(ic internalCallback) { + if ods.callbacks == nil { + ods.callbacks = ods.callbacksInline[:0] + } + ods.callbacks = append(ods.callbacks, ic) +} + +// findCycleFor returns when the given object is found in the blocking set. +func (ods *objectDecodeState) findCycleFor(target *objectDecodeState) []*objectDecodeState { + for _, ic := range ods.callbacks { + other := ic.source() + if other != nil && other == target { + return []*objectDecodeState{target} + } else if childList := other.findCycleFor(target); childList != nil { + return append(childList, other) + } + } + + // This should not occur. + Failf("no deadlock found?") + panic("unreachable") +} + +// findCycle finds a dependency cycle. +func (ods *objectDecodeState) findCycle() []*objectDecodeState { + return append(ods.findCycleFor(ods), ods) +} + +// source implements internalCallback.source. +func (ods *objectDecodeState) source() *objectDecodeState { + return ods +} + +// callbackRun implements internalCallback.callbackRun. +func (ods *objectDecodeState) callbackRun(ds *decodeState) { + ods.blockedBy-- + if ods.blockedBy == 0 { + ds.leaves.PushBack(&ods.leafEntry) + } else if ods.blockedBy < 0 { + Failf("object %d has negative blockedBy: %d", ods.id, ods.blockedBy) + } +} + +// decodeState is a graph of objects in the process of being decoded. +// +// The decode process involves loading the breadth-first graph generated by +// encode. This graph is read in it's entirety, ensuring that all object +// storage is complete. +// +// As the graph is being serialized, a set of completion callbacks are +// executed. These completion callbacks should form a set of acyclic subgraphs +// over the original one. After decoding is complete, the objects are scanned +// to ensure that all callbacks are executed, otherwise the callback graph was +// not acyclic. +type decodeState struct { + // ctx is the decode context. + ctx context.Context + + // r is the input stream. + r wire.Reader + + // types is the type database. + types typeDecodeDatabase + + // objectByID is the set of objects in progress. + objectsByID []*objectDecodeState + + // deferred are objects that have been read, by no interest has been + // registered yet. These will be decoded once interest in registered. + deferred map[objectID]wire.Object + + // pending is the set of objects that are not yet complete. + pending odsList + + // leaves is the set of objects that have no dependencies (blockedBy == 0). + // leaves are consumed from the front and appended to the back. + leaves odsList + + // stats tracks time data. + stats Stats +} + +// lookup looks up an object in decodeState or returns nil if no such object +// has been previously registered. +func (ds *decodeState) lookup(id objectID) *objectDecodeState { + if len(ds.objectsByID) < int(id) { + return nil + } + return ds.objectsByID[id-1] +} + +// checkComplete checks for completion. +func (ds *decodeState) checkComplete(ods *objectDecodeState) bool { + // Still blocked? + if ods.blockedBy > 0 { + return false + } + + // Track stats if relevant. + if ods.callbacks != nil && ods.typ != 0 { + ds.stats.start(ods.typ) + defer ds.stats.done() + } + + // Fire all callbacks. + for _, ic := range ods.callbacks { + ic.callbackRun(ds) + } + + // Mark completed. + ods.callbacks = nil + ds.pending.Remove(&ods.pendingEntry) + + return true // All set. +} + +// wait registers a dependency on an object. +// +// As a special case, we always allow _useable_ references back to the first +// decoding object because it may have fields that are already decoded. We also +// allow trivial self reference, since they can be handled internally. +func (ds *decodeState) wait(waiter *objectDecodeState, id objectID, callback func()) { + switch id { + case waiter.id: + // Trivial self reference. + fallthrough + case 1: + // Root object; see above. + if callback != nil { + callback() + } + return + } + + // Mark as blocked. + waiter.blockedBy++ + if waiter.blockedBy == 1 { + ds.leaves.Remove(&waiter.leafEntry) + } + + // No nil can be returned here. + other := ds.lookup(id) + if callback != nil { + // Add the additional user callback. + other.addCallback(userCallback(callback)) + } + + // Mark waiter as unblocked. + other.addCallback(waiter) +} + +// waitObject notes a blocking relationship. +func (ds *decodeState) waitObject(ods *objectDecodeState, encoded wire.Object, callback func()) { + if rv, ok := encoded.(*wire.Ref); ok && rv.Root != 0 { + // Refs can encode pointers and maps. + ds.wait(ods, objectID(rv.Root), callback) + } else if sv, ok := encoded.(*wire.Slice); ok && sv.Ref.Root != 0 { + // See decodeObject; we need to wait for the array (if non-nil). + ds.wait(ods, objectID(sv.Ref.Root), callback) + } else if iv, ok := encoded.(*wire.Interface); ok { + // It's an interface (wait recursively). + ds.waitObject(ods, iv.Value, callback) + } else if callback != nil { + // Nothing to wait for: execute the callback immediately. + callback() + } +} + +// walkChild returns a child object from obj, given an accessor path. This is +// the decode-side equivalent to traverse in encode.go. +// +// For the purposes of this function, a child object is either a field within a +// struct or an array element, with one such indirection per element in +// path. The returned value may be an unexported field, so it may not be +// directly assignable. See decode_unsafe.go. +func walkChild(path []wire.Dot, obj reflect.Value) reflect.Value { + // See wire.Ref.Dots. The path here is specified in reverse order. + for i := len(path) - 1; i >= 0; i-- { + switch pc := path[i].(type) { + case *wire.FieldName: // Must be a pointer. + if obj.Kind() != reflect.Struct { + Failf("next component in child path is a field name, but the current object is not a struct. Path: %v, current obj: %#v", path, obj) + } + obj = obj.FieldByName(string(*pc)) + case wire.Index: // Embedded. + if obj.Kind() != reflect.Array { + Failf("next component in child path is an array index, but the current object is not an array. Path: %v, current obj: %#v", path, obj) + } + obj = obj.Index(int(pc)) + default: + panic("unreachable: switch should be exhaustive") + } + } + return obj +} + +func (ds *decodeState) growObjectsByID(id objectID) { + if len(ds.objectsByID) < int(id) { + ds.objectsByID = append(ds.objectsByID, make([]*objectDecodeState, int(id)-len(ds.objectsByID))...) + } +} + +func (ds *decodeState) addObject(id objectID, obj reflect.Value) *objectDecodeState { + ods := &objectDecodeState{ + id: id, + obj: obj, + } + ods.pendingEntry.ods = ods + ods.leafEntry.ods = ods + ds.growObjectsByID(id) + ds.objectsByID[id-1] = ods + ds.pending.PushBack(&ods.pendingEntry) + ds.leaves.PushBack(&ods.leafEntry) + return ods +} + +// register registers a decode with a type. +// +// This type is only used to instantiate a new object if it has not been +// registered previously. This depends on the type provided if none is +// available in the object itself. +func (ds *decodeState) register(r *wire.Ref, typ reflect.Type) reflect.Value { + // Grow the objectsByID slice. + id := objectID(r.Root) + + // Does this object already exist? + ds.growObjectsByID(id) + ods := ds.objectsByID[id-1] + if ods != nil { + return walkChild(r.Dots, ods.obj) + } + + // Create the object. + if len(r.Dots) != 0 { + typ = ds.findType(r.Type) + } + v := reflect.New(typ) + ods = ds.addObject(id, v.Elem()) + + // Process any deferred objects & callbacks. + if encoded, ok := ds.deferred[id]; ok { + delete(ds.deferred, id) + ds.decodeObject(ods, ods.obj, encoded) + } + + return walkChild(r.Dots, ods.obj) +} + +// objectDecoder is for decoding structs. +type objectDecoder struct { + // ds is decodeState. + ds *decodeState + + // ods is current object being decoded. + ods *objectDecodeState + + // reconciledTypeEntry is the reconciled type information. + rte *reconciledTypeEntry + + // encoded is the encoded object state. + encoded *wire.Struct +} + +// load is helper for the public methods on Source. +func (od *objectDecoder) load(slot int, objPtr reflect.Value, wait bool, fn func()) { + // Note that we have reconciled the type and may remap the fields here + // to match what's expected by the decoder. The "slot" parameter here + // is in terms of the local type, where the fields in the encoded + // object are in terms of the wire object's type, which might be in a + // different order (but will have the same fields). + v := *od.encoded.Field(od.rte.FieldOrder[slot]) + od.ds.decodeObject(od.ods, objPtr.Elem(), v) + if wait { + // Mark this individual object a blocker. + od.ds.waitObject(od.ods, v, fn) + } +} + +// aterLoad implements Source.AfterLoad. +func (od *objectDecoder) afterLoad(fn func()) { + // Queue the local callback; this will execute when all of the above + // data dependencies have been cleared. + od.ods.addCallback(userCallback(fn)) +} + +// decodeStruct decodes a struct value. +func (ds *decodeState) decodeStruct(ods *objectDecodeState, obj reflect.Value, encoded *wire.Struct) { + if encoded.TypeID == 0 { + // Allow anonymous empty structs, but only if the encoded + // object also has no fields. + if encoded.Fields() == 0 && obj.NumField() == 0 { + return + } + + // Propagate an error. + Failf("empty struct on wire %#v has field mismatch with type %q", encoded, obj.Type().Name()) + } + + // Lookup the object type. + rte := ds.types.Lookup(typeID(encoded.TypeID), obj.Type()) + ods.typ = typeID(encoded.TypeID) + + // Invoke the loader. + od := objectDecoder{ + ds: ds, + ods: ods, + rte: rte, + encoded: encoded, + } + ds.stats.start(ods.typ) + defer ds.stats.done() + if sl, ok := obj.Addr().Interface().(SaverLoader); ok { + // Note: may be a registered empty struct which does not + // implement the saver/loader interfaces. + sl.StateLoad(ds.ctx, Source{internal: od}) + } +} + +// decodeMap decodes a map value. +func (ds *decodeState) decodeMap(ods *objectDecodeState, obj reflect.Value, encoded *wire.Map) { + if obj.IsNil() { + // See pointerTo. + obj.Set(reflect.MakeMap(obj.Type())) + } + for i := 0; i < len(encoded.Keys); i++ { + // Decode the objects. + kv := reflect.New(obj.Type().Key()).Elem() + vv := reflect.New(obj.Type().Elem()).Elem() + ds.decodeObject(ods, kv, encoded.Keys[i]) + ds.decodeObject(ods, vv, encoded.Values[i]) + ds.waitObject(ods, encoded.Keys[i], nil) + ds.waitObject(ods, encoded.Values[i], nil) + + // Set in the map. + obj.SetMapIndex(kv, vv) + } +} + +// decodeArray decodes an array value. +func (ds *decodeState) decodeArray(ods *objectDecodeState, obj reflect.Value, encoded *wire.Array) { + if len(encoded.Contents) != obj.Len() { + Failf("mismatching array length expect=%d, actual=%d", obj.Len(), len(encoded.Contents)) + } + // Decode the contents into the array. + for i := 0; i < len(encoded.Contents); i++ { + ds.decodeObject(ods, obj.Index(i), encoded.Contents[i]) + ds.waitObject(ods, encoded.Contents[i], nil) + } +} + +// findType finds the type for the given wire.TypeSpecs. +func (ds *decodeState) findType(t wire.TypeSpec) reflect.Type { + switch x := t.(type) { + case wire.TypeID: + typ := ds.types.LookupType(typeID(x)) + rte := ds.types.Lookup(typeID(x), typ) + return rte.LocalType + case *wire.TypeSpecPointer: + return reflect.PtrTo(ds.findType(x.Type)) + case *wire.TypeSpecArray: + return reflect.ArrayOf(int(x.Count), ds.findType(x.Type)) + case *wire.TypeSpecSlice: + return reflect.SliceOf(ds.findType(x.Type)) + case *wire.TypeSpecMap: + return reflect.MapOf(ds.findType(x.Key), ds.findType(x.Value)) + default: + // Should not happen. + Failf("unknown type %#v", t) + } + panic("unreachable") +} + +// decodeInterface decodes an interface value. +func (ds *decodeState) decodeInterface(ods *objectDecodeState, obj reflect.Value, encoded *wire.Interface) { + if _, ok := encoded.Type.(wire.TypeSpecNil); ok { + // Special case; the nil object. Just decode directly, which + // will read nil from the wire (if encoded correctly). + ds.decodeObject(ods, obj, encoded.Value) + return + } + + // We now need to resolve the actual type. + typ := ds.findType(encoded.Type) + + // We need to imbue type information here, then we can proceed to + // decode normally. In order to avoid issues with setting value-types, + // we create a new non-interface version of this object. We will then + // set the interface object to be equal to whatever we decode. + origObj := obj + obj = reflect.New(typ).Elem() + defer origObj.Set(obj) + + // With the object now having sufficient type information to actually + // have Set called on it, we can proceed to decode the value. + ds.decodeObject(ods, obj, encoded.Value) +} + +// isFloatEq determines if x and y represent the same value. +func isFloatEq(x float64, y float64) bool { + switch { + case math.IsNaN(x): + return math.IsNaN(y) + case math.IsInf(x, 1): + return math.IsInf(y, 1) + case math.IsInf(x, -1): + return math.IsInf(y, -1) + default: + return x == y + } +} + +// isComplexEq determines if x and y represent the same value. +func isComplexEq(x complex128, y complex128) bool { + return isFloatEq(real(x), real(y)) && isFloatEq(imag(x), imag(y)) +} + +// decodeObject decodes a object value. +func (ds *decodeState) decodeObject(ods *objectDecodeState, obj reflect.Value, encoded wire.Object) { + switch x := encoded.(type) { + case wire.Nil: // Fast path: first. + // We leave obj alone here. That's because if obj represents an + // interface, it may have been imbued with type information in + // decodeInterface, and we don't want to destroy that. + case *wire.Ref: + // Nil pointers may be encoded in a "forceValue" context. For + // those we just leave it alone as the value will already be + // correct (nil). + if id := objectID(x.Root); id == 0 { + return + } + + // Note that if this is a map type, we go through a level of + // indirection to allow for map aliasing. + if obj.Kind() == reflect.Map { + v := ds.register(x, obj.Type()) + if v.IsNil() { + // Note that we don't want to clobber the map + // if has already been decoded by decodeMap. We + // just make it so that we have a consistent + // reference when that eventually does happen. + v.Set(reflect.MakeMap(v.Type())) + } + obj.Set(v) + return + } + + // Normal assignment: authoritative only if no dots. + v := ds.register(x, obj.Type().Elem()) + obj.Set(reflectValueRWAddr(v)) + case wire.Bool: + obj.SetBool(bool(x)) + case wire.Int: + obj.SetInt(int64(x)) + if obj.Int() != int64(x) { + Failf("signed integer truncated from %v to %v", int64(x), obj.Int()) + } + case wire.Uint: + obj.SetUint(uint64(x)) + if obj.Uint() != uint64(x) { + Failf("unsigned integer truncated from %v to %v", uint64(x), obj.Uint()) + } + case wire.Float32: + obj.SetFloat(float64(x)) + case wire.Float64: + obj.SetFloat(float64(x)) + if !isFloatEq(obj.Float(), float64(x)) { + Failf("floating point number truncated from %v to %v", float64(x), obj.Float()) + } + case *wire.Complex64: + obj.SetComplex(complex128(*x)) + case *wire.Complex128: + obj.SetComplex(complex128(*x)) + if !isComplexEq(obj.Complex(), complex128(*x)) { + Failf("complex number truncated from %v to %v", complex128(*x), obj.Complex()) + } + case *wire.String: + obj.SetString(string(*x)) + case *wire.Slice: + // See *wire.Ref above; same applies. + if id := objectID(x.Ref.Root); id == 0 { + return + } + // Note that it's fine to slice the array here and assume that + // contents will still be filled in later on. + typ := reflect.ArrayOf(int(x.Capacity), obj.Type().Elem()) // The object type. + v := ds.register(&x.Ref, typ) + obj.Set(reflectValueRWSlice3(v, 0, int(x.Length), int(x.Capacity))) + case *wire.Array: + ds.decodeArray(ods, obj, x) + case *wire.Struct: + ds.decodeStruct(ods, obj, x) + case *wire.Map: + ds.decodeMap(ods, obj, x) + case *wire.Interface: + ds.decodeInterface(ods, obj, x) + default: + // Should not happen, not propagated as an error. + Failf("unknown object %#v for %q", encoded, obj.Type().Name()) + } +} + +// Load deserializes the object graph rooted at obj. +// +// This function may panic and should be run in safely(). +func (ds *decodeState) Load(obj reflect.Value) { + ds.stats.init() + defer ds.stats.fini(func(id typeID) string { + return ds.types.LookupName(id) + }) + + // Add the root object with ID 1. + _ = ds.addObject(1, obj) + + // Read the number of objects. + numObjects, object, err := ReadHeader(&ds.r) + if err != nil { + Failf("header error: %w", err) + } + if !object { + Failf("object missing") + } + + // Decode all objects. + var ( + encoded wire.Object + ods *objectDecodeState + id objectID + ) + if err := safely(func() { + // Decode all objects in the stream. + // + // Note that the structure of this decoding loop should match the raw + // decoding loop in state/pretty/pretty.printer.printStream(). + for i := uint64(0); i < numObjects; { + // Unmarshal either a type object or object ID. + encoded = wire.Load(&ds.r) + switch we := encoded.(type) { + case *wire.Type: + ds.types.Register(we) + encoded = nil + continue + case wire.Uint: + id = objectID(we) + i++ + // Unmarshal and resolve the actual object. + encoded = wire.Load(&ds.r) + ods = ds.lookup(id) + if ods != nil { + // Decode the object. + ds.decodeObject(ods, ods.obj, encoded) + } else { + // If an object hasn't had interest registered + // previously or isn't yet valid, we deferred + // decoding until interest is registered. + ds.deferred[id] = encoded + } + // For error handling. + ods = nil + encoded = nil + default: + Failf("wanted type or object ID, got %T", encoded) + } + } + }); err != nil { + // Include as much information as we can, taking into account + // the possible state transitions above. + if ods != nil { + Failf("error decoding object ID %d (%T) from %#v: %w", id, ods.obj.Interface(), encoded, err) + } else if encoded != nil { + Failf("error decoding from %#v: %w", encoded, err) + } else { + Failf("general decoding error: %w", err) + } + } + + // Check if we have any deferred objects. + numDeferred := 0 + for id, encoded := range ds.deferred { + numDeferred++ + if s, ok := encoded.(*wire.Struct); ok && s.TypeID != 0 { + typ := ds.types.LookupType(typeID(s.TypeID)) + Failf("unused deferred object: ID %d, type %v", id, typ) + } else { + Failf("unused deferred object: ID %d, %#v", id, encoded) + } + } + if numDeferred != 0 { + Failf("still had %d deferred objects", numDeferred) + } + + // Scan and fire all callbacks. We iterate over the list of incomplete + // objects until all have been finished. We stop iterating if no + // objects become complete (there is a dependency cycle). + // + // Note that we iterate backwards here, because there will be a strong + // tendency for blocking relationships to go from earlier objects to + // later (deeper) objects in the graph. This will reduce the number of + // iterations required to finish all objects. + if err := safely(func() { + for elem := ds.leaves.Front(); elem != nil; elem = elem.Next() { + ds.checkComplete(elem.ods) + } + }); err != nil { + Failf("error executing callbacks: %w\nfor object %#v", err, ods.obj.Interface()) + } + + // Check if we have any remaining dependency cycles. If there are any + // objects left in the pending list, then it must be due to a cycle. + if elem := ds.pending.Front(); elem != nil { + // This must be the result of a dependency cycle. + cycle := elem.ods.findCycle() + var buf bytes.Buffer + buf.WriteString("dependency cycle: {") + for i, cycleOS := range cycle { + if i > 0 { + buf.WriteString(" => ") + } + fmt.Fprintf(&buf, "%q", cycleOS.obj.Type()) + } + buf.WriteString("}") + Failf("incomplete graph: %s", string(buf.Bytes())) + } +} + +// ReadHeader reads an object header. +// +// Each object written to the statefile is prefixed with a header. See +// WriteHeader for more information; these functions are exported to allow +// non-state writes to the file to play nice with debugging tools. +func ReadHeader(r *wire.Reader) (length uint64, object bool, err error) { + // Read the header. + err = safely(func() { + length = wire.LoadUint(r) + }) + if err != nil { + // On the header, pass raw I/O errors. + if sErr, ok := err.(*ErrState); ok { + return 0, false, sErr.Unwrap() + } + } + + // Decode whether the object is valid. + object = length&objectFlag != 0 + length &^= objectFlag + return +} diff --git a/pkg/state/decode_unsafe.go b/pkg/state/decode_unsafe.go new file mode 100644 index 0000000..a2fdf11 --- /dev/null +++ b/pkg/state/decode_unsafe.go @@ -0,0 +1,76 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "fmt" + "reflect" + "runtime" + "unsafe" +) + +// reflectValueRWAddr is equivalent to obj.Addr(), except that the returned +// reflect.Value is usable in assignments even if obj was obtained by the use +// of unexported struct fields. +// +// Preconditions: obj.CanAddr(). +func reflectValueRWAddr(obj reflect.Value) reflect.Value { + return reflect.NewAt(obj.Type(), unsafe.Pointer(obj.UnsafeAddr())) +} + +// reflectValueRWSlice3 is equivalent to arr.Slice3(i, j, k), except that the +// returned reflect.Value is usable in assignments even if obj was obtained by +// the use of unexported struct fields. +// +// Preconditions: +// - arr.Kind() == reflect.Array. +// - i, j, k >= 0. +// - i <= j <= k <= arr.Len(). +func reflectValueRWSlice3(arr reflect.Value, i, j, k int) reflect.Value { + if arr.Kind() != reflect.Array { + panic(fmt.Sprintf("arr has kind %v, wanted %v", arr.Kind(), reflect.Array)) + } + if i < 0 || j < 0 || k < 0 { + panic(fmt.Sprintf("negative subscripts (%d, %d, %d)", i, j, k)) + } + if i > j { + panic(fmt.Sprintf("subscript i (%d) > j (%d)", i, j)) + } + if j > k { + panic(fmt.Sprintf("subscript j (%d) > k (%d)", j, k)) + } + if k > arr.Len() { + panic(fmt.Sprintf("subscript k (%d) > array length (%d)", k, arr.Len())) + } + + sliceTyp := reflect.SliceOf(arr.Type().Elem()) + if i == arr.Len() { + // By precondition, i == j == k == arr.Len(). + return reflect.MakeSlice(sliceTyp, 0, 0) + } + slh := reflect.SliceHeader{ + // reflect.Value.CanAddr() == false for arrays, so we need to get the + // address from the first element of the array. + Data: arr.Index(i).UnsafeAddr(), + Len: j - i, + Cap: k - i, + } + slobj := reflect.NewAt(sliceTyp, unsafe.Pointer(&slh)).Elem() + // Before slobj is constructed, arr holds the only pointer-typed pointer to + // the array since reflect.SliceHeader.Data is a uintptr, so arr must be + // kept alive. + runtime.KeepAlive(arr) + return slobj +} diff --git a/pkg/state/deferred_list.go b/pkg/state/deferred_list.go new file mode 100644 index 0000000..a18b8bc --- /dev/null +++ b/pkg/state/deferred_list.go @@ -0,0 +1,239 @@ +package state + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type deferredElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (deferredElementMapper) linkerFor(elem *objectEncodeState) *objectEncodeState { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type deferredList struct { + head *objectEncodeState + tail *objectEncodeState +} + +// Reset resets list l to the empty state. +func (l *deferredList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *deferredList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *deferredList) Front() *objectEncodeState { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *deferredList) Back() *objectEncodeState { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *deferredList) Len() (count int) { + for e := l.Front(); e != nil; e = (deferredElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *deferredList) PushFront(e *objectEncodeState) { + linker := deferredElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + deferredElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *deferredList) PushFrontList(m *deferredList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + deferredElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + deferredElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *deferredList) PushBack(e *objectEncodeState) { + linker := deferredElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + deferredElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *deferredList) PushBackList(m *deferredList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + deferredElementMapper{}.linkerFor(l.tail).SetNext(m.head) + deferredElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *deferredList) InsertAfter(b, e *objectEncodeState) { + bLinker := deferredElementMapper{}.linkerFor(b) + eLinker := deferredElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + deferredElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *deferredList) InsertBefore(a, e *objectEncodeState) { + aLinker := deferredElementMapper{}.linkerFor(a) + eLinker := deferredElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + deferredElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *deferredList) Remove(e *objectEncodeState) { + linker := deferredElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + deferredElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + deferredElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type deferredEntry struct { + next *objectEncodeState + prev *objectEncodeState +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *deferredEntry) Next() *objectEncodeState { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *deferredEntry) Prev() *objectEncodeState { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *deferredEntry) SetNext(elem *objectEncodeState) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *deferredEntry) SetPrev(elem *objectEncodeState) { + e.prev = elem +} diff --git a/pkg/state/encode.go b/pkg/state/encode.go new file mode 100644 index 0000000..e8786ba --- /dev/null +++ b/pkg/state/encode.go @@ -0,0 +1,873 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "context" + "reflect" + "sort" + + "github.com/sagernet/gvisor/pkg/state/wire" +) + +// objectEncodeState the type and identity of an object occupying a memory +// address range. This is the value type for addrSet, and the intrusive entry +// for the deferred list. +type objectEncodeState struct { + // id is the assigned ID for this object. + id objectID + + // obj is the object value. Note that this may be replaced if we + // encounter an object that contains this object. When this happens (in + // resolve), we will update existing references appropriately, below, + // and defer a re-encoding of the object. + obj reflect.Value + + // encoded is the encoded value of this object. Note that this may not + // be up to date if this object is still in the deferred list. + encoded wire.Object + + // how indicates whether this object should be encoded as a value. This + // is used only for deferred encoding. + how encodeStrategy + + // refs are the list of reference objects used by other objects + // referring to this object. When the object is updated, these + // references may be updated directly and automatically. + refs []*wire.Ref + + deferredEntry +} + +// encodeState is state used for encoding. +// +// The encoding process constructs a representation of the in-memory graph of +// objects before a single object is serialized. This is done to ensure that +// all references can be fully disambiguated. See resolve for more details. +type encodeState struct { + // ctx is the encode context. + ctx context.Context + + // w is the output stream. + w wire.Writer + + // types is the type database. + types typeEncodeDatabase + + // lastID is the last allocated object ID. + lastID objectID + + // values tracks the address ranges occupied by objects, along with the + // types of these objects. This is used to locate pointer targets, + // including pointers to fields within another type. + // + // Multiple objects may overlap in memory iff the larger object fully + // contains the smaller one, and the type of the smaller object matches + // a field or array element's type at the appropriate offset. An + // arbitrary number of objects may be nested in this manner. + // + // Note that this does not track zero-sized objects, those are tracked + // by zeroValues below. + values addrSet + + // zeroValues tracks zero-sized objects. + zeroValues map[reflect.Type]*objectEncodeState + + // deferred is the list of objects to be encoded. + deferred deferredList + + // pendingTypes is the list of types to be serialized. Serialization + // will occur when all objects have been encoded, but before pending is + // serialized. + pendingTypes []wire.Type + + // pending maps object IDs to objects to be serialized. Serialization does + // not actually occur until the full object graph is computed. + pending map[objectID]*objectEncodeState + + // encodedStructs maps reflect.Values representing structs to previous + // encodings of those structs. This is necessary to avoid duplicate calls + // to SaverLoader.StateSave() that may result in multiple calls to + // Sink.SaveValue() for a given field, resulting in object duplication. + encodedStructs map[reflect.Value]*wire.Struct + + // stats tracks time data. + stats Stats +} + +// isSameSizeParent returns true if child is a field value or element within +// parent. Only a struct or array can have a child value. +// +// isSameSizeParent deals with objects like this: +// +// struct child { +// // fields.. +// } +// +// struct parent { +// c child +// } +// +// var p parent +// record(&p.c) +// +// Here, &p and &p.c occupy the exact same address range. +// +// Or like this: +// +// struct child { +// // fields +// } +// +// var arr [1]parent +// record(&arr[0]) +// +// Similarly, &arr[0] and &arr[0].c have the exact same address range. +// +// Precondition: parent and child must occupy the same memory. +func isSameSizeParent(parent reflect.Value, childType reflect.Type) bool { + switch parent.Kind() { + case reflect.Struct: + for i := 0; i < parent.NumField(); i++ { + field := parent.Field(i) + if field.Type() == childType { + return true + } + // Recurse through any intermediate types. + if isSameSizeParent(field, childType) { + return true + } + // Does it make sense to keep going if the first field + // doesn't match? Yes, because there might be an + // arbitrary number of zero-sized fields before we get + // a match, and childType itself can be zero-sized. + } + return false + case reflect.Array: + // The only case where an array with more than one elements can + // return true is if childType is zero-sized. In such cases, + // it's ambiguous which element contains the match since a + // zero-sized child object fully fits in any of the zero-sized + // elements in an array... However since all elements are of + // the same type, we only need to check one element. + // + // For non-zero-sized childTypes, parent.Len() must be 1, but a + // combination of the precondition and an implicit comparison + // between the array element size and childType ensures this. + return parent.Len() > 0 && isSameSizeParent(parent.Index(0), childType) + default: + return false + } +} + +// nextID returns the next valid ID. +func (es *encodeState) nextID() objectID { + es.lastID++ + return objectID(es.lastID) +} + +// dummyAddr points to the dummy zero-sized address. +var dummyAddr = reflect.ValueOf(new(struct{})).Pointer() + +// resolve records the address range occupied by an object. +func (es *encodeState) resolve(obj reflect.Value, ref *wire.Ref) { + addr := obj.Pointer() + + // Is this a map pointer? Just record the single address. It is not + // possible to take any pointers into the map internals. + if obj.Kind() == reflect.Map { + if addr == 0 { + // Just leave the nil reference alone. This is fine, we + // may need to encode as a reference in this way. We + // return nil for our objectEncodeState so that anyone + // depending on this value knows there's nothing there. + return + } + seg, gap := es.values.Find(addr) + if seg.Ok() { + // Ensure the map types match. + existing := seg.Value() + if existing.obj.Type() != obj.Type() { + Failf("overlapping map objects at 0x%x: [new object] %#v [existing object type] %s", addr, obj, existing.obj) + } + + // No sense recording refs, maps may not be replaced by + // covering objects, they are maximal. + ref.Root = wire.Uint(existing.id) + return + } + + // Record the map. + r := addrRange{addr, addr + 1} + oes := &objectEncodeState{ + id: es.nextID(), + obj: obj, + how: encodeMapAsValue, + } + // Use Insert instead of InsertWithoutMergingUnchecked when race + // detection is enabled to get additional sanity-checking from Merge. + if !raceEnabled { + es.values.InsertWithoutMergingUnchecked(gap, r, oes) + } else { + es.values.Insert(gap, r, oes) + } + es.pending[oes.id] = oes + es.deferred.PushBack(oes) + + // See above: no ref recording. + ref.Root = wire.Uint(oes.id) + return + } + + // If not a map, then the object must be a pointer. + if obj.Kind() != reflect.Ptr { + Failf("attempt to record non-map and non-pointer object %#v", obj) + } + + obj = obj.Elem() // Value from here. + + // Is this a zero-sized type? + typ := obj.Type() + size := typ.Size() + if size == 0 { + if addr == dummyAddr { + // Zero-sized objects point to a dummy byte within the + // runtime. There's no sense recording this in the + // address map. We add this to the dedicated + // zeroValues. + // + // Note that zero-sized objects must be *true* + // zero-sized objects. They cannot be part of some + // larger object. In that case, they are assigned a + // 1-byte address at the end of the object. + oes, ok := es.zeroValues[typ] + if !ok { + oes = &objectEncodeState{ + id: es.nextID(), + obj: obj, + } + es.zeroValues[typ] = oes + es.pending[oes.id] = oes + es.deferred.PushBack(oes) + } + + // There's also no sense tracking back references. We + // know that this is a true zero-sized object, and not + // part of a larger container, so it will not change. + ref.Root = wire.Uint(oes.id) + return + } + size = 1 // See above. + } + + end := addr + size + r := addrRange{addr, end} + seg := es.values.LowerBoundSegment(addr) + var ( + oes *objectEncodeState + gap addrGapIterator + ) + + // Does at least one previously-registered object overlap this one? + if seg.Ok() && seg.Start() < end { + existing := seg.Value() + + if seg.Range() == r && typ == existing.obj.Type() { + // This exact object is already registered. Avoid the traversal and + // just return directly. We don't need to encode the type + // information or any dots here. + ref.Root = wire.Uint(existing.id) + existing.refs = append(existing.refs, ref) + return + } + + if seg.Range().IsSupersetOf(r) && (seg.Range() != r || isSameSizeParent(existing.obj, typ)) { + // This object is contained within a previously-registered object. + // Perform traversal from the container to the new object. + ref.Root = wire.Uint(existing.id) + ref.Dots = traverse(existing.obj.Type(), typ, seg.Start(), addr) + ref.Type = es.findType(existing.obj.Type()) + existing.refs = append(existing.refs, ref) + return + } + + // This object contains one or more previously-registered objects. + // Remove them and update existing references to use the new one. + oes := &objectEncodeState{ + // Reuse the root ID of the first contained element. + id: existing.id, + obj: obj, + } + type elementEncodeState struct { + addr uintptr + typ reflect.Type + refs []*wire.Ref + } + var ( + elems []elementEncodeState + gap addrGapIterator + ) + for { + // Each contained object should be completely contained within + // this one. + if raceEnabled && !r.IsSupersetOf(seg.Range()) { + Failf("containing object %#v does not contain existing object %#v", obj, existing.obj) + } + elems = append(elems, elementEncodeState{ + addr: seg.Start(), + typ: existing.obj.Type(), + refs: existing.refs, + }) + delete(es.pending, existing.id) + es.deferred.Remove(existing) + gap = es.values.Remove(seg) + seg = gap.NextSegment() + if !seg.Ok() || seg.Start() >= end { + break + } + existing = seg.Value() + } + wt := es.findType(typ) + for _, elem := range elems { + dots := traverse(typ, elem.typ, addr, elem.addr) + for _, ref := range elem.refs { + ref.Root = wire.Uint(oes.id) + ref.Dots = append(ref.Dots, dots...) + ref.Type = wt + } + oes.refs = append(oes.refs, elem.refs...) + } + // Finally register the new containing object. + if !raceEnabled { + es.values.InsertWithoutMergingUnchecked(gap, r, oes) + } else { + es.values.Insert(gap, r, oes) + } + es.pending[oes.id] = oes + es.deferred.PushBack(oes) + ref.Root = wire.Uint(oes.id) + oes.refs = append(oes.refs, ref) + return + } + + // No existing object overlaps this one. Register a new object. + oes = &objectEncodeState{ + id: es.nextID(), + obj: obj, + } + if seg.Ok() { + gap = seg.PrevGap() + } else { + gap = es.values.LastGap() + } + if !raceEnabled { + es.values.InsertWithoutMergingUnchecked(gap, r, oes) + } else { + es.values.Insert(gap, r, oes) + } + es.pending[oes.id] = oes + es.deferred.PushBack(oes) + ref.Root = wire.Uint(oes.id) + oes.refs = append(oes.refs, ref) +} + +// traverse searches for a target object within a root object, where the target +// object is a struct field or array element within root, with potentially +// multiple intervening types. traverse returns the set of field or element +// traversals required to reach the target. +// +// Note that for efficiency, traverse returns the dots in the reverse order. +// That is, the first traversal required will be the last element of the list. +// +// Precondition: The target object must lie completely within the range defined +// by [rootAddr, rootAddr + sizeof(rootType)]. +func traverse(rootType, targetType reflect.Type, rootAddr, targetAddr uintptr) []wire.Dot { + // Recursion base case: the types actually match. + if targetType == rootType && targetAddr == rootAddr { + return nil + } + + switch rootType.Kind() { + case reflect.Struct: + offset := targetAddr - rootAddr + for i := rootType.NumField(); i > 0; i-- { + field := rootType.Field(i - 1) + // The first field from the end with an offset that is + // smaller than or equal to our address offset is where + // the target is located. Traverse from there. + if field.Offset <= offset { + dots := traverse(field.Type, targetType, rootAddr+field.Offset, targetAddr) + fieldName := wire.FieldName(field.Name) + return append(dots, &fieldName) + } + } + // Should never happen; the target should be reachable. + Failf("no field in root type %v contains target type %v", rootType, targetType) + + case reflect.Array: + // Since arrays have homogeneous types, all elements have the + // same size and we can compute where the target lives. This + // does not matter for the purpose of typing, but matters for + // the purpose of computing the address of the given index. + elemSize := int(rootType.Elem().Size()) + n := int(targetAddr-rootAddr) / elemSize // Relies on integer division rounding down. + if rootType.Len() < n { + Failf("traversal target of type %v @%x is beyond the end of the array type %v @%x with %v elements", + targetType, targetAddr, rootType, rootAddr, rootType.Len()) + } + dots := traverse(rootType.Elem(), targetType, rootAddr+uintptr(n*elemSize), targetAddr) + return append(dots, wire.Index(n)) + + default: + // For any other type, there's no possibility of aliasing so if + // the types didn't match earlier then we have an address + // collision which shouldn't be possible at this point. + Failf("traverse failed for root type %v and target type %v", rootType, targetType) + } + panic("unreachable") +} + +// encodeMap encodes a map. +func (es *encodeState) encodeMap(obj reflect.Value, dest *wire.Object) { + if obj.IsNil() { + // Because there is a difference between a nil map and an empty + // map, we need to not decode in the case of a truly nil map. + *dest = wire.Nil{} + return + } + l := obj.Len() + m := &wire.Map{ + Keys: make([]wire.Object, l), + Values: make([]wire.Object, l), + } + *dest = m + for i, k := range obj.MapKeys() { + v := obj.MapIndex(k) + // Map keys must be encoded using the full value because the + // type will be omitted after the first key. + es.encodeObject(k, encodeAsValue, &m.Keys[i]) + es.encodeObject(v, encodeAsValue, &m.Values[i]) + } +} + +// objectEncoder is for encoding structs. +type objectEncoder struct { + // es is encodeState. + es *encodeState + + // encoded is the encoded struct. + encoded *wire.Struct +} + +// save is called by the public methods on Sink. +func (oe *objectEncoder) save(slot int, obj reflect.Value) { + fieldValue := oe.encoded.Field(slot) + oe.es.encodeObject(obj, encodeDefault, fieldValue) +} + +// encodeStruct encodes a composite object. +func (es *encodeState) encodeStruct(obj reflect.Value, dest *wire.Object) { + if s, ok := es.encodedStructs[obj]; ok { + *dest = s + return + } + s := &wire.Struct{} + *dest = s + es.encodedStructs[obj] = s + + // Ensure that the obj is addressable. There are two cases when it is + // not. First, is when this is dispatched via SaveValue. Second, when + // this is a map key as a struct. Either way, we need to make a copy to + // obtain an addressable value. + if !obj.CanAddr() { + localObj := reflect.New(obj.Type()) + localObj.Elem().Set(obj) + obj = localObj.Elem() + } + + // Look the type up in the database. + te, ok := es.types.Lookup(obj.Type()) + if te == nil { + if obj.NumField() == 0 { + // Allow unregistered anonymous, empty structs. This + // will just return success without ever invoking the + // passed function. This uses the immutable EmptyStruct + // variable to prevent an allocation in this case. + // + // Note that this mechanism does *not* work for + // interfaces in general. So you can't dispatch + // non-registered empty structs via interfaces because + // then they can't be restored. + s.Alloc(0) + return + } + // We need a SaverLoader for struct types. + Failf("struct %T does not implement SaverLoader", obj.Interface()) + } + if !ok { + // Queue the type to be serialized. + es.pendingTypes = append(es.pendingTypes, te.Type) + } + + // Invoke the provided saver. + s.TypeID = wire.TypeID(te.ID) + s.Alloc(len(te.Fields)) + oe := objectEncoder{ + es: es, + encoded: s, + } + es.stats.start(te.ID) + defer es.stats.done() + if sl, ok := obj.Addr().Interface().(SaverLoader); ok { + // Note: may be a registered empty struct which does not + // implement the saver/loader interfaces. + sl.StateSave(Sink{internal: oe}) + } +} + +// encodeArray encodes an array. +func (es *encodeState) encodeArray(obj reflect.Value, dest *wire.Object) { + l := obj.Len() + a := &wire.Array{ + Contents: make([]wire.Object, l), + } + *dest = a + for i := 0; i < l; i++ { + // We need to encode the full value because arrays are encoded + // using the type information from only the first element. + es.encodeObject(obj.Index(i), encodeAsValue, &a.Contents[i]) + } +} + +// findType recursively finds type information. +func (es *encodeState) findType(typ reflect.Type) wire.TypeSpec { + // First: check if this is a proper type. It's possible for pointers, + // slices, arrays, maps, etc to all have some different type. + te, ok := es.types.Lookup(typ) + if te != nil { + if !ok { + // See encodeStruct. + es.pendingTypes = append(es.pendingTypes, te.Type) + } + return wire.TypeID(te.ID) + } + + switch typ.Kind() { + case reflect.Ptr: + return &wire.TypeSpecPointer{ + Type: es.findType(typ.Elem()), + } + case reflect.Slice: + return &wire.TypeSpecSlice{ + Type: es.findType(typ.Elem()), + } + case reflect.Array: + return &wire.TypeSpecArray{ + Count: wire.Uint(typ.Len()), + Type: es.findType(typ.Elem()), + } + case reflect.Map: + return &wire.TypeSpecMap{ + Key: es.findType(typ.Key()), + Value: es.findType(typ.Elem()), + } + default: + // After potentially chasing many pointers, the + // ultimate type of the object is not known. + Failf("type %q is not known", typ) + } + panic("unreachable") +} + +// encodeInterface encodes an interface. +func (es *encodeState) encodeInterface(obj reflect.Value, dest *wire.Object) { + // Dereference the object. + obj = obj.Elem() + if !obj.IsValid() { + // Special case: the nil object. + *dest = &wire.Interface{ + Type: wire.TypeSpecNil{}, + Value: wire.Nil{}, + } + return + } + + // Encode underlying object. + i := &wire.Interface{ + Type: es.findType(obj.Type()), + } + *dest = i + es.encodeObject(obj, encodeAsValue, &i.Value) +} + +// isPrimitive returns true if this is a primitive object, or a composite +// object composed entirely of primitives. +func isPrimitiveZero(typ reflect.Type) bool { + switch typ.Kind() { + case reflect.Ptr: + // Pointers are always treated as primitive types because we + // won't encode directly from here. Returning true here won't + // prevent the object from being encoded correctly. + return true + case reflect.Bool: + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return true + case reflect.Float32, reflect.Float64: + return true + case reflect.Complex64, reflect.Complex128: + return true + case reflect.String: + return true + case reflect.Slice: + // The slice itself a primitive, but not necessarily the array + // that points to. This is similar to a pointer. + return true + case reflect.Array: + // We cannot treat an array as a primitive, because it may be + // composed of structures or other things with side-effects. + return isPrimitiveZero(typ.Elem()) + case reflect.Interface: + // Since we now that this type is the zero type, the interface + // value must be zero. Therefore this is primitive. + return true + case reflect.Struct: + return false + case reflect.Map: + // The isPrimitiveZero function is called only on zero-types to + // see if it's safe to serialize. Since a zero map has no + // elements, it is safe to treat as a primitive. + return true + default: + Failf("unknown type %q", typ.Name()) + } + panic("unreachable") +} + +// encodeStrategy is the strategy used for encodeObject. +type encodeStrategy int + +const ( + // encodeDefault means types are encoded normally as references. + encodeDefault encodeStrategy = iota + + // encodeAsValue means that types will never take short-circuited and + // will always be encoded as a normal value. + encodeAsValue + + // encodeMapAsValue means that even maps will be fully encoded. + encodeMapAsValue +) + +// encodeObject encodes an object. +func (es *encodeState) encodeObject(obj reflect.Value, how encodeStrategy, dest *wire.Object) { + if how == encodeDefault && isPrimitiveZero(obj.Type()) && obj.IsZero() { + *dest = wire.Nil{} + return + } + switch obj.Kind() { + case reflect.Ptr: // Fast path: first. + r := new(wire.Ref) + *dest = r + if obj.IsNil() { + // May be in an array or elsewhere such that a value is + // required. So we encode as a reference to the zero + // object, which does not exist. Note that this has to + // be handled correctly in the decode path as well. + return + } + es.resolve(obj, r) + case reflect.Bool: + *dest = wire.Bool(obj.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + *dest = wire.Int(obj.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + *dest = wire.Uint(obj.Uint()) + case reflect.Float32: + *dest = wire.Float32(obj.Float()) + case reflect.Float64: + *dest = wire.Float64(obj.Float()) + case reflect.Complex64: + c := wire.Complex64(obj.Complex()) + *dest = &c // Needs alloc. + case reflect.Complex128: + c := wire.Complex128(obj.Complex()) + *dest = &c // Needs alloc. + case reflect.String: + s := wire.String(obj.String()) + *dest = &s // Needs alloc. + case reflect.Array: + es.encodeArray(obj, dest) + case reflect.Slice: + s := &wire.Slice{ + Capacity: wire.Uint(obj.Cap()), + Length: wire.Uint(obj.Len()), + } + *dest = s + // Note that we do need to provide a wire.Slice type here as + // how is not encodeDefault. If this were the case, then it + // would have been caught by the IsZero check above and we + // would have just used wire.Nil{}. + if obj.IsNil() { + return + } + // Slices need pointer resolution. + es.resolve(arrayFromSlice(obj), &s.Ref) + case reflect.Interface: + es.encodeInterface(obj, dest) + case reflect.Struct: + es.encodeStruct(obj, dest) + case reflect.Map: + if how == encodeMapAsValue { + es.encodeMap(obj, dest) + return + } + r := new(wire.Ref) + *dest = r + es.resolve(obj, r) + default: + Failf("unknown object %#v", obj.Interface()) + panic("unreachable") + } +} + +// Save serializes the object graph rooted at obj. +func (es *encodeState) Save(obj reflect.Value) { + es.stats.init() + defer es.stats.fini(func(id typeID) string { + return es.pendingTypes[id-1].Name + }) + + // Resolve the first object, which should queue a pile of additional + // objects on the pending list. All queued objects should be fully + // resolved, and we should be able to serialize after this call. + var root wire.Ref + es.resolve(obj.Addr(), &root) + + // Encode the graph. + var oes *objectEncodeState + if err := safely(func() { + for oes = es.deferred.Front(); oes != nil; oes = es.deferred.Front() { + // Remove and encode the object. Note that as a result + // of this encoding, the object may be enqueued on the + // deferred list yet again. That's expected, and why it + // is removed first. + es.deferred.Remove(oes) + es.encodeObject(oes.obj, oes.how, &oes.encoded) + } + }); err != nil { + // Include the object in the error message. + Failf("encoding error: %w\nfor object %#v", err, oes.obj.Interface()) + } + + // Check that we have objects to serialize. + if len(es.pending) == 0 { + Failf("pending is empty?") + } + + // Write the header with the number of objects. + if err := WriteHeader(&es.w, uint64(len(es.pending)), true); err != nil { + Failf("error writing header: %w", err) + } + + // Serialize all pending types and pending objects. Note that we don't + // bother removing from this list as we walk it because that just + // wastes time. It will not change after this point. + if err := safely(func() { + for _, wt := range es.pendingTypes { + // Encode the type. + wire.Save(&es.w, &wt) + } + // Emit objects in ID order. + ids := make([]objectID, 0, len(es.pending)) + for id := range es.pending { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + for _, id := range ids { + // Encode the id. + wire.Save(&es.w, wire.Uint(id)) + // Marshal the object. + oes := es.pending[id] + wire.Save(&es.w, oes.encoded) + } + }); err != nil { + // Include the object and the error. + Failf("error serializing object %#v: %w", oes.encoded, err) + } +} + +// objectFlag indicates that the length is a # of objects, rather than a raw +// byte length. When this is set on a length header in the stream, it may be +// decoded appropriately. +const objectFlag uint64 = 1 << 63 + +// WriteHeader writes a header. +// +// Each object written to the statefile should be prefixed with a header. In +// order to generate statefiles that play nicely with debugging tools, raw +// writes should be prefixed with a header with object set to false and the +// appropriate length. This will allow tools to skip these regions. +func WriteHeader(w *wire.Writer, length uint64, object bool) error { + // Sanity check the length. + if length&objectFlag != 0 { + Failf("impossibly huge length: %d", length) + } + if object { + length |= objectFlag + } + + // Write a header. + return safely(func() { + wire.SaveUint(w, length) + }) +} + +// addrSetFunctions is used by addrSet. +type addrSetFunctions struct{} + +func (addrSetFunctions) MinKey() uintptr { + return 0 +} + +func (addrSetFunctions) MaxKey() uintptr { + return ^uintptr(0) +} + +func (addrSetFunctions) ClearValue(val **objectEncodeState) { + *val = nil +} + +func (addrSetFunctions) Merge(r1 addrRange, val1 *objectEncodeState, r2 addrRange, val2 *objectEncodeState) (*objectEncodeState, bool) { + if val1.obj == val2.obj { + // This, should never happen. It would indicate that the same + // object exists in two non-contiguous address ranges. Note + // that this assertion can only be triggered if the race + // detector is enabled. + Failf("unexpected merge in addrSet @ %v and %v: %#v and %#v", r1, r2, val1.obj, val2.obj) + } + // Reject the merge. + return val1, false +} + +func (addrSetFunctions) Split(r addrRange, val *objectEncodeState, _ uintptr) (*objectEncodeState, *objectEncodeState) { + // A split should never happen: we don't remove ranges. + Failf("unexpected split in addrSet @ %v: %#v", r, val.obj) + panic("unreachable") +} diff --git a/pkg/state/encode_unsafe.go b/pkg/state/encode_unsafe.go new file mode 100644 index 0000000..78e36e7 --- /dev/null +++ b/pkg/state/encode_unsafe.go @@ -0,0 +1,32 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "reflect" + "unsafe" +) + +// arrayFromSlice constructs a new pointer to the slice data. +// +// It would be similar to the following: +// +// x := make([]Foo, l, c) +// a := ([l]Foo*)(unsafe.Pointer(x[0])) +func arrayFromSlice(obj reflect.Value) reflect.Value { + return reflect.NewAt( + reflect.ArrayOf(obj.Cap(), obj.Type().Elem()), + unsafe.Pointer(obj.Pointer())) +} diff --git a/pkg/state/ods_list.go b/pkg/state/ods_list.go new file mode 100644 index 0000000..d71f1f5 --- /dev/null +++ b/pkg/state/ods_list.go @@ -0,0 +1,239 @@ +package state + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type odsElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (odsElementMapper) linkerFor(elem *odsListElem) *odsListElem { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type odsList struct { + head *odsListElem + tail *odsListElem +} + +// Reset resets list l to the empty state. +func (l *odsList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *odsList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *odsList) Front() *odsListElem { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *odsList) Back() *odsListElem { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *odsList) Len() (count int) { + for e := l.Front(); e != nil; e = (odsElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *odsList) PushFront(e *odsListElem) { + linker := odsElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + odsElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *odsList) PushFrontList(m *odsList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + odsElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + odsElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *odsList) PushBack(e *odsListElem) { + linker := odsElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + odsElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *odsList) PushBackList(m *odsList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + odsElementMapper{}.linkerFor(l.tail).SetNext(m.head) + odsElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *odsList) InsertAfter(b, e *odsListElem) { + bLinker := odsElementMapper{}.linkerFor(b) + eLinker := odsElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + odsElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *odsList) InsertBefore(a, e *odsListElem) { + aLinker := odsElementMapper{}.linkerFor(a) + eLinker := odsElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + odsElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *odsList) Remove(e *odsListElem) { + linker := odsElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + odsElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + odsElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type odsEntry struct { + next *odsListElem + prev *odsListElem +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *odsEntry) Next() *odsListElem { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *odsEntry) Prev() *odsListElem { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *odsEntry) SetNext(elem *odsListElem) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *odsEntry) SetPrev(elem *odsListElem) { + e.prev = elem +} diff --git a/pkg/state/pretty/pretty.go b/pkg/state/pretty/pretty.go new file mode 100644 index 0000000..6d02e79 --- /dev/null +++ b/pkg/state/pretty/pretty.go @@ -0,0 +1,307 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pretty is a pretty-printer for state streams. +package pretty + +import ( + "fmt" + "io" + "reflect" + "strings" + + "github.com/sagernet/gvisor/pkg/state" + "github.com/sagernet/gvisor/pkg/state/wire" +) + +type printer struct { + html bool + typeSpecs map[string]*wire.Type +} + +func (p *printer) formatRef(x *wire.Ref, graph uint64) string { + baseRef := fmt.Sprintf("g%dr%d", graph, x.Root) + fullRef := baseRef + if len(x.Dots) > 0 { + // See wire.Ref; Type valid if Dots non-zero. + typ, _ := p.formatType(x.Type, graph) + var buf strings.Builder + buf.WriteString("(*") + buf.WriteString(typ) + buf.WriteString(")(") + buf.WriteString(baseRef) + buf.WriteString(")") + for _, component := range x.Dots { + switch v := component.(type) { + case *wire.FieldName: + buf.WriteString(".") + buf.WriteString(string(*v)) + case wire.Index: + buf.WriteString(fmt.Sprintf("[%d]", v)) + default: + panic(fmt.Sprintf("unreachable: switch should be exhaustive, unhandled case %v", reflect.TypeOf(component))) + } + } + fullRef = buf.String() + } + if p.html { + return fmt.Sprintf("%s", baseRef, fullRef) + } + return fullRef +} + +func (p *printer) formatType(t wire.TypeSpec, graph uint64) (string, bool) { + switch x := t.(type) { + case wire.TypeID: + tag := fmt.Sprintf("g%dt%d", graph, x) + desc := tag + if spec, ok := p.typeSpecs[tag]; ok { + desc += fmt.Sprintf("=%s", spec.Name) + } else { + desc += "!missing-type-spec" + } + if p.html { + return fmt.Sprintf("%s", tag, desc), true + } + return desc, true + case wire.TypeSpecNil: + return "", false // Only nil type. + case *wire.TypeSpecPointer: + element, _ := p.formatType(x.Type, graph) + return fmt.Sprintf("(*%s)", element), true + case *wire.TypeSpecArray: + element, _ := p.formatType(x.Type, graph) + return fmt.Sprintf("[%d](%s)", x.Count, element), true + case *wire.TypeSpecSlice: + element, _ := p.formatType(x.Type, graph) + return fmt.Sprintf("([]%s)", element), true + case *wire.TypeSpecMap: + key, _ := p.formatType(x.Key, graph) + value, _ := p.formatType(x.Value, graph) + return fmt.Sprintf("(map[%s]%s)", key, value), true + default: + panic(fmt.Sprintf("unreachable: unknown type %T", t)) + } +} + +// format formats a single object, for pretty-printing. It also returns whether +// the value is a non-zero value. +func (p *printer) format(graph uint64, depth int, encoded wire.Object) (string, bool) { + switch x := encoded.(type) { + case wire.Nil: + return "nil", false + case *wire.String: + return fmt.Sprintf("%q", *x), *x != "" + case *wire.Complex64: + return fmt.Sprintf("%f+%fi", real(*x), imag(*x)), *x != 0.0 + case *wire.Complex128: + return fmt.Sprintf("%f+%fi", real(*x), imag(*x)), *x != 0.0 + case *wire.Ref: + return p.formatRef(x, graph), x.Root != 0 + case *wire.Type: + tabs := "\n" + strings.Repeat("\t", depth) + items := make([]string, 0, len(x.Fields)+2) + items = append(items, fmt.Sprintf("type %s {", x.Name)) + for i := 0; i < len(x.Fields); i++ { + items = append(items, fmt.Sprintf("\t%d: %s,", i, x.Fields[i])) + } + items = append(items, "}") + return strings.Join(items, tabs), true // No zero value. + case *wire.Slice: + return fmt.Sprintf("%s{len:%d,cap:%d}", p.formatRef(&x.Ref, graph), x.Length, x.Capacity), x.Capacity != 0 + case *wire.Array: + if len(x.Contents) == 0 { + return "[]", false + } + items := make([]string, 0, len(x.Contents)+2) + zeros := make([]string, 0) // used to eliminate zero entries. + items = append(items, "[") + tabs := "\n" + strings.Repeat("\t", depth) + for i := 0; i < len(x.Contents); i++ { + item, ok := p.format(graph, depth+1, x.Contents[i]) + if !ok { + zeros = append(zeros, fmt.Sprintf("\t%s,", item)) + continue + } + if len(zeros) > 0 { + items = append(items, zeros...) + zeros = nil + } + items = append(items, fmt.Sprintf("\t%s,", item)) + } + if len(zeros) > 0 { + items = append(items, fmt.Sprintf("\t... (%d zeros),", len(zeros))) + } + items = append(items, "]") + return strings.Join(items, tabs), len(zeros) < len(x.Contents) + case *wire.Struct: + tag := fmt.Sprintf("g%dt%d", graph, x.TypeID) + spec, _ := p.typeSpecs[tag] + typ, _ := p.formatType(x.TypeID, graph) + if x.Fields() == 0 { + return fmt.Sprintf("struct[%s]{}", typ), false + } + items := make([]string, 0, 2) + items = append(items, fmt.Sprintf("struct[%s]{", typ)) + tabs := "\n" + strings.Repeat("\t", depth) + allZero := true + for i := 0; i < x.Fields(); i++ { + var name string + if spec != nil && i < len(spec.Fields) { + name = spec.Fields[i] + } else { + name = fmt.Sprintf("%d", i) + } + element, ok := p.format(graph, depth+1, *x.Field(i)) + allZero = allZero && !ok + items = append(items, fmt.Sprintf("\t%s: %s,", name, element)) + } + items = append(items, "}") + return strings.Join(items, tabs), !allZero + case *wire.Map: + if len(x.Keys) == 0 { + return "map{}", false + } + items := make([]string, 0, len(x.Keys)+2) + items = append(items, "map{") + tabs := "\n" + strings.Repeat("\t", depth) + for i := 0; i < len(x.Keys); i++ { + key, _ := p.format(graph, depth+1, x.Keys[i]) + value, _ := p.format(graph, depth+1, x.Values[i]) + items = append(items, fmt.Sprintf("\t%s: %s,", key, value)) + } + items = append(items, "}") + return strings.Join(items, tabs), true + case *wire.Interface: + typ, typOk := p.formatType(x.Type, graph) + element, elementOk := p.format(graph, depth+1, x.Value) + return fmt.Sprintf("interface[%s]{%s}", typ, element), typOk || elementOk + default: + // Must be a primitive; use reflection. + return fmt.Sprintf("%v", encoded), true + } +} + +// printStream is the basic print implementation. +func (p *printer) printStream(w io.Writer, r io.Reader) (err error) { + wr := wire.Reader{Reader: r} + + // current graph ID. + var graph uint64 + + if p.html { + fmt.Fprintf(w, "
")
+		defer fmt.Fprintf(w, "
") + } + + defer func() { + if r := recover(); r != nil { + if rErr, ok := r.(error); ok { + err = rErr // Override return. + return + } + panic(r) // Propagate. + } + }() + + p.typeSpecs = make(map[string]*wire.Type) + + for { + // Find the first object to begin generation. + length, object, err := state.ReadHeader(&wr) + if err == io.EOF { + // Nothing else to do. + break + } else if err != nil { + return err + } + if !object { + graph++ // Increment the graph. + if length > 0 { + fmt.Fprintf(w, "(%d bytes non-object data)\n", length) + io.Copy(io.Discard, &io.LimitedReader{ + R: r, + N: int64(length), + }) + } + continue + } + + // Read & unmarshal the object. + // + // Note that this loop must match the general structure of the + // loop in decode.go. But we don't register type information, + // etc. and just print the raw structures. + type objectAndID struct { + id uint64 + obj wire.Object + } + var ( + tid uint64 = 1 + objects []objectAndID + ) + for i := uint64(0); i < length; { + // Unmarshal either a type object or object ID. + encoded := wire.Load(&wr) + switch we := encoded.(type) { + case *wire.Type: + str, _ := p.format(graph, 0, encoded) + tag := fmt.Sprintf("g%dt%d", graph, tid) + p.typeSpecs[tag] = we + if p.html { + // See below. + tag = fmt.Sprintf("%s⚓", tag, tag, tag) + } + if _, err := fmt.Fprintf(w, "%s = %s\n", tag, str); err != nil { + return err + } + tid++ + case wire.Uint: + // Unmarshal the actual object. + objects = append(objects, objectAndID{ + id: uint64(we), + obj: wire.Load(&wr), + }) + i++ + default: + return fmt.Errorf("wanted type or object ID, got %#v", encoded) + } + } + + for _, objAndID := range objects { + // Format the node. + str, _ := p.format(graph, 0, objAndID.obj) + tag := fmt.Sprintf("g%dr%d", graph, objAndID.id) + if p.html { + // Create a little tag with an anchor next to it for linking. + tag = fmt.Sprintf("%s⚓", tag, tag, tag) + } + if _, err := fmt.Fprintf(w, "%s = %s\n", tag, str); err != nil { + return err + } + } + } + + return nil +} + +// PrintText reads the stream from r and prints text to w. +func PrintText(w io.Writer, r io.Reader) error { + return (&printer{}).printStream(w, r) +} + +// PrintHTML reads the stream from r and prints html to w. +func PrintHTML(w io.Writer, r io.Reader) error { + return (&printer{html: true}).printStream(w, r) +} diff --git a/pkg/state/pretty/pretty_state_autogen.go b/pkg/state/pretty/pretty_state_autogen.go new file mode 100644 index 0000000..e772e34 --- /dev/null +++ b/pkg/state/pretty/pretty_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package pretty diff --git a/pkg/state/state.go b/pkg/state/state.go new file mode 100644 index 0000000..3a8f1f6 --- /dev/null +++ b/pkg/state/state.go @@ -0,0 +1,324 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package state provides functionality related to saving and loading object +// graphs. For most types, it provides a set of default saving / loading logic +// that will be invoked automatically if custom logic is not defined. +// +// Kind Support +// ---- ------- +// Bool default +// Int default +// Int8 default +// Int16 default +// Int32 default +// Int64 default +// Uint default +// Uint8 default +// Uint16 default +// Uint32 default +// Uint64 default +// Float32 default +// Float64 default +// Complex64 default +// Complex128 default +// Array default +// Chan custom +// Func custom +// Interface default +// Map default +// Ptr default +// Slice default +// String default +// Struct custom (*) Unless zero-sized. +// UnsafePointer custom +// +// See README.md for an overview of how encoding and decoding works. +package state + +import ( + "context" + "fmt" + "io" + "reflect" + "runtime" + + "github.com/sagernet/gvisor/pkg/state/wire" +) + +// objectID is a unique identifier assigned to each object to be serialized. +// Each instance of an object is considered separately, i.e. if there are two +// objects of the same type in the object graph being serialized, they'll be +// assigned unique objectIDs. +type objectID uint32 + +// typeID is the identifier for a type. Types are serialized and tracked +// alongside objects in order to avoid the overhead of encoding field names in +// all objects. +type typeID uint32 + +// ErrState is returned when an error is encountered during encode/decode. +type ErrState struct { + // err is the underlying error. + err error + + // trace is the stack trace. + trace string +} + +// Error returns a sensible description of the state error. +func (e *ErrState) Error() string { + return fmt.Sprintf("%v:\n%s", e.err, e.trace) +} + +// Unwrap implements standard unwrapping. +func (e *ErrState) Unwrap() error { + return e.err +} + +// Save saves the given object state. +func Save(ctx context.Context, w io.Writer, rootPtr any) (Stats, error) { + // Create the encoding state. + es := encodeState{ + ctx: ctx, + w: wire.Writer{Writer: w}, + types: makeTypeEncodeDatabase(), + zeroValues: make(map[reflect.Type]*objectEncodeState), + pending: make(map[objectID]*objectEncodeState), + encodedStructs: make(map[reflect.Value]*wire.Struct), + } + + // Perform the encoding. + err := safely(func() { + es.Save(reflect.ValueOf(rootPtr).Elem()) + }) + return es.stats, err +} + +// Load loads a checkpoint. +func Load(ctx context.Context, r io.Reader, rootPtr any) (Stats, error) { + // Create the decoding state. + ds := decodeState{ + ctx: ctx, + r: wire.Reader{Reader: r}, + types: makeTypeDecodeDatabase(), + deferred: make(map[objectID]wire.Object), + } + + // Attempt our decode. + err := safely(func() { + ds.Load(reflect.ValueOf(rootPtr).Elem()) + }) + return ds.stats, err +} + +// Sink is used for Type.StateSave. +type Sink struct { + internal objectEncoder +} + +// Save adds the given object to the map. +// +// You should pass always pointers to the object you are saving. For example: +// +// type X struct { +// A int +// B *int +// } +// +// func (x *X) StateTypeInfo(m Sink) state.TypeInfo { +// return state.TypeInfo{ +// Name: "pkg.X", +// Fields: []string{ +// "A", +// "B", +// }, +// } +// } +// +// func (x *X) StateSave(m Sink) { +// m.Save(0, &x.A) // Field is A. +// m.Save(1, &x.B) // Field is B. +// } +// +// func (x *X) StateLoad(m Source) { +// m.Load(0, &x.A) // Field is A. +// m.Load(1, &x.B) // Field is B. +// } +func (s Sink) Save(slot int, objPtr any) { + s.internal.save(slot, reflect.ValueOf(objPtr).Elem()) +} + +// SaveValue adds the given object value to the map. +// +// This should be used for values where pointers are not available, or casts +// are required during Save/Load. +// +// For example, if we want to cast external package type P.Foo to int64: +// +// func (x *X) StateSave(m Sink) { +// m.SaveValue(0, "A", int64(x.A)) +// } +// +// func (x *X) StateLoad(m Source) { +// m.LoadValue(0, new(int64), func(x any) { +// x.A = P.Foo(x.(int64)) +// }) +// } +func (s Sink) SaveValue(slot int, obj any) { + s.internal.save(slot, reflect.ValueOf(obj)) +} + +// Context returns the context object provided at save time. +func (s Sink) Context() context.Context { + return s.internal.es.ctx +} + +// Type is an interface that must be implemented by Struct objects. This allows +// these objects to be serialized while minimizing runtime reflection required. +// +// All these methods can be automatically generated by the go_statify tool. +type Type interface { + // StateTypeName returns the type's name. + // + // This is used for matching type information during encoding and + // decoding, as well as dynamic interface dispatch. This should be + // globally unique. + StateTypeName() string + + // StateFields returns information about the type. + // + // Fields is the set of fields for the object. Calls to Sink.Save and + // Source.Load must be made in-order with respect to these fields. + // + // This will be called at most once per serialization. + StateFields() []string +} + +// SaverLoader must be implemented by struct types. +type SaverLoader interface { + // StateSave saves the state of the object to the given Map. + StateSave(Sink) + + // StateLoad loads the state of the object. + StateLoad(context.Context, Source) +} + +// Source is used for Type.StateLoad. +type Source struct { + internal objectDecoder +} + +// Load loads the given object passed as a pointer.. +// +// See Sink.Save for an example. +func (s Source) Load(slot int, objPtr any) { + s.internal.load(slot, reflect.ValueOf(objPtr), false, nil) +} + +// LoadWait loads the given objects from the map, and marks it as requiring all +// AfterLoad executions to complete prior to running this object's AfterLoad. +// +// See Sink.Save for an example. +func (s Source) LoadWait(slot int, objPtr any) { + s.internal.load(slot, reflect.ValueOf(objPtr), true, nil) +} + +// LoadValue loads the given object value from the map. +// +// See Sink.SaveValue for an example. +func (s Source) LoadValue(slot int, objPtr any, fn func(any)) { + o := reflect.ValueOf(objPtr) + s.internal.load(slot, o, true, func() { fn(o.Elem().Interface()) }) +} + +// AfterLoad schedules a function execution when all objects have been +// allocated and their automated loading and customized load logic have been +// executed. fn will not be executed until all of current object's +// dependencies' AfterLoad() logic, if exist, have been executed. +func (s Source) AfterLoad(fn func()) { + s.internal.afterLoad(fn) +} + +// Context returns the context object provided at load time. +func (s Source) Context() context.Context { + return s.internal.ds.ctx +} + +// IsZeroValue checks if the given value is the zero value. +// +// This function is used by the stateify tool. +func IsZeroValue(val any) bool { + return val == nil || reflect.ValueOf(val).Elem().IsZero() +} + +// Failf is a wrapper around panic that should be used to generate errors that +// can be caught during saving and loading. +func Failf(fmtStr string, v ...any) { + panic(fmt.Errorf(fmtStr, v...)) +} + +// safely executes the given function, catching a panic and unpacking as an +// error. +// +// The error flow through the state package uses panic and recover. There are +// two important reasons for this: +// +// 1) Many of the reflection methods will already panic with invalid data or +// violated assumptions. We would want to recover anyways here. +// +// 2) It allows us to eliminate boilerplate within Save() and Load() functions. +// In nearly all cases, when the low-level serialization functions fail, you +// will want the checkpoint to fail anyways. Plumbing errors through every +// method doesn't add a lot of value. If there are specific error conditions +// that you'd like to handle, you should add appropriate functionality to +// objects themselves prior to calling Save() and Load(). +func safely(fn func()) (err error) { + defer func() { + if r := recover(); r != nil { + if es, ok := r.(*ErrState); ok { + err = es // Propagate. + return + } + + // Build a new state error. + es := new(ErrState) + if e, ok := r.(error); ok { + es.err = e + } else { + es.err = fmt.Errorf("%v", r) + } + + // Make a stack. We don't know how big it will be ahead + // of time, but want to make sure we get the whole + // thing. So we just do a stupid brute force approach. + var stack []byte + for sz := 1024; ; sz *= 2 { + stack = make([]byte, sz) + n := runtime.Stack(stack, false) + if n < sz { + es.trace = string(stack[:n]) + break + } + } + + // Set the error. + err = es + } + }() + + // Execute the function. + fn() + return nil +} diff --git a/pkg/state/state_norace.go b/pkg/state/state_norace.go new file mode 100644 index 0000000..be09d61 --- /dev/null +++ b/pkg/state/state_norace.go @@ -0,0 +1,20 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !race +// +build !race + +package state + +var raceEnabled = false diff --git a/pkg/state/state_race.go b/pkg/state/state_race.go new file mode 100644 index 0000000..c9f4fd5 --- /dev/null +++ b/pkg/state/state_race.go @@ -0,0 +1,20 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build race +// +build race + +package state + +var raceEnabled = true diff --git a/pkg/state/statefile/statefile.go b/pkg/state/statefile/statefile.go new file mode 100644 index 0000000..eff4966 --- /dev/null +++ b/pkg/state/statefile/statefile.go @@ -0,0 +1,338 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package statefile defines the state file data stream. +// +// This package currently does not include any details regarding the state +// encoding itself, only details regarding state metadata and data layout. +// +// The file format is defined as follows. +// +// /------------------------------------------------------\ +// | header (8-bytes) | +// +------------------------------------------------------+ +// | metadata length (8-bytes) | +// +------------------------------------------------------+ +// | metadata | +// +------------------------------------------------------+ +// | data | +// \------------------------------------------------------/ +// +// First, it includes a 8-byte magic header which is the following +// sequence of bytes [0x67, 0x56, 0x69, 0x73, 0x6f, 0x72, 0x53, 0x46] +// +// This header is followed by an 8-byte length N (big endian), and an +// ASCII-encoded JSON map that is exactly N bytes long. +// +// This map includes only strings for keys and strings for values. Keys in the +// map that begin with "_" are for internal use only. They may be read, but may +// not be provided by the user. In the future, this metadata may contain some +// information relating to the state encoding itself. +// +// After the map, the remainder of the file is the state data. +package statefile + +import ( + "bytes" + "compress/flate" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + "io" + "strings" + "time" + + "github.com/sagernet/gvisor/pkg/compressio" +) + +// keySize is the AES-256 key length. +const keySize = 32 + +// stateFileChunkSize is the chunk size used to read/write the state file. +const stateFileChunkSize = 1024 * 1024 + +// maxMetadataSize is the size limit of metadata section. +const maxMetadataSize = 16 * 1024 * 1024 + +// magicHeader is the byte sequence beginning each file. +var magicHeader = []byte("\x67\x56\x69\x73\x6f\x72\x53\x46") + +// ErrBadMagic is returned if the header does not match. +var ErrBadMagic = fmt.Errorf("bad magic header") + +// ErrMetadataMissing is returned if the state file is missing mandatory metadata. +var ErrMetadataMissing = fmt.Errorf("missing metadata") + +// ErrInvalidMetadataLength is returned if the metadata length is too large. +var ErrInvalidMetadataLength = fmt.Errorf("metadata length invalid, maximum size is %d", maxMetadataSize) + +// ErrMetadataInvalid is returned if passed metadata is invalid. +var ErrMetadataInvalid = fmt.Errorf("metadata invalid, can't start with _") + +// ErrInvalidFlags is returned if passed flags set is invalid. +var ErrInvalidFlags = fmt.Errorf("flags set is invalid") + +const ( + // CompressionKey is the key for the compression level in the metadata. + CompressionKey = "compression" +) + +// CompressionLevel is the image compression level. +type CompressionLevel string + +const ( + // CompressionLevelFlateBestSpeed represents flate algorithm in best-speed mode. + CompressionLevelFlateBestSpeed = CompressionLevel("flate-best-speed") + // CompressionLevelNone represents the absence of any compression on an image. + CompressionLevelNone = CompressionLevel("none") + // CompressionLevelDefault represents the default compression level. + CompressionLevelDefault = CompressionLevelFlateBestSpeed +) + +func (c CompressionLevel) String() string { + return string(c) +} + +// ToMetadata returns the compression level as a metadata map. +func (c CompressionLevel) ToMetadata() map[string]string { + return map[string]string{CompressionKey: string(c)} +} + +// CompressionLevelFromString parses a string into the CompressionLevel. +func CompressionLevelFromString(val string) (CompressionLevel, error) { + switch val { + case string(CompressionLevelFlateBestSpeed): + return CompressionLevelFlateBestSpeed, nil + case string(CompressionLevelNone): + return CompressionLevelNone, nil + case "": + return CompressionLevelDefault, nil + default: + return CompressionLevelNone, ErrInvalidFlags + } +} + +// CompressionLevelFromMetadata returns image compression type stored in the metadata. +// If the metadata doesn't contain compression information the default behavior +// is the "flate-best-speed" state because the default behavior used to be to always +// compress. If the parameter is missing it will be set to default. +func CompressionLevelFromMetadata(metadata map[string]string) (CompressionLevel, error) { + compression := CompressionLevelDefault + + if val, ok := metadata[CompressionKey]; ok { + var err error + if compression, err = CompressionLevelFromString(val); err != nil { + return CompressionLevelNone, err + } + } else { + metadata[CompressionKey] = string(compression) + } + + return compression, nil +} + +func writeMetadataLen(w io.Writer, val uint64) error { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], val) + _, err := w.Write(buf[:]) + return err +} + +// NewWriter returns a state data writer for a statefile. +// +// Note that the returned WriteCloser must be closed. +func NewWriter(w io.Writer, key []byte, metadata map[string]string) (io.WriteCloser, error) { + if metadata == nil { + metadata = make(map[string]string) + } + for k := range metadata { + if strings.HasPrefix(k, "_") { + return nil, ErrMetadataInvalid + } + } + + // Create our HMAC function. + h := hmac.New(sha256.New, key) + mw := io.MultiWriter(w, h) + + // First, write the header. + if _, err := mw.Write(magicHeader); err != nil { + return nil, err + } + + // Generate a timestamp, for convenience only. + metadata["_timestamp"] = time.Now().UTC().String() + defer delete(metadata, "_timestamp") + + // Save compression state + compression, err := CompressionLevelFromMetadata(metadata) + if err != nil { + return nil, err + } + + // Write the metadata. + b, err := json.Marshal(metadata) + if err != nil { + return nil, err + } + + if len(b) > maxMetadataSize { + return nil, ErrInvalidMetadataLength + } + + // Metadata length. + if err := writeMetadataLen(mw, uint64(len(b))); err != nil { + return nil, err + } + // Metadata bytes; io.MultiWriter will return a short write error if + // any of the writers returns < n. + if _, err := mw.Write(b); err != nil { + return nil, err + } + // Write the current hash. + cur := h.Sum(nil) + for done := 0; done < len(cur); { + n, err := mw.Write(cur[done:]) + done += n + if err != nil { + return nil, err + } + } + + // Wrap in compression. When using "best compression" mode, there is usually + // only a little gain in file size reduction, which translate to even smaller + // gain in restore latency reduction, while incurring much more CPU usage at + // save time. + if compression == CompressionLevelFlateBestSpeed { + return compressio.NewWriter(w, key, stateFileChunkSize, flate.BestSpeed) + } + + return compressio.NewSimpleWriter(w, key, stateFileChunkSize), nil +} + +// MetadataUnsafe reads out the metadata from a state file without verifying any +// HMAC. This function shouldn't be called for untrusted input files. +func MetadataUnsafe(r io.Reader) (map[string]string, error) { + return metadata(r, nil) +} + +func readMetadataLen(r io.Reader) (uint64, error) { + var buf [8]byte + if _, err := io.ReadFull(r, buf[:]); err != nil { + return 0, err + } + return binary.BigEndian.Uint64(buf[:]), nil +} + +// metadata validates the magic header and reads out the metadata from a state +// data stream. +func metadata(r io.Reader, h hash.Hash) (map[string]string, error) { + if h != nil { + r = io.TeeReader(r, h) + } + + // Read and validate magic header. + b := make([]byte, len(magicHeader)) + if _, err := r.Read(b); err != nil { + return nil, err + } + if !bytes.Equal(b, magicHeader) { + return nil, ErrBadMagic + } + + // Read and validate metadata. + b, err := func() (b []byte, err error) { + defer func() { + if r := recover(); r != nil { + b = nil + err = fmt.Errorf("%v", r) + } + }() + + metadataLen, err := readMetadataLen(r) + if err != nil { + return nil, err + } + if metadataLen > maxMetadataSize { + return nil, ErrInvalidMetadataLength + } + b = make([]byte, int(metadataLen)) + if _, err := io.ReadFull(r, b); err != nil { + return nil, err + } + return b, nil + }() + if err != nil { + return nil, err + } + + if h != nil { + // Check the hash prior to decoding. + cur := h.Sum(nil) + buf := make([]byte, len(cur)) + if _, err := io.ReadFull(r, buf); err != nil { + return nil, err + } + if !hmac.Equal(cur, buf) { + return nil, compressio.ErrHashMismatch + } + } + + // Decode the metadata. + metadata := make(map[string]string) + if err := json.Unmarshal(b, &metadata); err != nil { + return nil, err + } + + return metadata, nil +} + +// NewReader returns a reader for a statefile. +func NewReader(r io.ReadCloser, key []byte) (io.ReadCloser, map[string]string, error) { + // Read the metadata with the hash. + h := hmac.New(sha256.New, key) + metadata, err := metadata(r, h) + if err != nil { + return nil, nil, err + } + + // Determine image compression state. If the metadata doesn't contain + // compression information the default behavior is the "compressed" state + // because the default behavior used to be to always compress. + compression, err := CompressionLevelFromMetadata(metadata) + if err != nil { + return nil, nil, err + } + + // Pick correct reader + var cr io.ReadCloser + + if compression == CompressionLevelFlateBestSpeed { + cr, err = compressio.NewReader(r, key) + } else if compression == CompressionLevelNone { + cr = compressio.NewSimpleReader(r, key) + } else { + // Should never occur, as it has the default path. + return nil, nil, fmt.Errorf("metadata contains invalid compression flag value: %v", compression) + } + + if err != nil { + return nil, nil, err + } + + return cr, metadata, nil +} diff --git a/pkg/state/statefile/statefile_state_autogen.go b/pkg/state/statefile/statefile_state_autogen.go new file mode 100644 index 0000000..a2cdaa3 --- /dev/null +++ b/pkg/state/statefile/statefile_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package statefile diff --git a/pkg/state/stats.go b/pkg/state/stats.go new file mode 100644 index 0000000..eaec664 --- /dev/null +++ b/pkg/state/stats.go @@ -0,0 +1,145 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "bytes" + "fmt" + "sort" + "time" +) + +type statEntry struct { + count uint + total time.Duration +} + +// Stats tracks encode / decode timing. +// +// This currently provides a meaningful String function and no other way to +// extract stats about individual types. +// +// All exported receivers accept nil. +type Stats struct { + // byType contains a breakdown of time spent by type. + // + // This is indexed *directly* by typeID, including zero. + byType []statEntry + + // stack contains objects in progress. + stack []typeID + + // names contains type names. + // + // This is also indexed *directly* by typeID, including zero, which we + // hard-code as "state.default". This is only resolved by calling fini + // on the stats object. + names []string + + // last is the last start time. + last time.Time +} + +// init initializes statistics. +func (s *Stats) init() { + s.last = time.Now() + s.stack = append(s.stack, 0) +} + +// fini finalizes statistics. +func (s *Stats) fini(resolve func(id typeID) string) { + s.done() + + // Resolve all type names. + s.names = make([]string, len(s.byType)) + s.names[0] = "state.default" // See above. + for id := typeID(1); int(id) < len(s.names); id++ { + s.names[id] = resolve(id) + } +} + +// sample adds the samples to the given object. +func (s *Stats) sample(id typeID) { + now := time.Now() + if len(s.byType) <= int(id) { + // Allocate all the missing entries in one fell swoop. + s.byType = append(s.byType, make([]statEntry, 1+int(id)-len(s.byType))...) + } + s.byType[id].total += now.Sub(s.last) + s.last = now +} + +// start starts a sample. +func (s *Stats) start(id typeID) { + last := s.stack[len(s.stack)-1] + s.sample(last) + s.stack = append(s.stack, id) +} + +// done finishes the current sample. +func (s *Stats) done() { + last := s.stack[len(s.stack)-1] + s.sample(last) + s.byType[last].count++ + s.stack = s.stack[:len(s.stack)-1] +} + +type sliceEntry struct { + name string + entry *statEntry +} + +// String returns a table representation of the stats. +func (s *Stats) String() string { + // Build a list of stat entries. + ss := make([]sliceEntry, 0, len(s.byType)) + for id := 0; id < len(s.names); id++ { + ss = append(ss, sliceEntry{ + name: s.names[id], + entry: &s.byType[id], + }) + } + + // Sort by total time (descending). + sort.Slice(ss, func(i, j int) bool { + return ss[i].entry.total > ss[j].entry.total + }) + + // Print the stat results. + var ( + buf bytes.Buffer + count uint + total time.Duration + ) + buf.WriteString("\n") + buf.WriteString(fmt.Sprintf("% 16s | % 8s | % 16s | %s\n", "total", "count", "per", "type")) + buf.WriteString("-----------------+----------+------------------+----------------\n") + for _, se := range ss { + if se.entry.count == 0 { + // Since we store all types linearly, we are not + // guaranteed that any entry actually has time. + continue + } + count += se.entry.count + total += se.entry.total + per := se.entry.total / time.Duration(se.entry.count) + buf.WriteString(fmt.Sprintf("% 16s | %8d | % 16s | %s\n", + se.entry.total, se.entry.count, per, se.name)) + } + buf.WriteString("-----------------+----------+------------------+----------------\n") + buf.WriteString(fmt.Sprintf("% 16s | % 8d | % 16s | [all]", + total, count, total/time.Duration(count))) + return string(buf.Bytes()) +} diff --git a/pkg/state/types.go b/pkg/state/types.go new file mode 100644 index 0000000..e6d2e57 --- /dev/null +++ b/pkg/state/types.go @@ -0,0 +1,384 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "reflect" + "sort" + + "github.com/sagernet/gvisor/pkg/state/wire" +) + +// assertValidType asserts that the type is valid. +func assertValidType(name string, fields []string) { + if name == "" { + Failf("type has empty name") + } + fieldsCopy := make([]string, len(fields)) + for i := 0; i < len(fields); i++ { + if fields[i] == "" { + Failf("field has empty name for type %q", name) + } + fieldsCopy[i] = fields[i] + } + sort.Slice(fieldsCopy, func(i, j int) bool { + return fieldsCopy[i] < fieldsCopy[j] + }) + for i := range fieldsCopy { + if i > 0 && fieldsCopy[i-1] == fieldsCopy[i] { + Failf("duplicate field %q for type %s", fieldsCopy[i], name) + } + } +} + +// typeEntry is an entry in the typeDatabase. +type typeEntry struct { + ID typeID + wire.Type +} + +// reconciledTypeEntry is a reconciled entry in the typeDatabase. +type reconciledTypeEntry struct { + wire.Type + LocalType reflect.Type + FieldOrder []int +} + +// typeEncodeDatabase is an internal TypeInfo database for encoding. +type typeEncodeDatabase struct { + // byType maps by type to the typeEntry. + byType map[reflect.Type]*typeEntry + + // lastID is the last used ID. + lastID typeID +} + +// makeTypeEncodeDatabase makes a typeDatabase. +func makeTypeEncodeDatabase() typeEncodeDatabase { + return typeEncodeDatabase{ + byType: make(map[reflect.Type]*typeEntry), + } +} + +// typeDecodeDatabase is an internal TypeInfo database for decoding. +type typeDecodeDatabase struct { + // byID maps by ID to type. + byID []*reconciledTypeEntry + + // pending are entries that are pending validation by Lookup. These + // will be reconciled with actual objects. Note that these will also be + // used to lookup types by name, since they may not be reconciled and + // there's little value to deleting from this map. + pending []*wire.Type +} + +// makeTypeDecodeDatabase makes a typeDatabase. +func makeTypeDecodeDatabase() typeDecodeDatabase { + return typeDecodeDatabase{} +} + +// lookupNameFields extracts the name and fields from an object. +func lookupNameFields(typ reflect.Type) (string, []string, bool) { + v := reflect.Zero(reflect.PtrTo(typ)).Interface() + t, ok := v.(Type) + if !ok { + // Is this a primitive? + if typ.Kind() == reflect.Interface { + return interfaceType, nil, true + } + name := typ.Name() + if _, ok := primitiveTypeDatabase[name]; !ok { + // This is not a known type, and not a primitive. The + // encoder may proceed for anonymous empty structs, or + // it may deference the type pointer and try again. + return "", nil, false + } + return name, nil, true + } + // Sanity check the type. + if raceEnabled { + if _, ok := reverseTypeDatabase[typ]; !ok { + // The type was not registered? Must be an embedded + // structure or something else. + return "", nil, false + } + } + // Extract the name from the object. + name := t.StateTypeName() + fields := t.StateFields() + assertValidType(name, fields) + return name, fields, true +} + +// Lookup looks up or registers the given object. +// +// The bool indicates whether this is an existing entry: false means the entry +// did not exist, and true means the entry did exist. If this bool is false and +// the returned typeEntry are nil, then the obj did not implement the Type +// interface. +func (tdb *typeEncodeDatabase) Lookup(typ reflect.Type) (*typeEntry, bool) { + te, ok := tdb.byType[typ] + if !ok { + // Lookup the type information. + name, fields, ok := lookupNameFields(typ) + if !ok { + // Empty structs may still be encoded, so let the + // caller decide what to do from here. + return nil, false + } + + // Register the new type. + tdb.lastID++ + te = &typeEntry{ + ID: tdb.lastID, + Type: wire.Type{ + Name: name, + Fields: fields, + }, + } + + // All done. + tdb.byType[typ] = te + return te, false + } + return te, true +} + +// Register adds a typeID entry. +func (tbd *typeDecodeDatabase) Register(typ *wire.Type) { + assertValidType(typ.Name, typ.Fields) + tbd.pending = append(tbd.pending, typ) +} + +// LookupName looks up the type name by ID. +func (tbd *typeDecodeDatabase) LookupName(id typeID) string { + if len(tbd.pending) < int(id) { + // This is likely an encoder error? + Failf("type ID %d not available", id) + } + return tbd.pending[id-1].Name +} + +// LookupType looks up the type by ID. +func (tbd *typeDecodeDatabase) LookupType(id typeID) reflect.Type { + name := tbd.LookupName(id) + typ, ok := globalTypeDatabase[name] + if !ok { + // If not available, see if it's primitive. + typ, ok = primitiveTypeDatabase[name] + if !ok && name == interfaceType { + // Matches the built-in interface type. + var i any + return reflect.TypeOf(&i).Elem() + } + if !ok { + // The type is perhaps not registered? + Failf("type name %q is not available", name) + } + return typ // Primitive type. + } + return typ // Registered type. +} + +// singleFieldOrder defines the field order for a single field. +var singleFieldOrder = []int{0} + +// Lookup looks up or registers the given object. +// +// First, the typeID is searched to see if this has already been appropriately +// reconciled. If no, then a reconciliation will take place that may result in a +// field ordering. If a nil reconciledTypeEntry is returned from this method, +// then the object does not support the Type interface. +// +// This method never returns nil. +func (tbd *typeDecodeDatabase) Lookup(id typeID, typ reflect.Type) *reconciledTypeEntry { + if len(tbd.byID) >= int(id) && tbd.byID[id-1] != nil { + // Already reconciled. + return tbd.byID[id-1] + } + // The ID has not been reconciled yet. That's fine. We need to make + // sure it aligns with the current provided object. + if len(tbd.pending) < int(id) { + // This id was never registered. Probably an encoder error? + Failf("typeDatabase does not contain id %d", id) + } + // Extract the pending info. + pending := tbd.pending[id-1] + // Grow the byID list. + if len(tbd.byID) < int(id) { + tbd.byID = append(tbd.byID, make([]*reconciledTypeEntry, int(id)-len(tbd.byID))...) + } + // Reconcile the type. + name, fields, ok := lookupNameFields(typ) + if !ok { + // Empty structs are decoded only when the type is nil. Since + // this isn't the case, we fail here. + Failf("unsupported type %q during decode; can't reconcile", pending.Name) + } + if name != pending.Name { + // Are these the same type? Print a helpful message as this may + // actually happen in practice if types change. + Failf("typeDatabase contains conflicting definitions for id %d: %s->%v (current) and %s->%v (existing)", + id, name, fields, pending.Name, pending.Fields) + } + rte := &reconciledTypeEntry{ + Type: wire.Type{ + Name: name, + Fields: fields, + }, + LocalType: typ, + } + // If there are zero or one fields, then we skip allocating the field + // slice. There is special handling for decoding in this case. If the + // field name does not match, it will be caught in the general purpose + // code below. + if len(fields) != len(pending.Fields) { + Failf("type %q contains different fields: %v (decode) and %v (encode)", + name, fields, pending.Fields) + } + if len(fields) == 0 { + tbd.byID[id-1] = rte // Save. + return rte + } + if len(fields) == 1 && fields[0] == pending.Fields[0] { + tbd.byID[id-1] = rte // Save. + rte.FieldOrder = singleFieldOrder + return rte + } + // For each field in the current object's information, match it to a + // field in the destination object. We know from the assertion above + // and the insertion on insertion to pending that neither field + // contains any duplicates. + fieldOrder := make([]int, len(fields)) + for i, name := range fields { + fieldOrder[i] = -1 // Sentinel. + // Is it an exact match? + if pending.Fields[i] == name { + fieldOrder[i] = i + continue + } + // Find the matching field. + for j, otherName := range pending.Fields { + if name == otherName { + fieldOrder[i] = j + break + } + } + if fieldOrder[i] == -1 { + // The type name matches but we are lacking some common fields. + Failf("type %q has mismatched fields: %v (decode) and %v (encode)", + name, fields, pending.Fields) + } + } + // The type has been reeconciled. + rte.FieldOrder = fieldOrder + tbd.byID[id-1] = rte + return rte +} + +// interfaceType defines all interfaces. +const interfaceType = "interface" + +// primitiveTypeDatabase is a set of fixed types. +var primitiveTypeDatabase = func() map[string]reflect.Type { + r := make(map[string]reflect.Type) + for _, t := range []reflect.Type{ + reflect.TypeOf(false), + reflect.TypeOf(int(0)), + reflect.TypeOf(int8(0)), + reflect.TypeOf(int16(0)), + reflect.TypeOf(int32(0)), + reflect.TypeOf(int64(0)), + reflect.TypeOf(uint(0)), + reflect.TypeOf(uintptr(0)), + reflect.TypeOf(uint8(0)), + reflect.TypeOf(uint16(0)), + reflect.TypeOf(uint32(0)), + reflect.TypeOf(uint64(0)), + reflect.TypeOf(""), + reflect.TypeOf(float32(0.0)), + reflect.TypeOf(float64(0.0)), + reflect.TypeOf(complex64(0.0)), + reflect.TypeOf(complex128(0.0)), + } { + r[t.Name()] = t + } + return r +}() + +// globalTypeDatabase is used for dispatching interfaces on decode. +var globalTypeDatabase = map[string]reflect.Type{} + +// reverseTypeDatabase is a reverse mapping. +var reverseTypeDatabase = map[reflect.Type]string{} + +// Release releases references to global type databases. +// Must only be called in contexts where they will definitely never be used, +// in order to save memory. +func Release() { + globalTypeDatabase = nil + reverseTypeDatabase = nil +} + +// Register registers a type. +// +// This must be called on init and only done once. +func Register(t Type) { + name := t.StateTypeName() + typ := reflect.TypeOf(t) + if raceEnabled { + assertValidType(name, t.StateFields()) + // Register must always be called on pointers. + if typ.Kind() != reflect.Ptr { + Failf("Register must be called on pointers") + } + } + typ = typ.Elem() + if raceEnabled { + if typ.Kind() == reflect.Struct { + // All registered structs must implement SaverLoader. We allow + // the registration is non-struct types with just the Type + // interface, but we need to call StateSave/StateLoad methods + // on aggregate types. + if _, ok := t.(SaverLoader); !ok { + Failf("struct %T does not implement SaverLoader", t) + } + } else { + // Non-structs must not have any fields. We don't support + // calling StateSave/StateLoad methods on any non-struct types. + // If custom behavior is required, these types should be + // wrapped in a structure of some kind. + if fields := t.StateFields(); len(fields) != 0 { + Failf("non-struct %T has non-zero fields %v", t, fields) + } + // We don't allow non-structs to implement StateSave/StateLoad + // methods, because they won't be called and it's confusing. + if _, ok := t.(SaverLoader); ok { + Failf("non-struct %T implements SaverLoader", t) + } + } + if _, ok := primitiveTypeDatabase[name]; ok { + Failf("conflicting primitiveTypeDatabase entry for %T: used by primitive", t) + } + if _, ok := globalTypeDatabase[name]; ok { + Failf("conflicting globalTypeDatabase entries for %T: name conflict", t) + } + if name == interfaceType { + Failf("conflicting name for %T: matches interfaceType", t) + } + reverseTypeDatabase[typ] = name + } + globalTypeDatabase[name] = typ +} diff --git a/pkg/state/wire/wire.go b/pkg/state/wire/wire.go new file mode 100644 index 0000000..07a41ac --- /dev/null +++ b/pkg/state/wire/wire.go @@ -0,0 +1,983 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package wire contains a few basic types that can be composed to serialize +// graph information for the state package. This package defines the wire +// protocol. +// +// Note that these types are careful about how they implement the relevant +// interfaces (either value receiver or pointer receiver), so that native-sized +// types, such as integers and simple pointers, can fit inside the interface +// object. +// +// This package also uses panic as control flow, so called should be careful to +// wrap calls in appropriate handlers. +// +// Testing for this package is driven by the state test package. +package wire + +import ( + "fmt" + "io" + "math" + + "github.com/sagernet/gvisor/pkg/gohacks" +) + +// Reader bundles an io.Reader with a buffer used to implement readByte +// efficiently. +type Reader struct { + io.Reader + + buf [1]byte +} + +// readByte reads a single byte from r.Reader without allocation. It panics on +// error. +func (r *Reader) readByte() byte { + n, err := r.Read(r.buf[:]) + if n != 1 { + panic(err) + } + return r.buf[0] +} + +// Writer bundles an io.Writer with a buffer used to implement writeByte +// efficiently. +type Writer struct { + io.Writer + + // buf is used by Uint as a scratch buffer. + buf [10]byte +} + +// readFull is a utility. The equivalent is not needed for Write, but the API +// contract dictates that it must always complete all bytes given or return an +// error. +func readFull(r *Reader, p []byte) { + for done := 0; done < len(p); { + n, err := r.Read(p[done:]) + done += n + if n == 0 && err != nil { + panic(err) + } + } +} + +// Object is a generic object. +type Object interface { + // save saves the given object. + // + // Panic is used for error control flow. + save(*Writer) + + // load loads a new object of the given type. + // + // Panic is used for error control flow. + load(*Reader) Object +} + +// Bool is a boolean. +type Bool bool + +// loadBool loads an object of type Bool. +func loadBool(r *Reader) Bool { + b := loadUint(r) + return Bool(b == 1) +} + +// save implements Object.save. +func (b Bool) save(w *Writer) { + var v Uint + if b { + v = 1 + } else { + v = 0 + } + v.save(w) +} + +// load implements Object.load. +func (Bool) load(r *Reader) Object { return loadBool(r) } + +// Int is a signed integer. +// +// This uses varint encoding. +type Int int64 + +// loadInt loads an object of type Int. +func loadInt(r *Reader) Int { + u := loadUint(r) + x := Int(u >> 1) + if u&1 != 0 { + x = ^x + } + return x +} + +// save implements Object.save. +func (i Int) save(w *Writer) { + u := Uint(i) << 1 + if i < 0 { + u = ^u + } + u.save(w) +} + +// load implements Object.load. +func (Int) load(r *Reader) Object { return loadInt(r) } + +// Uint is an unsigned integer. +type Uint uint64 + +// loadUint loads an object of type Uint. +func loadUint(r *Reader) Uint { + var ( + u Uint + s uint + ) + for i := 0; i <= 9; i++ { + b := r.readByte() + if b < 0x80 { + if i == 9 && b > 1 { + panic("overflow") + } + u |= Uint(b) << s + return u + } + u |= Uint(b&0x7f) << s + s += 7 + } + panic("unreachable") +} + +// save implements Object.save. +func (u Uint) save(w *Writer) { + i := 0 + for u >= 0x80 { + w.buf[i] = byte(u) | 0x80 + i++ + u >>= 7 + } + w.buf[i] = byte(u) + if _, err := w.Write(w.buf[:i+1]); err != nil { + panic(err) + } +} + +// load implements Object.load. +func (Uint) load(r *Reader) Object { return loadUint(r) } + +// Float32 is a 32-bit floating point number. +type Float32 float32 + +// loadFloat32 loads an object of type Float32. +func loadFloat32(r *Reader) Float32 { + n := loadUint(r) + return Float32(math.Float32frombits(uint32(n))) +} + +// save implements Object.save. +func (f Float32) save(w *Writer) { + n := Uint(math.Float32bits(float32(f))) + n.save(w) +} + +// load implements Object.load. +func (Float32) load(r *Reader) Object { return loadFloat32(r) } + +// Float64 is a 64-bit floating point number. +type Float64 float64 + +// loadFloat64 loads an object of type Float64. +func loadFloat64(r *Reader) Float64 { + n := loadUint(r) + return Float64(math.Float64frombits(uint64(n))) +} + +// save implements Object.save. +func (f Float64) save(w *Writer) { + n := Uint(math.Float64bits(float64(f))) + n.save(w) +} + +// load implements Object.load. +func (Float64) load(r *Reader) Object { return loadFloat64(r) } + +// Complex64 is a 64-bit complex number. +type Complex64 complex128 + +// loadComplex64 loads an object of type Complex64. +func loadComplex64(r *Reader) Complex64 { + re := loadFloat32(r) + im := loadFloat32(r) + return Complex64(complex(float32(re), float32(im))) +} + +// save implements Object.save. +func (c *Complex64) save(w *Writer) { + re := Float32(real(*c)) + im := Float32(imag(*c)) + re.save(w) + im.save(w) +} + +// load implements Object.load. +func (*Complex64) load(r *Reader) Object { + c := loadComplex64(r) + return &c +} + +// Complex128 is a 128-bit complex number. +type Complex128 complex128 + +// loadComplex128 loads an object of type Complex128. +func loadComplex128(r *Reader) Complex128 { + re := loadFloat64(r) + im := loadFloat64(r) + return Complex128(complex(float64(re), float64(im))) +} + +// save implements Object.save. +func (c *Complex128) save(w *Writer) { + re := Float64(real(*c)) + im := Float64(imag(*c)) + re.save(w) + im.save(w) +} + +// load implements Object.load. +func (*Complex128) load(r *Reader) Object { + c := loadComplex128(r) + return &c +} + +// String is a string. +type String string + +// loadString loads an object of type String. +func loadString(r *Reader) String { + l := loadUint(r) + p := make([]byte, l) + readFull(r, p) + return String(gohacks.StringFromImmutableBytes(p)) +} + +// save implements Object.save. +func (s *String) save(w *Writer) { + l := Uint(len(*s)) + l.save(w) + p := gohacks.ImmutableBytesFromString(string(*s)) + _, err := w.Write(p) // Must write all bytes. + if err != nil { + panic(err) + } +} + +// load implements Object.load. +func (*String) load(r *Reader) Object { + s := loadString(r) + return &s +} + +// Dot is a kind of reference: one of Index and FieldName. +type Dot interface { + isDot() +} + +// Index is a reference resolution. +type Index uint32 + +func (Index) isDot() {} + +// FieldName is a reference resolution. +type FieldName string + +func (*FieldName) isDot() {} + +// Ref is a reference to an object. +type Ref struct { + // Root is the root object. + Root Uint + + // Dots is the set of traversals required from the Root object above. + // Note that this will be stored in reverse order for efficiency. + Dots []Dot + + // Type is the base type for the root object. This is non-nil iff Dots + // is non-zero length (that is, this is a complex reference). This is + // not *strictly* necessary, but can be used to simplify decoding. + Type TypeSpec +} + +// loadRef loads an object of type Ref (abstract). +func loadRef(r *Reader) Ref { + ref := Ref{ + Root: loadUint(r), + } + l := loadUint(r) + ref.Dots = make([]Dot, l) + for i := 0; i < int(l); i++ { + // Disambiguate between an Index (non-negative) and a field + // name (negative). This does some space and avoids a dedicate + // loadDot function. See Ref.save for the other side. + d := loadInt(r) + if d >= 0 { + ref.Dots[i] = Index(d) + continue + } + p := make([]byte, -d) + readFull(r, p) + fieldName := FieldName(gohacks.StringFromImmutableBytes(p)) + ref.Dots[i] = &fieldName + } + if l != 0 { + // Only if dots is non-zero. + ref.Type = loadTypeSpec(r) + } + return ref +} + +// save implements Object.save. +func (r *Ref) save(w *Writer) { + r.Root.save(w) + l := Uint(len(r.Dots)) + l.save(w) + for _, d := range r.Dots { + // See LoadRef. We use non-negative numbers to encode Index + // objects and negative numbers to encode field lengths. + switch x := d.(type) { + case Index: + i := Int(x) + i.save(w) + case *FieldName: + d := Int(-len(*x)) + d.save(w) + p := gohacks.ImmutableBytesFromString(string(*x)) + if _, err := w.Write(p); err != nil { + panic(err) + } + default: + panic("unknown dot implementation") + } + } + if l != 0 { + // See above. + saveTypeSpec(w, r.Type) + } +} + +// load implements Object.load. +func (*Ref) load(r *Reader) Object { + ref := loadRef(r) + return &ref +} + +// Nil is a primitive zero value of any type. +type Nil struct{} + +// loadNil loads an object of type Nil. +func loadNil(r *Reader) Nil { + return Nil{} +} + +// save implements Object.save. +func (Nil) save(w *Writer) {} + +// load implements Object.load. +func (Nil) load(r *Reader) Object { return loadNil(r) } + +// Slice is a slice value. +type Slice struct { + Length Uint + Capacity Uint + Ref Ref +} + +// loadSlice loads an object of type Slice. +func loadSlice(r *Reader) Slice { + return Slice{ + Length: loadUint(r), + Capacity: loadUint(r), + Ref: loadRef(r), + } +} + +// save implements Object.save. +func (s *Slice) save(w *Writer) { + s.Length.save(w) + s.Capacity.save(w) + s.Ref.save(w) +} + +// load implements Object.load. +func (*Slice) load(r *Reader) Object { + s := loadSlice(r) + return &s +} + +// Array is an array value. +type Array struct { + Contents []Object +} + +// loadArray loads an object of type Array. +func loadArray(r *Reader) Array { + l := loadUint(r) + if l == 0 { + // Note that there isn't a single object available to encode + // the type of, so we need this additional branch. + return Array{} + } + // All the objects here have the same type, so use dynamic dispatch + // only once. All other objects will automatically take the same type + // as the first object. + contents := make([]Object, l) + v := Load(r) + contents[0] = v + for i := 1; i < int(l); i++ { + contents[i] = v.load(r) + } + return Array{ + Contents: contents, + } +} + +// save implements Object.save. +func (a *Array) save(w *Writer) { + l := Uint(len(a.Contents)) + l.save(w) + if l == 0 { + // See LoadArray. + return + } + // See above. + Save(w, a.Contents[0]) + for i := 1; i < int(l); i++ { + a.Contents[i].save(w) + } +} + +// load implements Object.load. +func (*Array) load(r *Reader) Object { + a := loadArray(r) + return &a +} + +// Map is a map value. +type Map struct { + Keys []Object + Values []Object +} + +// loadMap loads an object of type Map. +func loadMap(r *Reader) Map { + l := loadUint(r) + if l == 0 { + // See LoadArray. + return Map{} + } + // See type dispatch notes in Array. + keys := make([]Object, l) + values := make([]Object, l) + k := Load(r) + v := Load(r) + keys[0] = k + values[0] = v + for i := 1; i < int(l); i++ { + keys[i] = k.load(r) + values[i] = v.load(r) + } + return Map{ + Keys: keys, + Values: values, + } +} + +// save implements Object.save. +func (m *Map) save(w *Writer) { + l := Uint(len(m.Keys)) + if int(l) != len(m.Values) { + panic(fmt.Sprintf("mismatched keys (%d) Aand values (%d)", len(m.Keys), len(m.Values))) + } + l.save(w) + if l == 0 { + // See LoadArray. + return + } + // See above. + Save(w, m.Keys[0]) + Save(w, m.Values[0]) + for i := 1; i < int(l); i++ { + m.Keys[i].save(w) + m.Values[i].save(w) + } +} + +// load implements Object.load. +func (*Map) load(r *Reader) Object { + m := loadMap(r) + return &m +} + +// TypeSpec is a type dereference. +type TypeSpec interface { + isTypeSpec() +} + +// TypeID is a concrete type ID. +type TypeID Uint + +func (TypeID) isTypeSpec() {} + +// TypeSpecPointer is a pointer type. +type TypeSpecPointer struct { + Type TypeSpec +} + +func (*TypeSpecPointer) isTypeSpec() {} + +// TypeSpecArray is an array type. +type TypeSpecArray struct { + Count Uint + Type TypeSpec +} + +func (*TypeSpecArray) isTypeSpec() {} + +// TypeSpecSlice is a slice type. +type TypeSpecSlice struct { + Type TypeSpec +} + +func (*TypeSpecSlice) isTypeSpec() {} + +// TypeSpecMap is a map type. +type TypeSpecMap struct { + Key TypeSpec + Value TypeSpec +} + +func (*TypeSpecMap) isTypeSpec() {} + +// TypeSpecNil is an empty type. +type TypeSpecNil struct{} + +func (TypeSpecNil) isTypeSpec() {} + +// TypeSpec types. +// +// These use a distinct encoding on the wire, as they are used only in the +// interface object. They are decoded through the dedicated loadTypeSpec and +// saveTypeSpec functions. +const ( + typeSpecTypeID Uint = iota + typeSpecPointer + typeSpecArray + typeSpecSlice + typeSpecMap + typeSpecNil +) + +// loadTypeSpec loads TypeSpec values. +func loadTypeSpec(r *Reader) TypeSpec { + switch hdr := loadUint(r); hdr { + case typeSpecTypeID: + return TypeID(loadUint(r)) + case typeSpecPointer: + return &TypeSpecPointer{ + Type: loadTypeSpec(r), + } + case typeSpecArray: + return &TypeSpecArray{ + Count: loadUint(r), + Type: loadTypeSpec(r), + } + case typeSpecSlice: + return &TypeSpecSlice{ + Type: loadTypeSpec(r), + } + case typeSpecMap: + return &TypeSpecMap{ + Key: loadTypeSpec(r), + Value: loadTypeSpec(r), + } + case typeSpecNil: + return TypeSpecNil{} + default: + // This is not a valid stream? + panic(fmt.Errorf("unknown header: %d", hdr)) + } +} + +// saveTypeSpec saves TypeSpec values. +func saveTypeSpec(w *Writer, t TypeSpec) { + switch x := t.(type) { + case TypeID: + typeSpecTypeID.save(w) + Uint(x).save(w) + case *TypeSpecPointer: + typeSpecPointer.save(w) + saveTypeSpec(w, x.Type) + case *TypeSpecArray: + typeSpecArray.save(w) + x.Count.save(w) + saveTypeSpec(w, x.Type) + case *TypeSpecSlice: + typeSpecSlice.save(w) + saveTypeSpec(w, x.Type) + case *TypeSpecMap: + typeSpecMap.save(w) + saveTypeSpec(w, x.Key) + saveTypeSpec(w, x.Value) + case TypeSpecNil: + typeSpecNil.save(w) + default: + // This should not happen? + panic(fmt.Errorf("unknown type %T", t)) + } +} + +// Interface is an interface value. +type Interface struct { + Type TypeSpec + Value Object +} + +// loadInterface loads an object of type Interface. +func loadInterface(r *Reader) Interface { + return Interface{ + Type: loadTypeSpec(r), + Value: Load(r), + } +} + +// save implements Object.save. +func (i *Interface) save(w *Writer) { + saveTypeSpec(w, i.Type) + Save(w, i.Value) +} + +// load implements Object.load. +func (*Interface) load(r *Reader) Object { + i := loadInterface(r) + return &i +} + +// Type is type information. +type Type struct { + Name string + Fields []string +} + +// loadType loads an object of type Type. +func loadType(r *Reader) Type { + name := string(loadString(r)) + l := loadUint(r) + fields := make([]string, l) + for i := 0; i < int(l); i++ { + fields[i] = string(loadString(r)) + } + return Type{ + Name: name, + Fields: fields, + } +} + +// save implements Object.save. +func (t *Type) save(w *Writer) { + s := String(t.Name) + s.save(w) + l := Uint(len(t.Fields)) + l.save(w) + for i := 0; i < int(l); i++ { + s := String(t.Fields[i]) + s.save(w) + } +} + +// load implements Object.load. +func (*Type) load(r *Reader) Object { + t := loadType(r) + return &t +} + +// multipleObjects is a special type for serializing multiple objects. +type multipleObjects []Object + +// loadMultipleObjects loads a series of objects. +func loadMultipleObjects(r *Reader) multipleObjects { + l := loadUint(r) + m := make(multipleObjects, l) + for i := 0; i < int(l); i++ { + m[i] = Load(r) + } + return m +} + +// save implements Object.save. +func (m *multipleObjects) save(w *Writer) { + l := Uint(len(*m)) + l.save(w) + for i := 0; i < int(l); i++ { + Save(w, (*m)[i]) + } +} + +// load implements Object.load. +func (*multipleObjects) load(r *Reader) Object { + m := loadMultipleObjects(r) + return &m +} + +// noObjects represents no objects. +type noObjects struct{} + +// loadNoObjects loads a sentinel. +func loadNoObjects(r *Reader) noObjects { return noObjects{} } + +// save implements Object.save. +func (noObjects) save(w *Writer) {} + +// load implements Object.load. +func (noObjects) load(r *Reader) Object { return loadNoObjects(r) } + +// Struct is a basic composite value. +type Struct struct { + TypeID TypeID + fields Object // Optionally noObjects or *multipleObjects. +} + +// Field returns a pointer to the given field slot. +// +// This must be called after Alloc. +func (s *Struct) Field(i int) *Object { + if fields, ok := s.fields.(*multipleObjects); ok { + return &((*fields)[i]) + } + if _, ok := s.fields.(noObjects); ok { + // Alloc may be optionally called; can't call twice. + panic("Field called inappropriately, wrong Alloc?") + } + return &s.fields +} + +// Alloc allocates the given number of fields. +// +// This must be called before Add and Save. +// +// Precondition: slots must be positive. +func (s *Struct) Alloc(slots int) { + switch { + case slots == 0: + s.fields = noObjects{} + case slots == 1: + // Leave it alone. + case slots > 1: + fields := make(multipleObjects, slots) + s.fields = &fields + default: + // Violates precondition. + panic(fmt.Sprintf("Alloc called with negative slots %d?", slots)) + } +} + +// Fields returns the number of fields. +func (s *Struct) Fields() int { + switch x := s.fields.(type) { + case *multipleObjects: + return len(*x) + case noObjects: + return 0 + default: + return 1 + } +} + +// loadStruct loads an object of type Struct. +func loadStruct(r *Reader) Struct { + return Struct{ + TypeID: TypeID(loadUint(r)), + fields: Load(r), + } +} + +// save implements Object.save. +// +// Precondition: Alloc must have been called, and the fields all filled in +// appropriately. See Alloc and Add for more details. +func (s *Struct) save(w *Writer) { + Uint(s.TypeID).save(w) + Save(w, s.fields) +} + +// load implements Object.load. +func (*Struct) load(r *Reader) Object { + s := loadStruct(r) + return &s +} + +// Object types. +// +// N.B. Be careful about changing the order or introducing new elements in the +// middle here. This is part of the wire format and shouldn't change. +const ( + typeBool Uint = iota + typeInt + typeUint + typeFloat32 + typeFloat64 + typeNil + typeRef + typeString + typeSlice + typeArray + typeMap + typeStruct + typeNoObjects + typeMultipleObjects + typeInterface + typeComplex64 + typeComplex128 + typeType +) + +// Save saves the given object. +// +// +checkescape all +// +// N.B. This function will panic on error. +func Save(w *Writer, obj Object) { + switch x := obj.(type) { + case Bool: + typeBool.save(w) + x.save(w) + case Int: + typeInt.save(w) + x.save(w) + case Uint: + typeUint.save(w) + x.save(w) + case Float32: + typeFloat32.save(w) + x.save(w) + case Float64: + typeFloat64.save(w) + x.save(w) + case Nil: + typeNil.save(w) + x.save(w) + case *Ref: + typeRef.save(w) + x.save(w) + case *String: + typeString.save(w) + x.save(w) + case *Slice: + typeSlice.save(w) + x.save(w) + case *Array: + typeArray.save(w) + x.save(w) + case *Map: + typeMap.save(w) + x.save(w) + case *Struct: + typeStruct.save(w) + x.save(w) + case noObjects: + typeNoObjects.save(w) + x.save(w) + case *multipleObjects: + typeMultipleObjects.save(w) + x.save(w) + case *Interface: + typeInterface.save(w) + x.save(w) + case *Type: + typeType.save(w) + x.save(w) + case *Complex64: + typeComplex64.save(w) + x.save(w) + case *Complex128: + typeComplex128.save(w) + x.save(w) + default: + panic(fmt.Errorf("unknown type: %#v", obj)) + } +} + +// Load loads a new object. +// +// +checkescape all +// +// N.B. This function will panic on error. +func Load(r *Reader) Object { + switch hdr := loadUint(r); hdr { + case typeBool: + return loadBool(r) + case typeInt: + return loadInt(r) + case typeUint: + return loadUint(r) + case typeFloat32: + return loadFloat32(r) + case typeFloat64: + return loadFloat64(r) + case typeNil: + return loadNil(r) + case typeRef: + return ((*Ref)(nil)).load(r) // Escapes. + case typeString: + return ((*String)(nil)).load(r) // Escapes. + case typeSlice: + return ((*Slice)(nil)).load(r) // Escapes. + case typeArray: + return ((*Array)(nil)).load(r) // Escapes. + case typeMap: + return ((*Map)(nil)).load(r) // Escapes. + case typeStruct: + return ((*Struct)(nil)).load(r) // Escapes. + case typeNoObjects: // Special for struct. + return loadNoObjects(r) + case typeMultipleObjects: // Special for struct. + return ((*multipleObjects)(nil)).load(r) // Escapes. + case typeInterface: + return ((*Interface)(nil)).load(r) // Escapes. + case typeComplex64: + return ((*Complex64)(nil)).load(r) // Escapes. + case typeComplex128: + return ((*Complex128)(nil)).load(r) // Escapes. + case typeType: + return ((*Type)(nil)).load(r) // Escapes. + default: + // This is not a valid stream? + panic(fmt.Errorf("unknown header: %d", hdr)) + } +} + +// LoadUint loads a single unsigned integer. +// +// N.B. This function will panic on error. +func LoadUint(r *Reader) uint64 { + return uint64(loadUint(r)) +} + +// SaveUint saves a single unsigned integer. +// +// N.B. This function will panic on error. +func SaveUint(w *Writer, v uint64) { + Uint(v).save(w) +} diff --git a/pkg/sync/aliases.go b/pkg/sync/aliases.go new file mode 100644 index 0000000..3c425d1 --- /dev/null +++ b/pkg/sync/aliases.go @@ -0,0 +1,52 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package sync + +import ( + "sync" +) + +// Aliases of standard library types. +type ( + // Cond is an alias of sync.Cond. + Cond = sync.Cond + + // Locker is an alias of sync.Locker. + Locker = sync.Locker + + // Once is an alias of sync.Once. + Once = sync.Once + + // Pool is an alias of sync.Pool. + Pool = sync.Pool + + // WaitGroup is an alias of sync.WaitGroup. + WaitGroup = sync.WaitGroup + + // Map is an alias of sync.Map. + Map = sync.Map +) + +// NewCond is a wrapper around sync.NewCond. +func NewCond(l Locker) *Cond { + return sync.NewCond(l) +} + +// OnceFunc is a wrapper around sync.OnceFunc. +func OnceFunc(f func()) func() { + return sync.OnceFunc(f) +} + +// OnceValue is a wrapper around sync.OnceValue. +func OnceValue[T any](f func() T) func() T { + return sync.OnceValue(f) +} + +// OnceValues is a wrapper around sync.OnceValues. +func OnceValues[T1, T2 any](f func() (T1, T2)) func() (T1, T2) { + return sync.OnceValues(f) +} diff --git a/pkg/sync/checklocks_off_unsafe.go b/pkg/sync/checklocks_off_unsafe.go new file mode 100644 index 0000000..d305569 --- /dev/null +++ b/pkg/sync/checklocks_off_unsafe.go @@ -0,0 +1,20 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build !checklocks +// +build !checklocks + +package sync + +import ( + "unsafe" +) + +func noteLock(l unsafe.Pointer) { +} + +func noteUnlock(l unsafe.Pointer) { +} diff --git a/pkg/sync/checklocks_on_unsafe.go b/pkg/sync/checklocks_on_unsafe.go new file mode 100644 index 0000000..ecc2ec1 --- /dev/null +++ b/pkg/sync/checklocks_on_unsafe.go @@ -0,0 +1,110 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build checklocks +// +build checklocks + +package sync + +import ( + "fmt" + "strings" + "sync" + "unsafe" + + "github.com/sagernet/gvisor/pkg/goid" +) + +// gLocks contains metadata about the locks held by a goroutine. +type gLocks struct { + locksHeld []unsafe.Pointer +} + +// map[goid int]*gLocks +// +// Each key may only be written by the G with the goid it refers to. +// +// Note that entries are not evicted when a G exit, causing unbounded growth +// with new G creation / destruction. If this proves problematic, entries could +// be evicted when no locks are held at the expense of more allocations when +// taking top-level locks. +var locksHeld sync.Map + +func getGLocks() *gLocks { + id := goid.Get() + + var locks *gLocks + if l, ok := locksHeld.Load(id); ok { + locks = l.(*gLocks) + } else { + locks = &gLocks{ + // Initialize space for a few locks. + locksHeld: make([]unsafe.Pointer, 0, 8), + } + locksHeld.Store(id, locks) + } + + return locks +} + +func noteLock(l unsafe.Pointer) { + locks := getGLocks() + + for _, lock := range locks.locksHeld { + if lock == l { + panic(fmt.Sprintf("Deadlock on goroutine %d! Double lock of %p: %+v", goid.Get(), l, locks)) + } + } + + // Commit only after checking for panic conditions so that this lock + // isn't on the list if the above panic is recovered. + locks.locksHeld = append(locks.locksHeld, l) +} + +func noteUnlock(l unsafe.Pointer) { + locks := getGLocks() + + if len(locks.locksHeld) == 0 { + panic(fmt.Sprintf("Unlock of %p on goroutine %d without any locks held! All locks:\n%s", l, goid.Get(), dumpLocks())) + } + + // Search backwards since callers are most likely to unlock in LIFO order. + length := len(locks.locksHeld) + for i := length - 1; i >= 0; i-- { + if l == locks.locksHeld[i] { + copy(locks.locksHeld[i:length-1], locks.locksHeld[i+1:length]) + // Clear last entry to ensure addr can be GC'd. + locks.locksHeld[length-1] = nil + locks.locksHeld = locks.locksHeld[:length-1] + return + } + } + + panic(fmt.Sprintf("Unlock of %p on goroutine %d without matching lock! All locks:\n%s", l, goid.Get(), dumpLocks())) +} + +func dumpLocks() string { + var s strings.Builder + locksHeld.Range(func(key, value any) bool { + goid := key.(int64) + locks := value.(*gLocks) + + // N.B. accessing gLocks of another G is fundamentally racy. + + fmt.Fprintf(&s, "goroutine %d:\n", goid) + if len(locks.locksHeld) == 0 { + fmt.Fprintf(&s, "\t\n") + } + for _, lock := range locks.locksHeld { + fmt.Fprintf(&s, "\t%p\n", lock) + } + fmt.Fprintf(&s, "\n") + + return true + }) + + return s.String() +} diff --git a/pkg/sync/fence.go b/pkg/sync/fence.go new file mode 100644 index 0000000..6706676 --- /dev/null +++ b/pkg/sync/fence.go @@ -0,0 +1,19 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +// MemoryFenceReads ensures that all preceding memory loads happen before +// following memory loads. +func MemoryFenceReads() diff --git a/pkg/sync/fence_amd64.s b/pkg/sync/fence_amd64.s new file mode 100644 index 0000000..87766f1 --- /dev/null +++ b/pkg/sync/fence_amd64.s @@ -0,0 +1,26 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 +// +build amd64 + +#include "textflag.h" + +// func MemoryFenceReads() +TEXT ·MemoryFenceReads(SB),NOSPLIT|NOFRAME,$0-0 + // No memory fence is required on x86. However, a compiler fence is + // required to prevent the compiler from reordering memory accesses. The Go + // compiler will not reorder memory accesses around a call to an assembly + // function; compare runtime.publicationBarrier. + RET diff --git a/pkg/sync/fence_arm64.s b/pkg/sync/fence_arm64.s new file mode 100644 index 0000000..f4f9ce9 --- /dev/null +++ b/pkg/sync/fence_arm64.s @@ -0,0 +1,23 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +#include "textflag.h" + +// func MemoryFenceReads() +TEXT ·MemoryFenceReads(SB),NOSPLIT|NOFRAME,$0-0 + DMB $0x9 // ISHLD + RET diff --git a/pkg/sync/gate_unsafe.go b/pkg/sync/gate_unsafe.go new file mode 100644 index 0000000..40c44da --- /dev/null +++ b/pkg/sync/gate_unsafe.go @@ -0,0 +1,151 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "fmt" + "math" + "sync/atomic" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" +) + +// Gate is a synchronization primitive that allows concurrent goroutines to +// "enter" it as long as it hasn't been closed yet. Once it's been closed, +// goroutines cannot enter it anymore, but are allowed to leave, and the closer +// will be informed when all goroutines have left. +// +// Gate is similar to WaitGroup: +// +// - Gate.Enter() is analogous to WaitGroup.Add(1), but may be called even if +// the Gate counter is 0 and fails if Gate.Close() has been called. +// +// - Gate.Leave() is equivalent to WaitGroup.Done(). +// +// - Gate.Close() is analogous to WaitGroup.Wait(), but also causes future +// +// calls to Gate.Enter() to fail and may only be called once, from a single +// goroutine. +// +// This is useful, for example, in cases when a goroutine is trying to clean up +// an object for which multiple goroutines have pointers. In such a case, users +// would be required to enter and leave the Gate, and the cleaner would wait +// until all users are gone (and no new ones are allowed) before proceeding. +// +// Users: +// +// if !g.Enter() { +// // Gate is closed, we can't use the object. +// return +// } +// +// // Do something with object. +// [...] +// +// g.Leave() +// +// Closer: +// +// // Prevent new users from using the object, and wait for the existing +// // ones to complete. +// g.Close() +// +// // Clean up the object. +// [...] +type Gate struct { + userCount int32 + closingG uintptr +} + +const preparingG = 1 + +// Enter tries to enter the gate. It will succeed if it hasn't been closed yet, +// in which case the caller must eventually call Leave(). +// +// This function is thread-safe. +func (g *Gate) Enter() bool { + if atomic.AddInt32(&g.userCount, 1) > 0 { + return true + } + g.leaveAfterFailedEnter() + return false +} + +// leaveAfterFailedEnter is identical to Leave, but is marked noinline to +// prevent it from being inlined into Enter, since as of this writing inlining +// Leave into Enter prevents Enter from being inlined into its callers. +// +//go:noinline +func (g *Gate) leaveAfterFailedEnter() { + if atomic.AddInt32(&g.userCount, -1) == math.MinInt32 { + g.leaveClosed() + } +} + +// Leave leaves the gate. This must only be called after a successful call to +// Enter(). If the gate has been closed and this is the last one inside the +// gate, it will notify the closer that the gate is done. +// +// This function is thread-safe. +func (g *Gate) Leave() { + if atomic.AddInt32(&g.userCount, -1) == math.MinInt32 { + g.leaveClosed() + } +} + +func (g *Gate) leaveClosed() { + if atomic.LoadUintptr(&g.closingG) == 0 { + return + } + if g := atomic.SwapUintptr(&g.closingG, 0); g > preparingG { + goready(g, 0) + } +} + +// Close closes the gate, causing future calls to Enter to fail, and waits +// until all goroutines that are currently inside the gate leave before +// returning. +// +// Only one goroutine can call this function. +func (g *Gate) Close() { + if atomic.LoadInt32(&g.userCount) == math.MinInt32 { + // The gate is already closed, with no goroutines inside. For legacy + // reasons, we have to allow Close to be called again in this case. + return + } + if v := atomic.AddInt32(&g.userCount, math.MinInt32); v == math.MinInt32 { + // userCount was already 0. + return + } else if v >= 0 { + panic("concurrent Close of sync.Gate") + } + + if g := atomic.SwapUintptr(&g.closingG, preparingG); g != 0 { + panic(fmt.Sprintf("invalid sync.Gate.closingG during Close: %#x", g)) + } + if atomic.LoadInt32(&g.userCount) == math.MinInt32 { + // The last call to Leave arrived while we were setting up closingG. + return + } + // WaitReasonSemacquire/TraceBlockSync are consistent with WaitGroup. + gopark(gateCommit, gohacks.Noescape(unsafe.Pointer(&g.closingG)), WaitReasonSemacquire, TraceBlockSync, 0) +} + +//go:norace +//go:nosplit +func gateCommit(g uintptr, closingG unsafe.Pointer) bool { + return RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(closingG), preparingG, g) +} diff --git a/pkg/sync/goyield_go113_unsafe.go b/pkg/sync/goyield_go113_unsafe.go new file mode 100644 index 0000000..2a1da77 --- /dev/null +++ b/pkg/sync/goyield_go113_unsafe.go @@ -0,0 +1,19 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build go1.13 && !go1.14 +// +build go1.13,!go1.14 + +package sync + +import ( + "runtime" +) + +func goyield() { + // goyield is not available until Go 1.14. + runtime.Gosched() +} diff --git a/pkg/sync/goyield_unsafe.go b/pkg/sync/goyield_unsafe.go new file mode 100644 index 0000000..db9eae6 --- /dev/null +++ b/pkg/sync/goyield_unsafe.go @@ -0,0 +1,21 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build go1.14 +// +build go1.14 + +// //go:linkname directives type-checked by checklinkname. Any other +// non-linkname assumptions outside the Go 1 compatibility guarantee should +// have an accompanied vet check or version guard build tag. + +package sync + +import ( + _ "unsafe" // for go:linkname +) + +//go:linkname goyield runtime.goyield +func goyield() diff --git a/pkg/sync/locking/atomicptrmap_ancestors_unsafe.go b/pkg/sync/locking/atomicptrmap_ancestors_unsafe.go new file mode 100644 index 0000000..d72d26e --- /dev/null +++ b/pkg/sync/locking/atomicptrmap_ancestors_unsafe.go @@ -0,0 +1,439 @@ +package locking + +import ( + "sync/atomic" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/sync" +) + +const ( + // ShardOrder is an optional parameter specifying the base-2 log of the + // number of shards per AtomicPtrMap. Higher values of ShardOrder reduce + // unnecessary synchronization between unrelated concurrent operations, + // improving performance for write-heavy workloads, but increase memory + // usage for small maps. + ancestorsShardOrder = 0 +) + +// Hasher is an optional type parameter. If Hasher is provided, it must define +// the Init and Hash methods. One Hasher will be shared by all AtomicPtrMaps. +type ancestorsHasher struct { + ancestorsdefaultHasher +} + +// defaultHasher is the default Hasher. This indirection exists because +// defaultHasher must exist even if a custom Hasher is provided, to prevent the +// Go compiler from complaining about defaultHasher's unused imports. +type ancestorsdefaultHasher struct { + fn func(unsafe.Pointer, uintptr) uintptr + seed uintptr +} + +// Init initializes the Hasher. +func (h *ancestorsdefaultHasher) Init() { + h.fn = sync.MapKeyHasher(map[*MutexClass]*string(nil)) + h.seed = sync.RandUintptr() +} + +// Hash returns the hash value for the given Key. +func (h *ancestorsdefaultHasher) Hash(key *MutexClass) uintptr { + return h.fn(gohacks.Noescape(unsafe.Pointer(&key)), h.seed) +} + +var ancestorshasher ancestorsHasher + +func init() { + ancestorshasher.Init() +} + +// An AtomicPtrMap maps Keys to non-nil pointers to Values. AtomicPtrMap are +// safe for concurrent use from multiple goroutines without additional +// synchronization. +// +// The zero value of AtomicPtrMap is empty (maps all Keys to nil) and ready for +// use. AtomicPtrMaps must not be copied after first use. +// +// sync.Map may be faster than AtomicPtrMap if most operations on the map are +// concurrent writes to a fixed set of keys. AtomicPtrMap is usually faster in +// other circumstances. +type ancestorsAtomicPtrMap struct { + shards [1 << ancestorsShardOrder]ancestorsapmShard +} + +func (m *ancestorsAtomicPtrMap) shard(hash uintptr) *ancestorsapmShard { + // Go defines right shifts >= width of shifted unsigned operand as 0, so + // this is correct even if ShardOrder is 0 (although nogo complains because + // nogo is dumb). + const indexLSB = unsafe.Sizeof(uintptr(0))*8 - ancestorsShardOrder + index := hash >> indexLSB + return (*ancestorsapmShard)(unsafe.Pointer(uintptr(unsafe.Pointer(&m.shards)) + (index * unsafe.Sizeof(ancestorsapmShard{})))) +} + +type ancestorsapmShard struct { + ancestorsapmShardMutationData + _ [ancestorsapmShardMutationDataPadding]byte + ancestorsapmShardLookupData + _ [ancestorsapmShardLookupDataPadding]byte +} + +type ancestorsapmShardMutationData struct { + dirtyMu sync.Mutex // serializes slot transitions out of empty + dirty uintptr // # slots with val != nil + count uintptr // # slots with val != nil and val != tombstone() + rehashMu sync.Mutex // serializes rehashing +} + +type ancestorsapmShardLookupData struct { + seq sync.SeqCount // allows atomic reads of slots+mask + slots unsafe.Pointer // [mask+1]slot or nil; protected by rehashMu/seq + mask uintptr // always (a power of 2) - 1; protected by rehashMu/seq +} + +const ( + ancestorscacheLineBytes = 64 + // Cache line padding is enabled if sharding is. + ancestorsapmEnablePadding = (ancestorsShardOrder + 63) >> 6 // 0 if ShardOrder == 0, 1 otherwise + // The -1 and +1 below are required to ensure that if unsafe.Sizeof(T) % + // cacheLineBytes == 0, then padding is 0 (rather than cacheLineBytes). + ancestorsapmShardMutationDataRequiredPadding = ancestorscacheLineBytes - (((unsafe.Sizeof(ancestorsapmShardMutationData{}) - 1) % ancestorscacheLineBytes) + 1) + ancestorsapmShardMutationDataPadding = ancestorsapmEnablePadding * ancestorsapmShardMutationDataRequiredPadding + ancestorsapmShardLookupDataRequiredPadding = ancestorscacheLineBytes - (((unsafe.Sizeof(ancestorsapmShardLookupData{}) - 1) % ancestorscacheLineBytes) + 1) + ancestorsapmShardLookupDataPadding = ancestorsapmEnablePadding * ancestorsapmShardLookupDataRequiredPadding + + // These define fractional thresholds for when apmShard.rehash() is called + // (i.e. the load factor) and when it rehases to a larger table + // respectively. They are chosen such that the rehash threshold = the + // expansion threshold + 1/2, so that when reuse of deleted slots is rare + // or non-existent, rehashing occurs after the insertion of at least 1/2 + // the table's size in new entries, which is acceptably infrequent. + ancestorsapmRehashThresholdNum = 2 + ancestorsapmRehashThresholdDen = 3 + ancestorsapmExpansionThresholdNum = 1 + ancestorsapmExpansionThresholdDen = 6 +) + +type ancestorsapmSlot struct { + // slot states are indicated by val: + // + // * Empty: val == nil; key is meaningless. May transition to full or + // evacuated with dirtyMu locked. + // + // * Full: val != nil, tombstone(), or evacuated(); key is immutable. val + // is the Value mapped to key. May transition to deleted or evacuated. + // + // * Deleted: val == tombstone(); key is still immutable. key is mapped to + // no Value. May transition to full or evacuated. + // + // * Evacuated: val == evacuated(); key is immutable. Set by rehashing on + // slots that have already been moved, requiring readers to wait for + // rehashing to complete and use the new table. Terminal state. + // + // Note that once val is non-nil, it cannot become nil again. That is, the + // transition from empty to non-empty is irreversible for a given slot; + // the only way to create more empty slots is by rehashing. + val unsafe.Pointer + key *MutexClass +} + +func ancestorsapmSlotAt(slots unsafe.Pointer, pos uintptr) *ancestorsapmSlot { + return (*ancestorsapmSlot)(unsafe.Pointer(uintptr(slots) + pos*unsafe.Sizeof(ancestorsapmSlot{}))) +} + +var ancestorstombstoneObj byte + +func ancestorstombstone() unsafe.Pointer { + return unsafe.Pointer(&ancestorstombstoneObj) +} + +var ancestorsevacuatedObj byte + +func ancestorsevacuated() unsafe.Pointer { + return unsafe.Pointer(&ancestorsevacuatedObj) +} + +// Load returns the Value stored in m for key. +func (m *ancestorsAtomicPtrMap) Load(key *MutexClass) *string { + hash := ancestorshasher.Hash(key) + shard := m.shard(hash) + +retry: + epoch := shard.seq.BeginRead() + slots := atomic.LoadPointer(&shard.slots) + mask := atomic.LoadUintptr(&shard.mask) + if !shard.seq.ReadOk(epoch) { + goto retry + } + if slots == nil { + return nil + } + + i := hash & mask + inc := uintptr(1) + for { + slot := ancestorsapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == nil { + return nil + } + if slotVal == ancestorsevacuated() { + goto retry + } + if slot.key == key { + if slotVal == ancestorstombstone() { + return nil + } + return (*string)(slotVal) + } + i = (i + inc) & mask + inc++ + } +} + +// Store stores the Value val for key. +func (m *ancestorsAtomicPtrMap) Store(key *MutexClass, val *string) { + m.maybeCompareAndSwap(key, false, nil, val) +} + +// Swap stores the Value val for key and returns the previously-mapped Value. +func (m *ancestorsAtomicPtrMap) Swap(key *MutexClass, val *string) *string { + return m.maybeCompareAndSwap(key, false, nil, val) +} + +// CompareAndSwap checks that the Value stored for key is oldVal; if it is, it +// stores the Value newVal for key. CompareAndSwap returns the previous Value +// stored for key, whether or not it stores newVal. +func (m *ancestorsAtomicPtrMap) CompareAndSwap(key *MutexClass, oldVal, newVal *string) *string { + return m.maybeCompareAndSwap(key, true, oldVal, newVal) +} + +func (m *ancestorsAtomicPtrMap) maybeCompareAndSwap(key *MutexClass, compare bool, typedOldVal, typedNewVal *string) *string { + hash := ancestorshasher.Hash(key) + shard := m.shard(hash) + oldVal := ancestorstombstone() + if typedOldVal != nil { + oldVal = unsafe.Pointer(typedOldVal) + } + newVal := ancestorstombstone() + if typedNewVal != nil { + newVal = unsafe.Pointer(typedNewVal) + } + +retry: + epoch := shard.seq.BeginRead() + slots := atomic.LoadPointer(&shard.slots) + mask := atomic.LoadUintptr(&shard.mask) + if !shard.seq.ReadOk(epoch) { + goto retry + } + if slots == nil { + if (compare && oldVal != ancestorstombstone()) || newVal == ancestorstombstone() { + return nil + } + + shard.rehash(nil) + goto retry + } + + i := hash & mask + inc := uintptr(1) + for { + slot := ancestorsapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == nil { + if (compare && oldVal != ancestorstombstone()) || newVal == ancestorstombstone() { + return nil + } + + shard.dirtyMu.Lock() + slotVal = atomic.LoadPointer(&slot.val) + if slotVal == nil { + + if dirty, capacity := shard.dirty+1, mask+1; dirty*ancestorsapmRehashThresholdDen >= capacity*ancestorsapmRehashThresholdNum { + shard.dirtyMu.Unlock() + shard.rehash(slots) + goto retry + } + slot.key = key + atomic.StorePointer(&slot.val, newVal) + shard.dirty++ + atomic.AddUintptr(&shard.count, 1) + shard.dirtyMu.Unlock() + return nil + } + + shard.dirtyMu.Unlock() + } + if slotVal == ancestorsevacuated() { + goto retry + } + if slot.key == key { + for { + if (compare && oldVal != slotVal) || newVal == slotVal { + if slotVal == ancestorstombstone() { + return nil + } + return (*string)(slotVal) + } + if atomic.CompareAndSwapPointer(&slot.val, slotVal, newVal) { + if slotVal == ancestorstombstone() { + atomic.AddUintptr(&shard.count, 1) + return nil + } + if newVal == ancestorstombstone() { + atomic.AddUintptr(&shard.count, ^uintptr(0)) + } + return (*string)(slotVal) + } + slotVal = atomic.LoadPointer(&slot.val) + if slotVal == ancestorsevacuated() { + goto retry + } + } + } + + i = (i + inc) & mask + inc++ + } +} + +// rehash is marked nosplit to avoid preemption during table copying. +// +//go:nosplit +func (shard *ancestorsapmShard) rehash(oldSlots unsafe.Pointer) { + shard.rehashMu.Lock() + defer shard.rehashMu.Unlock() + + if shard.slots != oldSlots { + return + } + + newSize := uintptr(8) + if oldSlots != nil { + oldSize := shard.mask + 1 + newSize = oldSize + if count := atomic.LoadUintptr(&shard.count) + 1; count*ancestorsapmExpansionThresholdDen > oldSize*ancestorsapmExpansionThresholdNum { + newSize *= 2 + } + } + + newSlotsSlice := make([]ancestorsapmSlot, newSize) + newSlots := unsafe.Pointer(&newSlotsSlice[0]) + newMask := newSize - 1 + + shard.dirtyMu.Lock() + shard.seq.BeginWrite() + + if oldSlots != nil { + realCount := uintptr(0) + + oldMask := shard.mask + for i := uintptr(0); i <= oldMask; i++ { + oldSlot := ancestorsapmSlotAt(oldSlots, i) + val := atomic.SwapPointer(&oldSlot.val, ancestorsevacuated()) + if val == nil || val == ancestorstombstone() { + continue + } + hash := ancestorshasher.Hash(oldSlot.key) + j := hash & newMask + inc := uintptr(1) + for { + newSlot := ancestorsapmSlotAt(newSlots, j) + if newSlot.val == nil { + newSlot.val = val + newSlot.key = oldSlot.key + break + } + j = (j + inc) & newMask + inc++ + } + realCount++ + } + + shard.dirty = realCount + } + + atomic.StorePointer(&shard.slots, newSlots) + atomic.StoreUintptr(&shard.mask, newMask) + + shard.seq.EndWrite() + shard.dirtyMu.Unlock() +} + +// Range invokes f on each Key-Value pair stored in m. If any call to f returns +// false, Range stops iteration and returns. +// +// Range does not necessarily correspond to any consistent snapshot of the +// Map's contents: no Key will be visited more than once, but if the Value for +// any Key is stored or deleted concurrently, Range may reflect any mapping for +// that Key from any point during the Range call. +// +// f must not call other methods on m. +func (m *ancestorsAtomicPtrMap) Range(f func(key *MutexClass, val *string) bool) { + for si := 0; si < len(m.shards); si++ { + shard := &m.shards[si] + if !shard.doRange(f) { + return + } + } +} + +func (shard *ancestorsapmShard) doRange(f func(key *MutexClass, val *string) bool) bool { + shard.rehashMu.Lock() + defer shard.rehashMu.Unlock() + slots := shard.slots + if slots == nil { + return true + } + mask := shard.mask + for i := uintptr(0); i <= mask; i++ { + slot := ancestorsapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == nil || slotVal == ancestorstombstone() { + continue + } + if !f(slot.key, (*string)(slotVal)) { + return false + } + } + return true +} + +// RangeRepeatable is like Range, but: +// +// - RangeRepeatable may visit the same Key multiple times in the presence of +// concurrent mutators, possibly passing different Values to f in different +// calls. +// +// - It is safe for f to call other methods on m. +func (m *ancestorsAtomicPtrMap) RangeRepeatable(f func(key *MutexClass, val *string) bool) { + for si := 0; si < len(m.shards); si++ { + shard := &m.shards[si] + + retry: + epoch := shard.seq.BeginRead() + slots := atomic.LoadPointer(&shard.slots) + mask := atomic.LoadUintptr(&shard.mask) + if !shard.seq.ReadOk(epoch) { + goto retry + } + if slots == nil { + continue + } + + for i := uintptr(0); i <= mask; i++ { + slot := ancestorsapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == ancestorsevacuated() { + goto retry + } + if slotVal == nil || slotVal == ancestorstombstone() { + continue + } + if !f(slot.key, (*string)(slotVal)) { + return + } + } + } +} diff --git a/pkg/sync/locking/atomicptrmap_goroutine_unsafe.go b/pkg/sync/locking/atomicptrmap_goroutine_unsafe.go new file mode 100644 index 0000000..44143ed --- /dev/null +++ b/pkg/sync/locking/atomicptrmap_goroutine_unsafe.go @@ -0,0 +1,439 @@ +package locking + +import ( + "sync/atomic" + "unsafe" + + "github.com/sagernet/gvisor/pkg/gohacks" + "github.com/sagernet/gvisor/pkg/sync" +) + +const ( + // ShardOrder is an optional parameter specifying the base-2 log of the + // number of shards per AtomicPtrMap. Higher values of ShardOrder reduce + // unnecessary synchronization between unrelated concurrent operations, + // improving performance for write-heavy workloads, but increase memory + // usage for small maps. + goroutineLocksShardOrder = 0 +) + +// Hasher is an optional type parameter. If Hasher is provided, it must define +// the Init and Hash methods. One Hasher will be shared by all AtomicPtrMaps. +type goroutineLocksHasher struct { + goroutineLocksdefaultHasher +} + +// defaultHasher is the default Hasher. This indirection exists because +// defaultHasher must exist even if a custom Hasher is provided, to prevent the +// Go compiler from complaining about defaultHasher's unused imports. +type goroutineLocksdefaultHasher struct { + fn func(unsafe.Pointer, uintptr) uintptr + seed uintptr +} + +// Init initializes the Hasher. +func (h *goroutineLocksdefaultHasher) Init() { + h.fn = sync.MapKeyHasher(map[int64]*goroutineLocks(nil)) + h.seed = sync.RandUintptr() +} + +// Hash returns the hash value for the given Key. +func (h *goroutineLocksdefaultHasher) Hash(key int64) uintptr { + return h.fn(gohacks.Noescape(unsafe.Pointer(&key)), h.seed) +} + +var goroutineLockshasher goroutineLocksHasher + +func init() { + goroutineLockshasher.Init() +} + +// An AtomicPtrMap maps Keys to non-nil pointers to Values. AtomicPtrMap are +// safe for concurrent use from multiple goroutines without additional +// synchronization. +// +// The zero value of AtomicPtrMap is empty (maps all Keys to nil) and ready for +// use. AtomicPtrMaps must not be copied after first use. +// +// sync.Map may be faster than AtomicPtrMap if most operations on the map are +// concurrent writes to a fixed set of keys. AtomicPtrMap is usually faster in +// other circumstances. +type goroutineLocksAtomicPtrMap struct { + shards [1 << goroutineLocksShardOrder]goroutineLocksapmShard +} + +func (m *goroutineLocksAtomicPtrMap) shard(hash uintptr) *goroutineLocksapmShard { + // Go defines right shifts >= width of shifted unsigned operand as 0, so + // this is correct even if ShardOrder is 0 (although nogo complains because + // nogo is dumb). + const indexLSB = unsafe.Sizeof(uintptr(0))*8 - goroutineLocksShardOrder + index := hash >> indexLSB + return (*goroutineLocksapmShard)(unsafe.Pointer(uintptr(unsafe.Pointer(&m.shards)) + (index * unsafe.Sizeof(goroutineLocksapmShard{})))) +} + +type goroutineLocksapmShard struct { + goroutineLocksapmShardMutationData + _ [goroutineLocksapmShardMutationDataPadding]byte + goroutineLocksapmShardLookupData + _ [goroutineLocksapmShardLookupDataPadding]byte +} + +type goroutineLocksapmShardMutationData struct { + dirtyMu sync.Mutex // serializes slot transitions out of empty + dirty uintptr // # slots with val != nil + count uintptr // # slots with val != nil and val != tombstone() + rehashMu sync.Mutex // serializes rehashing +} + +type goroutineLocksapmShardLookupData struct { + seq sync.SeqCount // allows atomic reads of slots+mask + slots unsafe.Pointer // [mask+1]slot or nil; protected by rehashMu/seq + mask uintptr // always (a power of 2) - 1; protected by rehashMu/seq +} + +const ( + goroutineLockscacheLineBytes = 64 + // Cache line padding is enabled if sharding is. + goroutineLocksapmEnablePadding = (goroutineLocksShardOrder + 63) >> 6 // 0 if ShardOrder == 0, 1 otherwise + // The -1 and +1 below are required to ensure that if unsafe.Sizeof(T) % + // cacheLineBytes == 0, then padding is 0 (rather than cacheLineBytes). + goroutineLocksapmShardMutationDataRequiredPadding = goroutineLockscacheLineBytes - (((unsafe.Sizeof(goroutineLocksapmShardMutationData{}) - 1) % goroutineLockscacheLineBytes) + 1) + goroutineLocksapmShardMutationDataPadding = goroutineLocksapmEnablePadding * goroutineLocksapmShardMutationDataRequiredPadding + goroutineLocksapmShardLookupDataRequiredPadding = goroutineLockscacheLineBytes - (((unsafe.Sizeof(goroutineLocksapmShardLookupData{}) - 1) % goroutineLockscacheLineBytes) + 1) + goroutineLocksapmShardLookupDataPadding = goroutineLocksapmEnablePadding * goroutineLocksapmShardLookupDataRequiredPadding + + // These define fractional thresholds for when apmShard.rehash() is called + // (i.e. the load factor) and when it rehases to a larger table + // respectively. They are chosen such that the rehash threshold = the + // expansion threshold + 1/2, so that when reuse of deleted slots is rare + // or non-existent, rehashing occurs after the insertion of at least 1/2 + // the table's size in new entries, which is acceptably infrequent. + goroutineLocksapmRehashThresholdNum = 2 + goroutineLocksapmRehashThresholdDen = 3 + goroutineLocksapmExpansionThresholdNum = 1 + goroutineLocksapmExpansionThresholdDen = 6 +) + +type goroutineLocksapmSlot struct { + // slot states are indicated by val: + // + // * Empty: val == nil; key is meaningless. May transition to full or + // evacuated with dirtyMu locked. + // + // * Full: val != nil, tombstone(), or evacuated(); key is immutable. val + // is the Value mapped to key. May transition to deleted or evacuated. + // + // * Deleted: val == tombstone(); key is still immutable. key is mapped to + // no Value. May transition to full or evacuated. + // + // * Evacuated: val == evacuated(); key is immutable. Set by rehashing on + // slots that have already been moved, requiring readers to wait for + // rehashing to complete and use the new table. Terminal state. + // + // Note that once val is non-nil, it cannot become nil again. That is, the + // transition from empty to non-empty is irreversible for a given slot; + // the only way to create more empty slots is by rehashing. + val unsafe.Pointer + key int64 +} + +func goroutineLocksapmSlotAt(slots unsafe.Pointer, pos uintptr) *goroutineLocksapmSlot { + return (*goroutineLocksapmSlot)(unsafe.Pointer(uintptr(slots) + pos*unsafe.Sizeof(goroutineLocksapmSlot{}))) +} + +var goroutineLockstombstoneObj byte + +func goroutineLockstombstone() unsafe.Pointer { + return unsafe.Pointer(&goroutineLockstombstoneObj) +} + +var goroutineLocksevacuatedObj byte + +func goroutineLocksevacuated() unsafe.Pointer { + return unsafe.Pointer(&goroutineLocksevacuatedObj) +} + +// Load returns the Value stored in m for key. +func (m *goroutineLocksAtomicPtrMap) Load(key int64) *goroutineLocks { + hash := goroutineLockshasher.Hash(key) + shard := m.shard(hash) + +retry: + epoch := shard.seq.BeginRead() + slots := atomic.LoadPointer(&shard.slots) + mask := atomic.LoadUintptr(&shard.mask) + if !shard.seq.ReadOk(epoch) { + goto retry + } + if slots == nil { + return nil + } + + i := hash & mask + inc := uintptr(1) + for { + slot := goroutineLocksapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == nil { + return nil + } + if slotVal == goroutineLocksevacuated() { + goto retry + } + if slot.key == key { + if slotVal == goroutineLockstombstone() { + return nil + } + return (*goroutineLocks)(slotVal) + } + i = (i + inc) & mask + inc++ + } +} + +// Store stores the Value val for key. +func (m *goroutineLocksAtomicPtrMap) Store(key int64, val *goroutineLocks) { + m.maybeCompareAndSwap(key, false, nil, val) +} + +// Swap stores the Value val for key and returns the previously-mapped Value. +func (m *goroutineLocksAtomicPtrMap) Swap(key int64, val *goroutineLocks) *goroutineLocks { + return m.maybeCompareAndSwap(key, false, nil, val) +} + +// CompareAndSwap checks that the Value stored for key is oldVal; if it is, it +// stores the Value newVal for key. CompareAndSwap returns the previous Value +// stored for key, whether or not it stores newVal. +func (m *goroutineLocksAtomicPtrMap) CompareAndSwap(key int64, oldVal, newVal *goroutineLocks) *goroutineLocks { + return m.maybeCompareAndSwap(key, true, oldVal, newVal) +} + +func (m *goroutineLocksAtomicPtrMap) maybeCompareAndSwap(key int64, compare bool, typedOldVal, typedNewVal *goroutineLocks) *goroutineLocks { + hash := goroutineLockshasher.Hash(key) + shard := m.shard(hash) + oldVal := goroutineLockstombstone() + if typedOldVal != nil { + oldVal = unsafe.Pointer(typedOldVal) + } + newVal := goroutineLockstombstone() + if typedNewVal != nil { + newVal = unsafe.Pointer(typedNewVal) + } + +retry: + epoch := shard.seq.BeginRead() + slots := atomic.LoadPointer(&shard.slots) + mask := atomic.LoadUintptr(&shard.mask) + if !shard.seq.ReadOk(epoch) { + goto retry + } + if slots == nil { + if (compare && oldVal != goroutineLockstombstone()) || newVal == goroutineLockstombstone() { + return nil + } + + shard.rehash(nil) + goto retry + } + + i := hash & mask + inc := uintptr(1) + for { + slot := goroutineLocksapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == nil { + if (compare && oldVal != goroutineLockstombstone()) || newVal == goroutineLockstombstone() { + return nil + } + + shard.dirtyMu.Lock() + slotVal = atomic.LoadPointer(&slot.val) + if slotVal == nil { + + if dirty, capacity := shard.dirty+1, mask+1; dirty*goroutineLocksapmRehashThresholdDen >= capacity*goroutineLocksapmRehashThresholdNum { + shard.dirtyMu.Unlock() + shard.rehash(slots) + goto retry + } + slot.key = key + atomic.StorePointer(&slot.val, newVal) + shard.dirty++ + atomic.AddUintptr(&shard.count, 1) + shard.dirtyMu.Unlock() + return nil + } + + shard.dirtyMu.Unlock() + } + if slotVal == goroutineLocksevacuated() { + goto retry + } + if slot.key == key { + for { + if (compare && oldVal != slotVal) || newVal == slotVal { + if slotVal == goroutineLockstombstone() { + return nil + } + return (*goroutineLocks)(slotVal) + } + if atomic.CompareAndSwapPointer(&slot.val, slotVal, newVal) { + if slotVal == goroutineLockstombstone() { + atomic.AddUintptr(&shard.count, 1) + return nil + } + if newVal == goroutineLockstombstone() { + atomic.AddUintptr(&shard.count, ^uintptr(0)) + } + return (*goroutineLocks)(slotVal) + } + slotVal = atomic.LoadPointer(&slot.val) + if slotVal == goroutineLocksevacuated() { + goto retry + } + } + } + + i = (i + inc) & mask + inc++ + } +} + +// rehash is marked nosplit to avoid preemption during table copying. +// +//go:nosplit +func (shard *goroutineLocksapmShard) rehash(oldSlots unsafe.Pointer) { + shard.rehashMu.Lock() + defer shard.rehashMu.Unlock() + + if shard.slots != oldSlots { + return + } + + newSize := uintptr(8) + if oldSlots != nil { + oldSize := shard.mask + 1 + newSize = oldSize + if count := atomic.LoadUintptr(&shard.count) + 1; count*goroutineLocksapmExpansionThresholdDen > oldSize*goroutineLocksapmExpansionThresholdNum { + newSize *= 2 + } + } + + newSlotsSlice := make([]goroutineLocksapmSlot, newSize) + newSlots := unsafe.Pointer(&newSlotsSlice[0]) + newMask := newSize - 1 + + shard.dirtyMu.Lock() + shard.seq.BeginWrite() + + if oldSlots != nil { + realCount := uintptr(0) + + oldMask := shard.mask + for i := uintptr(0); i <= oldMask; i++ { + oldSlot := goroutineLocksapmSlotAt(oldSlots, i) + val := atomic.SwapPointer(&oldSlot.val, goroutineLocksevacuated()) + if val == nil || val == goroutineLockstombstone() { + continue + } + hash := goroutineLockshasher.Hash(oldSlot.key) + j := hash & newMask + inc := uintptr(1) + for { + newSlot := goroutineLocksapmSlotAt(newSlots, j) + if newSlot.val == nil { + newSlot.val = val + newSlot.key = oldSlot.key + break + } + j = (j + inc) & newMask + inc++ + } + realCount++ + } + + shard.dirty = realCount + } + + atomic.StorePointer(&shard.slots, newSlots) + atomic.StoreUintptr(&shard.mask, newMask) + + shard.seq.EndWrite() + shard.dirtyMu.Unlock() +} + +// Range invokes f on each Key-Value pair stored in m. If any call to f returns +// false, Range stops iteration and returns. +// +// Range does not necessarily correspond to any consistent snapshot of the +// Map's contents: no Key will be visited more than once, but if the Value for +// any Key is stored or deleted concurrently, Range may reflect any mapping for +// that Key from any point during the Range call. +// +// f must not call other methods on m. +func (m *goroutineLocksAtomicPtrMap) Range(f func(key int64, val *goroutineLocks) bool) { + for si := 0; si < len(m.shards); si++ { + shard := &m.shards[si] + if !shard.doRange(f) { + return + } + } +} + +func (shard *goroutineLocksapmShard) doRange(f func(key int64, val *goroutineLocks) bool) bool { + shard.rehashMu.Lock() + defer shard.rehashMu.Unlock() + slots := shard.slots + if slots == nil { + return true + } + mask := shard.mask + for i := uintptr(0); i <= mask; i++ { + slot := goroutineLocksapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == nil || slotVal == goroutineLockstombstone() { + continue + } + if !f(slot.key, (*goroutineLocks)(slotVal)) { + return false + } + } + return true +} + +// RangeRepeatable is like Range, but: +// +// - RangeRepeatable may visit the same Key multiple times in the presence of +// concurrent mutators, possibly passing different Values to f in different +// calls. +// +// - It is safe for f to call other methods on m. +func (m *goroutineLocksAtomicPtrMap) RangeRepeatable(f func(key int64, val *goroutineLocks) bool) { + for si := 0; si < len(m.shards); si++ { + shard := &m.shards[si] + + retry: + epoch := shard.seq.BeginRead() + slots := atomic.LoadPointer(&shard.slots) + mask := atomic.LoadUintptr(&shard.mask) + if !shard.seq.ReadOk(epoch) { + goto retry + } + if slots == nil { + continue + } + + for i := uintptr(0); i <= mask; i++ { + slot := goroutineLocksapmSlotAt(slots, i) + slotVal := atomic.LoadPointer(&slot.val) + if slotVal == goroutineLocksevacuated() { + goto retry + } + if slotVal == nil || slotVal == goroutineLockstombstone() { + continue + } + if !f(slot.key, (*goroutineLocks)(slotVal)) { + return + } + } + } +} diff --git a/pkg/sync/locking/lockdep.go b/pkg/sync/locking/lockdep.go new file mode 100644 index 0000000..e92b912 --- /dev/null +++ b/pkg/sync/locking/lockdep.go @@ -0,0 +1,191 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build lockdep +// +build lockdep + +package locking + +import ( + "fmt" + "reflect" + "strings" + + "github.com/sagernet/gvisor/pkg/goid" + "github.com/sagernet/gvisor/pkg/log" +) + +// NewMutexClass allocates a new mutex class. +func NewMutexClass(t reflect.Type, lockNames []string) *MutexClass { + c := &MutexClass{ + typ: t, + nestedLockNames: lockNames, + nestedLockClasses: make([]*MutexClass, len(lockNames)), + } + for i := range lockNames { + c.nestedLockClasses[i] = NewMutexClass(t, nil) + c.nestedLockClasses[i].lockName = lockNames[i] + } + return c +} + +// MutexClass describes dependencies of a specific class. +type MutexClass struct { + // The type of the mutex. + typ reflect.Type + + // Name of the nested lock of the above type. + lockName string + + // ancestors are locks that are locked before the current class. + ancestors ancestorsAtomicPtrMap + // nestedLockNames is a list of names for nested locks which are considered difference instances + // of the same lock class. + nestedLockNames []string + // namedLockClasses is a list of MutexClass instances of the same mutex class, but that are + // considered OK to lock simultaneously with each other, as well as with this mutex class. + // This is used for nested locking, where multiple instances of the same lock class are used + // simultaneously. + // Maps one-to-one with nestedLockNames. + nestedLockClasses []*MutexClass +} + +func (m *MutexClass) String() string { + if m.lockName == "" { + return m.typ.String() + } + return fmt.Sprintf("%s[%s]", m.typ.String(), m.lockName) +} + +type goroutineLocks map[*MutexClass]bool + +var routineLocks goroutineLocksAtomicPtrMap + +// maxChainLen is the maximum length of a lock chain. +const maxChainLen = 32 + +// checkLock checks that class isn't in the ancestors of prevClass. +func checkLock(class *MutexClass, prevClass *MutexClass, chain []*MutexClass) { + chain = append(chain, prevClass) + if len(chain) >= maxChainLen { + // It can be a race condition with another thread that added + // the lock to the graph but don't complete the validation. + var b strings.Builder + fmt.Fprintf(&b, "WARNING: The maximum lock depth has been reached: %s", chain[0]) + for i := 1; i < len(chain); i++ { + fmt.Fprintf(&b, "-> %s", chain[i]) + } + log.Warningf("%s", b.String()) + return + } + if c := prevClass.ancestors.Load(class); c != nil { + var b strings.Builder + fmt.Fprintf(&b, "WARNING: circular locking detected: %s -> %s:\n%s\n", + chain[0], class, log.LocalStack(3)) + + fmt.Fprintf(&b, "known lock chain: ") + c := class + for i := len(chain) - 1; i >= 0; i-- { + fmt.Fprintf(&b, "%s -> ", c) + c = chain[i] + } + fmt.Fprintf(&b, "%s\n", chain[0]) + c = class + for i := len(chain) - 1; i >= 0; i-- { + fmt.Fprintf(&b, "\n====== %s -> %s =====\n%s", + c, chain[i], *chain[i].ancestors.Load(c)) + c = chain[i] + } + panic(b.String()) + } + prevClass.ancestors.RangeRepeatable(func(parentClass *MutexClass, stacks *string) bool { + // The recursion is fine here. If it fails, you need to reduce + // a number of nested locks. + checkLock(class, parentClass, chain) + return true + }) +} + +// AddGLock records a lock to the current goroutine and updates dependencies. +func AddGLock(class *MutexClass, lockNameIndex int) { + gid := goid.Get() + + if lockNameIndex != -1 { + class = class.nestedLockClasses[lockNameIndex] + } + currentLocks := routineLocks.Load(gid) + if currentLocks == nil { + locks := goroutineLocks(make(map[*MutexClass]bool)) + locks[class] = true + routineLocks.Store(gid, &locks) + return + } + + if (*currentLocks)[class] { + panic(fmt.Sprintf("nested locking: %s:\n%s", class, log.LocalStack(2))) + } + (*currentLocks)[class] = true + // Check dependencies and add locked mutexes to the ancestors list. + for prevClass := range *currentLocks { + if prevClass == class { + continue + } + checkLock(class, prevClass, nil) + + if c := class.ancestors.Load(prevClass); c == nil { + stacks := string(log.LocalStack(2)) + class.ancestors.Store(prevClass, &stacks) + } + } +} + +// DelGLock deletes a lock from the current goroutine. +func DelGLock(class *MutexClass, lockNameIndex int) { + if lockNameIndex != -1 { + class = class.nestedLockClasses[lockNameIndex] + } + gid := goid.Get() + currentLocks := routineLocks.Load(gid) + if currentLocks == nil { + panic("the current goroutine doesn't have locks") + } + if _, ok := (*currentLocks)[class]; !ok { + var b strings.Builder + fmt.Fprintf(&b, "Lock not held: %s:\n", class) + fmt.Fprintf(&b, "Current stack:\n%s\n", string(log.LocalStack(2))) + fmt.Fprintf(&b, "Current locks:\n") + for c := range *currentLocks { + heldToClass := class.ancestors.Load(c) + classToHeld := c.ancestors.Load(class) + if heldToClass == nil && classToHeld == nil { + fmt.Fprintf(&b, "\t- Holding lock: %s (no dependency to/from %s found)\n", c, class) + } else if heldToClass != nil && classToHeld != nil { + fmt.Fprintf(&b, "\t- Holding lock: %s (mutual dependency with %s found, this should never happen)\n", c, class) + } else if heldToClass != nil && classToHeld == nil { + fmt.Fprintf(&b, "\t- Holding lock: %s (dependency: %s -> %s)\n", c, c, class) + fmt.Fprintf(&b, "%s\n\n", *heldToClass) + } else if heldToClass == nil && classToHeld != nil { + fmt.Fprintf(&b, "\t- Holding lock: %s (dependency: %s -> %s)\n", c, class, c) + fmt.Fprintf(&b, "%s\n\n", *classToHeld) + } + } + fmt.Fprintf(&b, "** End of locks held **\n") + panic(b.String()) + } + + delete(*currentLocks, class) + if len(*currentLocks) == 0 { + routineLocks.Store(gid, nil) + } +} diff --git a/pkg/sync/locking/lockdep_norace.go b/pkg/sync/locking/lockdep_norace.go new file mode 100644 index 0000000..379dc9e --- /dev/null +++ b/pkg/sync/locking/lockdep_norace.go @@ -0,0 +1,42 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !lockdep +// +build !lockdep + +package locking + +import ( + "reflect" +) + +type goroutineLocks map[*MutexClass]bool + +// MutexClass is a stub class without the lockdep tag. +type MutexClass struct{} + +// NewMutexClass is no-op without the lockdep tag. +func NewMutexClass(reflect.Type, []string) *MutexClass { + return nil +} + +// AddGLock is no-op without the lockdep tag. +// +//go:inline +func AddGLock(*MutexClass, int) {} + +// DelGLock is no-op without the lockdep tag. +// +//go:inline +func DelGLock(*MutexClass, int) {} diff --git a/pkg/sync/locking/locking.go b/pkg/sync/locking/locking.go new file mode 100644 index 0000000..1b99bc3 --- /dev/null +++ b/pkg/sync/locking/locking.go @@ -0,0 +1,28 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package locking implements lock primitives with the correctness validator. +// +// All mutexes are divided on classes and the validator check following conditions: +// - Mutexes of the same class are not taken more than once except cases when +// that is expected. +// - Mutexes are never locked in a reverse order. Lock dependencies are tracked +// on the class level. +// +// The validator is implemented in a very straightforward way. For each mutex +// class, we maintain the ancestors list of all classes that have ever been +// taken before the target one. For each goroutine, we have the list of +// currently locked mutexes. And finally, all lock methods check that +// ancestors of currently locked mutexes don't contain the target one. +package locking diff --git a/pkg/sync/mutex_unsafe.go b/pkg/sync/mutex_unsafe.go new file mode 100644 index 0000000..49f0af4 --- /dev/null +++ b/pkg/sync/mutex_unsafe.go @@ -0,0 +1,80 @@ +// Copyright 2019 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package sync + +import ( + "sync" + "unsafe" +) + +// CrossGoroutineMutex is equivalent to Mutex, but it need not be unlocked by a +// the same goroutine that locked the mutex. +type CrossGoroutineMutex struct { + m sync.Mutex +} + +// Lock locks the underlying Mutex. +// +checklocksignore +func (m *CrossGoroutineMutex) Lock() { + m.m.Lock() +} + +// Unlock unlocks the underlying Mutex. +// +checklocksignore +func (m *CrossGoroutineMutex) Unlock() { + m.m.Unlock() +} + +// TryLock tries to acquire the mutex. It returns true if it succeeds and false +// otherwise. TryLock does not block. +func (m *CrossGoroutineMutex) TryLock() bool { + return m.m.TryLock() +} + +// Mutex is a mutual exclusion lock. The zero value for a Mutex is an unlocked +// mutex. +// +// A Mutex must not be copied after first use. +// +// A Mutex must be unlocked by the same goroutine that locked it. This +// invariant is enforced with the 'checklocks' build tag. +type Mutex struct { + m CrossGoroutineMutex +} + +// Lock locks m. If the lock is already in use, the calling goroutine blocks +// until the mutex is available. +// +checklocksignore +func (m *Mutex) Lock() { + noteLock(unsafe.Pointer(m)) + m.m.Lock() +} + +// Unlock unlocks m. +// +// Preconditions: +// - m is locked. +// - m was locked by this goroutine. +// +// +checklocksignore +func (m *Mutex) Unlock() { + noteUnlock(unsafe.Pointer(m)) + m.m.Unlock() +} + +// TryLock tries to acquire the mutex. It returns true if it succeeds and false +// otherwise. TryLock does not block. +// +checklocksignore +func (m *Mutex) TryLock() bool { + // Note lock first to enforce proper locking even if unsuccessful. + noteLock(unsafe.Pointer(m)) + locked := m.m.TryLock() + if !locked { + noteUnlock(unsafe.Pointer(m)) + } + return locked +} diff --git a/pkg/sync/nocopy.go b/pkg/sync/nocopy.go new file mode 100644 index 0000000..722b295 --- /dev/null +++ b/pkg/sync/nocopy.go @@ -0,0 +1,28 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +// NoCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://golang.org/issues/8005#issuecomment-190753527 +// for details. +type NoCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*NoCopy) Lock() {} + +// Unlock is a no-op used by -copylocks checker from `go vet`. +func (*NoCopy) Unlock() {} diff --git a/pkg/sync/norace_unsafe.go b/pkg/sync/norace_unsafe.go new file mode 100644 index 0000000..e9403ae --- /dev/null +++ b/pkg/sync/norace_unsafe.go @@ -0,0 +1,48 @@ +// Copyright 2019 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build !race +// +build !race + +package sync + +import ( + "sync/atomic" + "unsafe" +) + +// RaceEnabled is true if the Go data race detector is enabled. +const RaceEnabled = false + +// RaceDisable has the same semantics as runtime.RaceDisable. +func RaceDisable() { +} + +// RaceEnable has the same semantics as runtime.RaceEnable. +func RaceEnable() { +} + +// RaceAcquire has the same semantics as runtime.RaceAcquire. +func RaceAcquire(addr unsafe.Pointer) { +} + +// RaceRelease has the same semantics as runtime.RaceRelease. +func RaceRelease(addr unsafe.Pointer) { +} + +// RaceReleaseMerge has the same semantics as runtime.RaceReleaseMerge. +func RaceReleaseMerge(addr unsafe.Pointer) { +} + +// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to +// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector. +// This is necessary when implementing gopark callbacks, since no race context +// is available during their execution. +func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool { + // Use atomic.CompareAndSwapUintptr outside of race builds for + // inlinability. + return atomic.CompareAndSwapUintptr(ptr, old, new) +} diff --git a/pkg/sync/race_amd64.s b/pkg/sync/race_amd64.s new file mode 100644 index 0000000..c994814 --- /dev/null +++ b/pkg/sync/race_amd64.s @@ -0,0 +1,33 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build race && amd64 +// +build race,amd64 + +#include "textflag.h" + +// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool +TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT|NOFRAME,$0-25 + MOVQ ptr+0(FP), DI + MOVQ old+8(FP), AX + MOVQ new+16(FP), SI + + LOCK + CMPXCHGQ SI, 0(DI) + + SETEQ AX + MOVB AX, ret+24(FP) + + RET + diff --git a/pkg/sync/race_arm64.s b/pkg/sync/race_arm64.s new file mode 100644 index 0000000..c4192e8 --- /dev/null +++ b/pkg/sync/race_arm64.s @@ -0,0 +1,35 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build race && arm64 +// +build race,arm64 + +#include "textflag.h" + +// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool +TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25 + MOVD ptr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R1 +again: + LDAXR (R0), R3 + CMP R1, R3 + BNE ok + STLXR R2, (R0), R3 + CBNZ R3, again +ok: + CSET EQ, R0 + MOVB R0, ret+24(FP) + RET + diff --git a/pkg/sync/race_unsafe.go b/pkg/sync/race_unsafe.go new file mode 100644 index 0000000..eb66ce1 --- /dev/null +++ b/pkg/sync/race_unsafe.go @@ -0,0 +1,49 @@ +// Copyright 2019 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build race +// +build race + +package sync + +import ( + "runtime" + "unsafe" +) + +// RaceEnabled is true if the Go data race detector is enabled. +const RaceEnabled = true + +// RaceDisable has the same semantics as runtime.RaceDisable. +func RaceDisable() { + runtime.RaceDisable() +} + +// RaceEnable has the same semantics as runtime.RaceEnable. +func RaceEnable() { + runtime.RaceEnable() +} + +// RaceAcquire has the same semantics as runtime.RaceAcquire. +func RaceAcquire(addr unsafe.Pointer) { + runtime.RaceAcquire(addr) +} + +// RaceRelease has the same semantics as runtime.RaceRelease. +func RaceRelease(addr unsafe.Pointer) { + runtime.RaceRelease(addr) +} + +// RaceReleaseMerge has the same semantics as runtime.RaceReleaseMerge. +func RaceReleaseMerge(addr unsafe.Pointer) { + runtime.RaceReleaseMerge(addr) +} + +// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to +// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector. +// This is necessary when implementing gopark callbacks, since no race context +// is available during their execution. +func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool diff --git a/pkg/sync/runtime.go b/pkg/sync/runtime.go new file mode 100644 index 0000000..e4604e8 --- /dev/null +++ b/pkg/sync/runtime.go @@ -0,0 +1,22 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "runtime" +) + +// Dummy reference for facts. +const _ = runtime.Compiler diff --git a/pkg/sync/runtime_amd64.go b/pkg/sync/runtime_amd64.go new file mode 100644 index 0000000..cdca5ca --- /dev/null +++ b/pkg/sync/runtime_amd64.go @@ -0,0 +1,31 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build amd64 + +package sync + +import ( + "sync/atomic" +) + +const supportsWakeSuppression = true + +// addrOfSpinning returns the address of runtime.sched.nmspinning. +func addrOfSpinning() *int32 + +// nmspinning caches addrOfSpinning. +var nmspinning = addrOfSpinning() + +//go:nosplit +func preGoReadyWakeSuppression() { + atomic.AddInt32(nmspinning, 1) +} + +//go:nosplit +func postGoReadyWakeSuppression() { + atomic.AddInt32(nmspinning, -1) +} diff --git a/pkg/sync/runtime_constants_go125.go b/pkg/sync/runtime_constants_go125.go new file mode 100644 index 0000000..1dcd70c --- /dev/null +++ b/pkg/sync/runtime_constants_go125.go @@ -0,0 +1,25 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// https://go.dev/cl/670497 (1.25) adds a new wait reason, adjusting the value of waitReasonSemacquire. +//go:build go1.25 + +package sync + +// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. +const ( + WaitReasonSelect uint8 = 9 // +checkconst runtime waitReasonSelect + WaitReasonChanReceive uint8 = 14 // +checkconst runtime waitReasonChanReceive + WaitReasonSemacquire uint8 = 19 // +checkconst runtime waitReasonSemacquire +) diff --git a/pkg/sync/runtime_constants_not_go125.go b/pkg/sync/runtime_constants_not_go125.go new file mode 100644 index 0000000..b27a899 --- /dev/null +++ b/pkg/sync/runtime_constants_not_go125.go @@ -0,0 +1,25 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// https://go.dev/cl/670497 (1.25) adds a new wait reason, adjusting the value of waitReasonSemacquire. +//go:build !go1.25 + +package sync + +// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. +const ( + WaitReasonSelect uint8 = 9 // +checkconst runtime waitReasonSelect + WaitReasonChanReceive uint8 = 14 // +checkconst runtime waitReasonChanReceive + WaitReasonSemacquire uint8 = 18 // +checkconst runtime waitReasonSemacquire +) diff --git a/pkg/sync/runtime_exectracer2.go b/pkg/sync/runtime_exectracer2.go new file mode 100644 index 0000000..58630af --- /dev/null +++ b/pkg/sync/runtime_exectracer2.go @@ -0,0 +1,21 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +// TraceBlockReason constants, from Go's src/runtime/trace2runtime.go. +const ( + TraceBlockSelect TraceBlockReason = 3 // +checkconst runtime traceBlockSelect + TraceBlockSync TraceBlockReason = 5 // +checkconst runtime traceBlockSync +) diff --git a/pkg/sync/runtime_go121_unsafe.go b/pkg/sync/runtime_go121_unsafe.go new file mode 100644 index 0000000..c9f2c76 --- /dev/null +++ b/pkg/sync/runtime_go121_unsafe.go @@ -0,0 +1,17 @@ +// Copyright 2023 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build go1.21 && !go1.24 + +package sync + +import ( + "unsafe" +) + +// Use checkoffset to assert that maptype.hasher (the only field we use) has +// the correct offset. +const maptypeHasherOffset = unsafe.Offsetof(maptype{}.Hasher) // +checkoffset internal/abi MapType.Hasher diff --git a/pkg/sync/runtime_go124_unsafe.go b/pkg/sync/runtime_go124_unsafe.go new file mode 100644 index 0000000..96eef8c --- /dev/null +++ b/pkg/sync/runtime_go124_unsafe.go @@ -0,0 +1,16 @@ +// Copyright 2024 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// https://go.dev/cl/691596 (1.26) renames the internal map type which nogo relies on. +//go:build go1.24 && !go1.26 + +package sync + +import "unsafe" + +// Use checkoffset to assert that maptype.hasher (the only field we use) has +// the correct offset. +const maptypeHasherOffset = unsafe.Offsetof(maptype{}.Hasher) // +checkoffset internal/abi SwissMapType.Hasher diff --git a/pkg/sync/runtime_go126_unsafe.go b/pkg/sync/runtime_go126_unsafe.go new file mode 100644 index 0000000..4bfcb47 --- /dev/null +++ b/pkg/sync/runtime_go126_unsafe.go @@ -0,0 +1,16 @@ +// Copyright 2024 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// https://go.dev/cl/691596 (1.26) renames the internal map type which nogo relies on. +//go:build go1.26 + +package sync + +import "unsafe" + +// Use checkoffset to assert that maptype.hasher (the only field we use) has +// the correct offset. +const maptypeHasherOffset = unsafe.Offsetof(maptype{}.Hasher) // +checkoffset internal/abi MapType.Hasher diff --git a/pkg/sync/runtime_not_go121_unsafe.go b/pkg/sync/runtime_not_go121_unsafe.go new file mode 100644 index 0000000..74f6d32 --- /dev/null +++ b/pkg/sync/runtime_not_go121_unsafe.go @@ -0,0 +1,19 @@ +// Copyright 2023 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// runtime.maptype is moved to internal/abi.MapType in Go 1.21. +// +//go:build !go1.21 + +package sync + +import ( + "unsafe" +) + +// Use checkoffset to assert that maptype.hasher (the only field we use) has +// the correct offset. +const maptypeHasherOffset = unsafe.Offsetof(maptype{}.Hasher) // +checkoffset runtime maptype.hasher diff --git a/pkg/sync/runtime_other.go b/pkg/sync/runtime_other.go new file mode 100644 index 0000000..705b1f2 --- /dev/null +++ b/pkg/sync/runtime_other.go @@ -0,0 +1,15 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +//go:build !amd64 +// +build !amd64 + +package sync + +const supportsWakeSuppression = false + +func preGoReadyWakeSuppression() {} // Never called. +func postGoReadyWakeSuppression() {} // Never called. diff --git a/pkg/sync/runtime_spinning_amd64.s b/pkg/sync/runtime_spinning_amd64.s new file mode 100644 index 0000000..b238c8d --- /dev/null +++ b/pkg/sync/runtime_spinning_amd64.s @@ -0,0 +1,26 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// https://go.dev/cl/669235 (1.25) adds a new schedt field prior to nmspinning. +//go:build amd64 && !go1.25 + +#include "textflag.h" + +#define NMSPINNING_OFFSET 92 // +checkoffset runtime schedt.nmspinning + +TEXT ·addrOfSpinning(SB),NOSPLIT|NOFRAME,$0-8 + LEAQ runtime·sched(SB), AX + ADDQ $NMSPINNING_OFFSET, AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/sync/runtime_spinning_go125_amd64.s b/pkg/sync/runtime_spinning_go125_amd64.s new file mode 100644 index 0000000..46b2f06 --- /dev/null +++ b/pkg/sync/runtime_spinning_go125_amd64.s @@ -0,0 +1,26 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// https://go.dev/cl/669235 (1.25) adds a new schedt field prior to nmspinning. +//go:build amd64 && go1.25 + +#include "textflag.h" + +#define NMSPINNING_OFFSET 100 // +checkoffset runtime schedt.nmspinning + +TEXT ·addrOfSpinning(SB),NOSPLIT|NOFRAME,$0-8 + LEAQ runtime·sched(SB), AX + ADDQ $NMSPINNING_OFFSET, AX + MOVQ AX, ret+0(FP) + RET diff --git a/pkg/sync/runtime_spinning_other.s b/pkg/sync/runtime_spinning_other.s new file mode 100644 index 0000000..b6391d2 --- /dev/null +++ b/pkg/sync/runtime_spinning_other.s @@ -0,0 +1,18 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !amd64 + +// This file is intentionally left blank. Other arches don't use +// addrOfSpinning, but we still need an input to the nogo template rule. diff --git a/pkg/sync/runtime_unsafe.go b/pkg/sync/runtime_unsafe.go new file mode 100644 index 0000000..acaf173 --- /dev/null +++ b/pkg/sync/runtime_unsafe.go @@ -0,0 +1,141 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// //go:linkname directives type-checked by checklinkname. +// Runtime type copies checked by checkoffset. + +package sync + +import ( + "fmt" + "reflect" + "unsafe" +) + +// Goyield is runtime.goyield, which is similar to runtime.Gosched but only +// yields the processor to other goroutines already on the processor's +// runqueue. +// +//go:nosplit +func Goyield() { + goyield() +} + +// Gopark is runtime.gopark. Gopark calls unlockf(pointer to runtime.g, lock); +// if unlockf returns true, Gopark blocks until Goready(pointer to runtime.g) +// is called. unlockf and its callees must be nosplit and norace, since stack +// splitting and race context are not available where it is called. +// +//go:nosplit +func Gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceReason TraceBlockReason, traceskip int) { + gopark(unlockf, lock, reason, traceReason, traceskip) +} + +//go:linkname gopark runtime.gopark +func gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceReason TraceBlockReason, traceskip int) + +// TraceBlockReason is equivalent to runtime.traceBlockReason. +type TraceBlockReason uint8 + +//go:linkname wakep runtime.wakep +func wakep() + +// Wakep is runtime.wakep. +// +//go:nosplit +func Wakep() { + // This is only supported if we can suppress the wakep called + // from Goready below, which is in certain architectures only. + if supportsWakeSuppression { + wakep() + } +} + +//go:linkname goready runtime.goready +func goready(gp uintptr, traceskip int) + +// Goready is runtime.goready. +// +// The additional wakep argument controls whether a new thread will be kicked to +// execute the P. This should be true in most circumstances. However, if the +// current thread is about to sleep, then this can be false for efficiency. +// +//go:nosplit +func Goready(gp uintptr, traceskip int, wakep bool) { + if supportsWakeSuppression && !wakep { + preGoReadyWakeSuppression() + } + goready(gp, traceskip) + if supportsWakeSuppression && !wakep { + postGoReadyWakeSuppression() + } +} + +// Rand32 returns a non-cryptographically-secure random uint32. +func Rand32() uint32 { + return fastrand() +} + +// Rand64 returns a non-cryptographically-secure random uint64. +func Rand64() uint64 { + return uint64(fastrand())<<32 | uint64(fastrand()) +} + +//go:linkname fastrand runtime.fastrand +func fastrand() uint32 + +// RandUintptr returns a non-cryptographically-secure random uintptr. +func RandUintptr() uintptr { + if unsafe.Sizeof(uintptr(0)) == 4 { + return uintptr(Rand32()) + } + return uintptr(Rand64()) +} + +// MapKeyHasher returns a hash function for pointers of m's key type. +// +// Preconditions: m must be a map. +func MapKeyHasher(m any) func(unsafe.Pointer, uintptr) uintptr { + if rtyp := reflect.TypeOf(m); rtyp.Kind() != reflect.Map { + panic(fmt.Sprintf("sync.MapKeyHasher: m is %v, not map", rtyp)) + } + mtyp := *(**maptype)(unsafe.Pointer(&m)) + return mtyp.Hasher +} + +// maptype is equivalent to the beginning of internal/abi.MapType. +type maptype struct { + size uintptr + ptrdata uintptr + hash uint32 + tflag uint8 + align uint8 + fieldAlign uint8 + kind uint8 + equal func(unsafe.Pointer, unsafe.Pointer) bool + gcdata *byte + str int32 + ptrToThis int32 + key unsafe.Pointer + elem unsafe.Pointer + bucket unsafe.Pointer + Hasher func(unsafe.Pointer, uintptr) uintptr + // more fields +} + +// These functions are only used within the sync package. + +//go:linkname semacquire sync.runtime_Semacquire +func semacquire(addr *uint32) + +//go:linkname semrelease sync.runtime_Semrelease +func semrelease(addr *uint32, handoff bool, skipframes int) + +//go:linkname canSpin sync.runtime_canSpin +func canSpin(i int) bool + +//go:linkname doSpin sync.runtime_doSpin +func doSpin() diff --git a/pkg/sync/rwmutex_unsafe.go b/pkg/sync/rwmutex_unsafe.go new file mode 100644 index 0000000..921c65b --- /dev/null +++ b/pkg/sync/rwmutex_unsafe.go @@ -0,0 +1,316 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2019 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// This is mostly copied from the standard library's sync/rwmutex.go. +// +// Happens-before relationships indicated to the race detector: +// - Unlock -> Lock (via writerSem) +// - Unlock -> RLock (via readerSem) +// - RUnlock -> Lock (via writerSem) +// - DowngradeLock -> RLock (via readerSem) + +package sync + +import ( + "sync/atomic" + "unsafe" +) + +// CrossGoroutineRWMutex is equivalent to RWMutex, but it need not be unlocked +// by a the same goroutine that locked the mutex. +type CrossGoroutineRWMutex struct { + // w is held if there are pending writers + // + // We use CrossGoroutineMutex rather than Mutex because the lock + // annotation instrumentation in Mutex will trigger false positives in + // the race detector when called inside of RaceDisable. + w CrossGoroutineMutex + writerSem uint32 // semaphore for writers to wait for completing readers + readerSem uint32 // semaphore for readers to wait for completing writers + readerCount int32 // number of pending readers + readerWait int32 // number of departing readers +} + +const rwmutexMaxReaders = 1 << 30 + +// TryRLock locks rw for reading. It returns true if it succeeds and false +// otherwise. It does not block. +// +checklocksignore +func (rw *CrossGoroutineRWMutex) TryRLock() bool { + if RaceEnabled { + RaceDisable() + } + for { + rc := atomic.LoadInt32(&rw.readerCount) + if rc < 0 { + if RaceEnabled { + RaceEnable() + } + return false + } + if !atomic.CompareAndSwapInt32(&rw.readerCount, rc, rc+1) { + continue + } + if RaceEnabled { + RaceEnable() + RaceAcquire(unsafe.Pointer(&rw.readerSem)) + } + return true + } +} + +// RLock locks rw for reading. +// +// It should not be used for recursive read locking; a blocked Lock call +// excludes new readers from acquiring the lock. See the documentation on the +// RWMutex type. +// +checklocksignore +func (rw *CrossGoroutineRWMutex) RLock() { + if RaceEnabled { + RaceDisable() + } + if atomic.AddInt32(&rw.readerCount, 1) < 0 { + // A writer is pending, wait for it. + semacquire(&rw.readerSem) + } + if RaceEnabled { + RaceEnable() + RaceAcquire(unsafe.Pointer(&rw.readerSem)) + } +} + +// RUnlock undoes a single RLock call. +// +// Preconditions: +// - rw is locked for reading. +// +// +checklocksignore +func (rw *CrossGoroutineRWMutex) RUnlock() { + if RaceEnabled { + RaceReleaseMerge(unsafe.Pointer(&rw.writerSem)) + RaceDisable() + } + if r := atomic.AddInt32(&rw.readerCount, -1); r < 0 { + if r+1 == 0 || r+1 == -rwmutexMaxReaders { + panic("RUnlock of unlocked RWMutex") + } + // A writer is pending. + if atomic.AddInt32(&rw.readerWait, -1) == 0 { + // The last reader unblocks the writer. + semrelease(&rw.writerSem, false, 0) + } + } + if RaceEnabled { + RaceEnable() + } +} + +// TryLock locks rw for writing. It returns true if it succeeds and false +// otherwise. It does not block. +// +checklocksignore +func (rw *CrossGoroutineRWMutex) TryLock() bool { + if RaceEnabled { + RaceDisable() + } + // First, resolve competition with other writers. + if !rw.w.TryLock() { + if RaceEnabled { + RaceEnable() + } + return false + } + // Only proceed if there are no readers. + if !atomic.CompareAndSwapInt32(&rw.readerCount, 0, -rwmutexMaxReaders) { + rw.w.Unlock() + if RaceEnabled { + RaceEnable() + } + return false + } + if RaceEnabled { + RaceEnable() + RaceAcquire(unsafe.Pointer(&rw.writerSem)) + } + return true +} + +// Lock locks rw for writing. If the lock is already locked for reading or +// writing, Lock blocks until the lock is available. +// +checklocksignore +func (rw *CrossGoroutineRWMutex) Lock() { + if RaceEnabled { + RaceDisable() + } + // First, resolve competition with other writers. + rw.w.Lock() + // Announce to readers there is a pending writer. + r := atomic.AddInt32(&rw.readerCount, -rwmutexMaxReaders) + rwmutexMaxReaders + // Wait for active readers. + if r != 0 && atomic.AddInt32(&rw.readerWait, r) != 0 { + semacquire(&rw.writerSem) + } + if RaceEnabled { + RaceEnable() + RaceAcquire(unsafe.Pointer(&rw.writerSem)) + } +} + +// Unlock unlocks rw for writing. +// +// Preconditions: +// - rw is locked for writing. +// +// +checklocksignore +func (rw *CrossGoroutineRWMutex) Unlock() { + if RaceEnabled { + RaceRelease(unsafe.Pointer(&rw.writerSem)) + RaceRelease(unsafe.Pointer(&rw.readerSem)) + RaceDisable() + } + // Announce to readers there is no active writer. + r := atomic.AddInt32(&rw.readerCount, rwmutexMaxReaders) + if r >= rwmutexMaxReaders { + panic("Unlock of unlocked RWMutex") + } + // Unblock blocked readers, if any. + for i := 0; i < int(r); i++ { + semrelease(&rw.readerSem, false, 0) + } + // Allow other writers to proceed. + rw.w.Unlock() + if RaceEnabled { + RaceEnable() + } +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +// Preconditions: +// - rw is locked for writing. +// +// +checklocksignore +func (rw *CrossGoroutineRWMutex) DowngradeLock() { + if RaceEnabled { + RaceRelease(unsafe.Pointer(&rw.readerSem)) + RaceDisable() + } + // Announce to readers there is no active writer and one additional reader. + r := atomic.AddInt32(&rw.readerCount, rwmutexMaxReaders+1) + if r >= rwmutexMaxReaders+1 { + panic("DowngradeLock of unlocked RWMutex") + } + // Unblock blocked readers, if any. Note that this loop starts as 1 since r + // includes this goroutine. + for i := 1; i < int(r); i++ { + semrelease(&rw.readerSem, false, 0) + } + // Allow other writers to proceed to rw.w.Lock(). Note that they will still + // block on rw.writerSem since at least this reader exists, such that + // DowngradeLock() is atomic with the previous write lock. + rw.w.Unlock() + if RaceEnabled { + RaceEnable() + } +} + +// A RWMutex is a reader/writer mutual exclusion lock. The lock can be held by +// an arbitrary number of readers or a single writer. The zero value for a +// RWMutex is an unlocked mutex. +// +// A RWMutex must not be copied after first use. +// +// If a goroutine holds a RWMutex for reading and another goroutine might call +// Lock, no goroutine should expect to be able to acquire a read lock until the +// initial read lock is released. In particular, this prohibits recursive read +// locking. This is to ensure that the lock eventually becomes available; a +// blocked Lock call excludes new readers from acquiring the lock. +// +// A Mutex must be unlocked by the same goroutine that locked it. This +// invariant is enforced with the 'checklocks' build tag. +type RWMutex struct { + m CrossGoroutineRWMutex +} + +// TryRLock locks rw for reading. It returns true if it succeeds and false +// otherwise. It does not block. +// +checklocksignore +func (rw *RWMutex) TryRLock() bool { + // Note lock first to enforce proper locking even if unsuccessful. + noteLock(unsafe.Pointer(rw)) + locked := rw.m.TryRLock() + if !locked { + noteUnlock(unsafe.Pointer(rw)) + } + return locked +} + +// RLock locks rw for reading. +// +// It should not be used for recursive read locking; a blocked Lock call +// excludes new readers from acquiring the lock. See the documentation on the +// RWMutex type. +// +checklocksignore +func (rw *RWMutex) RLock() { + noteLock(unsafe.Pointer(rw)) + rw.m.RLock() +} + +// RUnlock undoes a single RLock call. +// +// Preconditions: +// - rw is locked for reading. +// - rw was locked by this goroutine. +// +// +checklocksignore +func (rw *RWMutex) RUnlock() { + rw.m.RUnlock() + noteUnlock(unsafe.Pointer(rw)) +} + +// TryLock locks rw for writing. It returns true if it succeeds and false +// otherwise. It does not block. +// +checklocksignore +func (rw *RWMutex) TryLock() bool { + // Note lock first to enforce proper locking even if unsuccessful. + noteLock(unsafe.Pointer(rw)) + locked := rw.m.TryLock() + if !locked { + noteUnlock(unsafe.Pointer(rw)) + } + return locked +} + +// Lock locks rw for writing. If the lock is already locked for reading or +// writing, Lock blocks until the lock is available. +// +checklocksignore +func (rw *RWMutex) Lock() { + noteLock(unsafe.Pointer(rw)) + rw.m.Lock() +} + +// Unlock unlocks rw for writing. +// +// Preconditions: +// - rw is locked for writing. +// - rw was locked by this goroutine. +// +// +checklocksignore +func (rw *RWMutex) Unlock() { + rw.m.Unlock() + noteUnlock(unsafe.Pointer(rw)) +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +// Preconditions: +// - rw is locked for writing. +// +// +checklocksignore +func (rw *RWMutex) DowngradeLock() { + // No note change for DowngradeLock. + rw.m.DowngradeLock() +} diff --git a/pkg/sync/seqcount.go b/pkg/sync/seqcount.go new file mode 100644 index 0000000..71fa4c3 --- /dev/null +++ b/pkg/sync/seqcount.go @@ -0,0 +1,120 @@ +// Copyright 2019 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package sync + +import ( + "sync/atomic" +) + +// SeqCount is a synchronization primitive for optimistic reader/writer +// synchronization in cases where readers can work with stale data and +// therefore do not need to block writers. +// +// Compared to sync/atomic.Value: +// +// - Mutation of SeqCount-protected data does not require memory allocation, +// whereas atomic.Value generally does. This is a significant advantage when +// writes are common. +// +// - Atomic reads of SeqCount-protected data require copying. This is a +// disadvantage when atomic reads are common. +// +// - SeqCount may be more flexible: correct use of SeqCount.ReadOk allows other +// operations to be made atomic with reads of SeqCount-protected data. +// +// - SeqCount is more cumbersome to use; atomic reads of SeqCount-protected +// data require instantiating function templates using go_generics (see +// seqatomic.go). +type SeqCount struct { + // epoch is incremented by BeginWrite and EndWrite, such that epoch is odd + // if a writer critical section is active, and a read from data protected + // by this SeqCount is atomic iff epoch is the same even value before and + // after the read. + epoch uint32 +} + +// SeqCountEpoch tracks writer critical sections in a SeqCount. +type SeqCountEpoch uint32 + +// BeginRead indicates the beginning of a reader critical section. Reader +// critical sections DO NOT BLOCK writer critical sections, so operations in a +// reader critical section MAY RACE with writer critical sections. Races are +// detected by ReadOk at the end of the reader critical section. Thus, the +// low-level structure of readers is generally: +// +// for { +// epoch := seq.BeginRead() +// // do something idempotent with seq-protected data +// if seq.ReadOk(epoch) { +// break +// } +// } +// +// However, since reader critical sections may race with writer critical +// sections, the Go race detector will (accurately) flag data races in readers +// using this pattern. Most users of SeqCount will need to use the +// SeqAtomicLoad function template in seqatomic.go. +func (s *SeqCount) BeginRead() SeqCountEpoch { + if epoch := atomic.LoadUint32(&s.epoch); epoch&1 == 0 { + return SeqCountEpoch(epoch) + } + return s.beginReadSlow() +} + +func (s *SeqCount) beginReadSlow() SeqCountEpoch { + i := 0 + for { + if canSpin(i) { + i++ + doSpin() + } else { + goyield() + } + if epoch := atomic.LoadUint32(&s.epoch); epoch&1 == 0 { + return SeqCountEpoch(epoch) + } + } +} + +// ReadOk returns true if the reader critical section initiated by a previous +// call to BeginRead() that returned epoch did not race with any writer critical +// sections. +// +// ReadOk may be called any number of times during a reader critical section. +// Reader critical sections do not need to be explicitly terminated; the last +// call to ReadOk is implicitly the end of the reader critical section. +func (s *SeqCount) ReadOk(epoch SeqCountEpoch) bool { + MemoryFenceReads() + return atomic.LoadUint32(&s.epoch) == uint32(epoch) +} + +// BeginWrite indicates the beginning of a writer critical section. +// +// SeqCount does not support concurrent writer critical sections; clients with +// concurrent writers must synchronize them using e.g. sync.Mutex. +func (s *SeqCount) BeginWrite() { + if epoch := atomic.AddUint32(&s.epoch, 1); epoch&1 == 0 { + panic("SeqCount.BeginWrite during writer critical section") + } +} + +// BeginWriteOk combines the semantics of ReadOk and BeginWrite. If the reader +// critical section initiated by a previous call to BeginRead() that returned +// epoch did not race with any writer critical sections, it begins a writer +// critical section and returns true. Otherwise it does nothing and returns +// false. +func (s *SeqCount) BeginWriteOk(epoch SeqCountEpoch) bool { + return atomic.CompareAndSwapUint32(&s.epoch, uint32(epoch), uint32(epoch)+1) +} + +// EndWrite ends the effect of a preceding BeginWrite or successful +// BeginWriteOk. +func (s *SeqCount) EndWrite() { + if epoch := atomic.AddUint32(&s.epoch, 1); epoch&1 != 0 { + panic("SeqCount.EndWrite outside writer critical section") + } +} diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go new file mode 100644 index 0000000..c78b1e7 --- /dev/null +++ b/pkg/sync/sync.go @@ -0,0 +1,10 @@ +// Copyright 2019 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// Package sync provides synchronization primitives. +// +// +checkalignedignore +package sync diff --git a/pkg/syncevent/broadcaster.go b/pkg/syncevent/broadcaster.go new file mode 100644 index 0000000..57f9018 --- /dev/null +++ b/pkg/syncevent/broadcaster.go @@ -0,0 +1,220 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syncevent + +import ( + "github.com/sagernet/gvisor/pkg/sync" +) + +// Broadcaster is an implementation of Source that supports any number of +// subscribed Receivers. +// +// The zero value of Broadcaster is valid and has no subscribed Receivers. +// Broadcaster is not copyable by value. +// +// All Broadcaster methods may be called concurrently from multiple goroutines. +type Broadcaster struct { + // Broadcaster is implemented as a hash table where keys are assigned by + // the Broadcaster and returned as SubscriptionIDs, making it safe to use + // the identity function for hashing. The hash table resolves collisions + // using linear probing and features Robin Hood insertion and backward + // shift deletion in order to support a relatively high load factor + // efficiently, which matters since the cost of Broadcast is linear in the + // size of the table. + + // mu protects the following fields. + mu sync.Mutex + + // Invariants: len(table) is 0 or a power of 2. + table []broadcasterSlot + + // load is the number of entries in table with receiver != nil. + load int + + lastID SubscriptionID +} + +type broadcasterSlot struct { + // Invariants: If receiver == nil, then filter == NoEvents and id == 0. + // Otherwise, id != 0. + receiver *Receiver + filter Set + id SubscriptionID +} + +const ( + broadcasterMinNonZeroTableSize = 2 // must be a power of 2 > 1 + + broadcasterMaxLoadNum = 13 + broadcasterMaxLoadDen = 16 +) + +// SubscribeEvents implements Source.SubscribeEvents. +func (b *Broadcaster) SubscribeEvents(r *Receiver, filter Set) SubscriptionID { + b.mu.Lock() + + // Assign an ID for this subscription. + b.lastID++ + id := b.lastID + + // Expand the table if over the maximum load factor: + // + // load / len(b.table) > broadcasterMaxLoadNum / broadcasterMaxLoadDen + // load * broadcasterMaxLoadDen > broadcasterMaxLoadNum * len(b.table) + b.load++ + if (b.load * broadcasterMaxLoadDen) > (broadcasterMaxLoadNum * len(b.table)) { + // Double the number of slots in the new table. + newlen := broadcasterMinNonZeroTableSize + if len(b.table) != 0 { + newlen = 2 * len(b.table) + } + if newlen <= cap(b.table) { + // Reuse excess capacity in the current table, moving entries not + // already in their first-probed positions to better ones. + newtable := b.table[:newlen] + newmask := uint64(newlen - 1) + for i := range b.table { + if b.table[i].receiver != nil && uint64(b.table[i].id)&newmask != uint64(i) { + entry := b.table[i] + b.table[i] = broadcasterSlot{} + broadcasterTableInsert(newtable, entry.id, entry.receiver, entry.filter) + } + } + b.table = newtable + } else { + newtable := make([]broadcasterSlot, newlen) + // Copy existing entries to the new table. + for i := range b.table { + if b.table[i].receiver != nil { + broadcasterTableInsert(newtable, b.table[i].id, b.table[i].receiver, b.table[i].filter) + } + } + // Switch to the new table. + b.table = newtable + } + } + + broadcasterTableInsert(b.table, id, r, filter) + b.mu.Unlock() + return id +} + +// Preconditions: +// - table must not be full. +// - len(table) is a power of 2. +func broadcasterTableInsert(table []broadcasterSlot, id SubscriptionID, r *Receiver, filter Set) { + entry := broadcasterSlot{ + receiver: r, + filter: filter, + id: id, + } + mask := uint64(len(table) - 1) + i := uint64(id) & mask + disp := uint64(0) + for { + if table[i].receiver == nil { + table[i] = entry + return + } + // If we've been displaced farther from our first-probed slot than the + // element stored in this one, swap elements and switch to inserting + // the replaced one. (This is Robin Hood insertion.) + slotDisp := (i - uint64(table[i].id)) & mask + if disp > slotDisp { + table[i], entry = entry, table[i] + disp = slotDisp + } + i = (i + 1) & mask + disp++ + } +} + +// UnsubscribeEvents implements Source.UnsubscribeEvents. +func (b *Broadcaster) UnsubscribeEvents(id SubscriptionID) { + b.mu.Lock() + + mask := uint64(len(b.table) - 1) + i := uint64(id) & mask + for { + if b.table[i].id == id { + // Found the element to remove. Move all subsequent elements + // backward until we either find an empty slot, or an element that + // is already in its first-probed slot. (This is backward shift + // deletion.) + for { + next := (i + 1) & mask + if b.table[next].receiver == nil { + break + } + if uint64(b.table[next].id)&mask == next { + break + } + b.table[i] = b.table[next] + i = next + } + b.table[i] = broadcasterSlot{} + break + } + i = (i + 1) & mask + } + + // If a table 1/4 of the current size would still be at or under the + // maximum load factor (i.e. the current table size is at least two + // expansions bigger than necessary), halve the size of the table to reduce + // the cost of Broadcast. Since we are concerned with iteration time and + // not memory usage, reuse the existing slice to reduce future allocations + // from table re-expansion. + b.load-- + if len(b.table) > broadcasterMinNonZeroTableSize && (b.load*(4*broadcasterMaxLoadDen)) <= (broadcasterMaxLoadNum*len(b.table)) { + newlen := len(b.table) / 2 + newtable := b.table[:newlen] + for i := newlen; i < len(b.table); i++ { + if b.table[i].receiver != nil { + broadcasterTableInsert(newtable, b.table[i].id, b.table[i].receiver, b.table[i].filter) + b.table[i] = broadcasterSlot{} + } + } + b.table = newtable + } + + b.mu.Unlock() +} + +// Broadcast notifies all Receivers subscribed to the Broadcaster of the subset +// of events to which they subscribed. The order in which Receivers are +// notified is unspecified. +func (b *Broadcaster) Broadcast(events Set) { + b.mu.Lock() + for i := range b.table { + if intersection := events & b.table[i].filter; intersection != 0 { + // We don't need to check if broadcasterSlot.receiver is nil, since + // if it is then broadcasterSlot.filter is 0. + b.table[i].receiver.Notify(intersection) + } + } + b.mu.Unlock() +} + +// FilteredEvents returns the set of events for which Broadcast will notify at +// least one Receiver, i.e. the union of filters for all subscribed Receivers. +func (b *Broadcaster) FilteredEvents() Set { + var es Set + b.mu.Lock() + for i := range b.table { + es |= b.table[i].filter + } + b.mu.Unlock() + return es +} diff --git a/pkg/syncevent/receiver.go b/pkg/syncevent/receiver.go new file mode 100644 index 0000000..f8b3955 --- /dev/null +++ b/pkg/syncevent/receiver.go @@ -0,0 +1,101 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syncevent + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" +) + +// Receiver is an event sink that holds pending events and invokes a callback +// whenever new events become pending. Receiver's methods may be called +// concurrently from multiple goroutines. +// +// Receiver.Init() must be called before first use. +type Receiver struct { + // pending is the set of pending events. pending is accessed using atomic + // memory operations. + pending atomicbitops.Uint64 + + // cb is notified when new events become pending. cb is immutable after + // Init(). + cb ReceiverCallback +} + +// ReceiverCallback receives callbacks from a Receiver. +type ReceiverCallback interface { + // NotifyPending is called when the corresponding Receiver has new pending + // events. + // + // NotifyPending is called synchronously from Receiver.Notify(), so + // implementations must not take locks that may be held by callers of + // Receiver.Notify(). NotifyPending may be called concurrently from + // multiple goroutines. + NotifyPending() +} + +// Init must be called before first use of r. +func (r *Receiver) Init(cb ReceiverCallback) { + r.cb = cb +} + +// Pending returns the set of pending events. +func (r *Receiver) Pending() Set { + return Set(r.pending.Load()) +} + +// Notify sets the given events as pending. +func (r *Receiver) Notify(es Set) { + p := Set(r.pending.Load()) + // Optimization: Skip the atomic CAS on r.pending if all events are + // already pending. + if p&es == es { + return + } + // When this is uncontended (the common case), CAS is faster than + // atomic-OR because the former is inlined and the latter (which we + // implement in assembly ourselves) is not. + if !r.pending.CompareAndSwap(uint64(p), uint64(p|es)) { + // If the CAS fails, fall back to atomic-OR. + atomicbitops.OrUint64(&r.pending, uint64(es)) + } + r.cb.NotifyPending() +} + +// Ack unsets the given events as pending. +func (r *Receiver) Ack(es Set) { + p := Set(r.pending.Load()) + // Optimization: Skip the atomic CAS on r.pending if all events are + // already not pending. + if p&es == 0 { + return + } + // When this is uncontended (the common case), CAS is faster than + // atomic-AND because the former is inlined and the latter (which we + // implement in assembly ourselves) is not. + if !r.pending.CompareAndSwap(uint64(p), uint64(p&^es)) { + // If the CAS fails, fall back to atomic-AND. + atomicbitops.AndUint64(&r.pending, ^uint64(es)) + } +} + +// PendingAndAckAll unsets all events as pending and returns the set of +// previously-pending events. +// +// PendingAndAckAll should only be used in preference to a call to Pending +// followed by a conditional call to Ack when the caller expects events to be +// pending (e.g. after a call to ReceiverCallback.NotifyPending()). +func (r *Receiver) PendingAndAckAll() Set { + return Set(r.pending.Swap(0)) +} diff --git a/pkg/syncevent/source.go b/pkg/syncevent/source.go new file mode 100644 index 0000000..b924aee --- /dev/null +++ b/pkg/syncevent/source.go @@ -0,0 +1,61 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syncevent + +// Source represents an event source. +type Source interface { + // SubscribeEvents causes the Source to notify the given Receiver of the + // given subset of events. + // + // Preconditions: + // * r != nil. + // * The ReceiverCallback for r must not take locks that are ordered + // prior to the Source; for example, it cannot call any Source + // methods. + SubscribeEvents(r *Receiver, filter Set) SubscriptionID + + // UnsubscribeEvents causes the Source to stop notifying the Receiver + // subscribed by a previous call to SubscribeEvents that returned the given + // SubscriptionID. + // + // Preconditions: UnsubscribeEvents may be called at most once for any + // given SubscriptionID. + UnsubscribeEvents(id SubscriptionID) +} + +// SubscriptionID identifies a call to Source.SubscribeEvents. +type SubscriptionID uint64 + +// UnsubscribeAndAck is a convenience function that unsubscribes r from the +// given events from src and also clears them from r. +func UnsubscribeAndAck(src Source, r *Receiver, filter Set, id SubscriptionID) { + src.UnsubscribeEvents(id) + r.Ack(filter) +} + +// NoopSource implements Source by never sending events to subscribed +// Receivers. +type NoopSource struct{} + +// SubscribeEvents implements Source.SubscribeEvents. +func (NoopSource) SubscribeEvents(*Receiver, Set) SubscriptionID { + return 0 +} + +// UnsubscribeEvents implements Source.UnsubscribeEvents. +func (NoopSource) UnsubscribeEvents(SubscriptionID) { +} + +// See Broadcaster for a non-noop implementations of Source. diff --git a/pkg/syncevent/syncevent.go b/pkg/syncevent/syncevent.go new file mode 100644 index 0000000..9fb6a06 --- /dev/null +++ b/pkg/syncevent/syncevent.go @@ -0,0 +1,32 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package syncevent provides efficient primitives for goroutine +// synchronization based on event bitmasks. +package syncevent + +// Set is a bitmask where each bit represents a distinct user-defined event. +// The event package does not treat any bits in Set specially. +type Set uint64 + +const ( + // NoEvents is a Set containing no events. + NoEvents = Set(0) + + // AllEvents is a Set containing all possible events. + AllEvents = ^Set(0) + + // MaxEvents is the number of distinct events that can be represented by a Set. + MaxEvents = 64 +) diff --git a/pkg/syncevent/syncevent_state_autogen.go b/pkg/syncevent/syncevent_state_autogen.go new file mode 100644 index 0000000..2c69115 --- /dev/null +++ b/pkg/syncevent/syncevent_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package syncevent diff --git a/pkg/syncevent/syncevent_unsafe_state_autogen.go b/pkg/syncevent/syncevent_unsafe_state_autogen.go new file mode 100644 index 0000000..2c69115 --- /dev/null +++ b/pkg/syncevent/syncevent_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package syncevent diff --git a/pkg/syncevent/waiter_unsafe.go b/pkg/syncevent/waiter_unsafe.go new file mode 100644 index 0000000..6528155 --- /dev/null +++ b/pkg/syncevent/waiter_unsafe.go @@ -0,0 +1,197 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syncevent + +import ( + "sync/atomic" + "unsafe" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// Waiter allows a goroutine to block on pending events received by a Receiver. +// +// Waiter.Init() must be called before first use. +type Waiter struct { + r Receiver + + // g is one of: + // + // - 0: No goroutine is blocking in Wait. + // + // - preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet + // completed waiterUnlock(). Thus the wait can only be interrupted by + // replacing the value of g with 0 (the G may not be in state Gwaiting yet, + // so we can't call goready.) + // + // - Otherwise: g is a pointer to the runtime.g in state Gwaiting for the + // goroutine blocked in Wait, which can only be woken by calling goready. + g uintptr `state:"zerovalue"` +} + +const preparingG = 1 + +// Init must be called before first use of w. +func (w *Waiter) Init() { + w.r.Init(w) +} + +// Receiver returns the Receiver that receives events that unblock calls to +// w.Wait(). +func (w *Waiter) Receiver() *Receiver { + return &w.r +} + +// Pending returns the set of pending events. +func (w *Waiter) Pending() Set { + return w.r.Pending() +} + +// Wait blocks until at least one event is pending, then returns the set of +// pending events. It does not affect the set of pending events; callers must +// call w.Ack() to do so, or use w.WaitAndAck() instead. +// +// Precondition: Only one goroutine may call any Wait* method at a time. +func (w *Waiter) Wait() Set { + return w.WaitFor(AllEvents) +} + +// WaitFor blocks until at least one event in es is pending, then returns the +// set of pending events (including those not in es). It does not affect the +// set of pending events; callers must call w.Ack() to do so. +// +// Precondition: Only one goroutine may call any Wait* method at a time. +func (w *Waiter) WaitFor(es Set) Set { + for { + // Optimization: Skip the atomic store to w.g if an event is already + // pending. + if p := w.r.Pending(); p&es != NoEvents { + return p + } + + // Indicate that we're preparing to go to sleep. + atomic.StoreUintptr(&w.g, preparingG) + + // If an event is pending, abort the sleep. + if p := w.r.Pending(); p&es != NoEvents { + atomic.StoreUintptr(&w.g, 0) + return p + } + + // If w.g is still preparingG (i.e. w.NotifyPending() has not been + // called or has not reached atomic.SwapUintptr()), go to sleep until + // w.NotifyPending() => goready(). + sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceBlockSelect, 0) + } +} + +//go:norace +//go:nosplit +func waiterCommit(g uintptr, wg unsafe.Pointer) bool { + // The only way this CAS can fail is if a call to Waiter.NotifyPending() + // has replaced *wg with nil, in which case we should not sleep. + return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(wg), preparingG, g) +} + +// Ack marks the given events as not pending. +func (w *Waiter) Ack(es Set) { + w.r.Ack(es) +} + +// WaitAndAckAll blocks until at least one event is pending, then marks all +// events as not pending and returns the set of previously-pending events. +// +// Precondition: Only one goroutine may call any Wait* method at a time. +func (w *Waiter) WaitAndAckAll() Set { + // Optimization: Skip the atomic store to w.g if an event is already + // pending. Call Pending() first since, in the common case that events are + // not yet pending, this skips an atomic swap on w.r.pending. + if w.r.Pending() != NoEvents { + if p := w.r.PendingAndAckAll(); p != NoEvents { + return p + } + } + + for { + // Indicate that we're preparing to go to sleep. + atomic.StoreUintptr(&w.g, preparingG) + + // If an event is pending, abort the sleep. + if w.r.Pending() != NoEvents { + if p := w.r.PendingAndAckAll(); p != NoEvents { + atomic.StoreUintptr(&w.g, 0) + return p + } + } + + // If w.g is still preparingG (i.e. w.NotifyPending() has not been + // called or has not reached atomic.SwapUintptr()), go to sleep until + // w.NotifyPending() => goready(). + sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceBlockSelect, 0) + + // Check for pending events. We call PendingAndAckAll() directly now since + // we only expect to be woken after events become pending. + if p := w.r.PendingAndAckAll(); p != NoEvents { + return p + } + } +} + +// Notify marks the given events as pending, possibly unblocking concurrent +// calls to w.Wait() or w.WaitFor(). +func (w *Waiter) Notify(es Set) { + w.r.Notify(es) +} + +// NotifyPending implements ReceiverCallback.NotifyPending. Users of Waiter +// should not call NotifyPending. +func (w *Waiter) NotifyPending() { + // Optimization: Skip the atomic swap on w.g if there is no sleeping + // goroutine. NotifyPending is called after w.r.Pending() is updated, so + // concurrent and future calls to w.Wait() will observe pending events and + // abort sleeping. + if atomic.LoadUintptr(&w.g) == 0 { + return + } + // Wake a sleeping G, or prevent a G that is preparing to sleep from doing + // so. Swap is needed here to ensure that only one call to NotifyPending + // calls goready. + if g := atomic.SwapUintptr(&w.g, 0); g > preparingG { + sync.Goready(g, 0, true /* wakep */) + } +} + +var waiterPool = sync.Pool{ + New: func() any { + w := &Waiter{} + w.Init() + return w + }, +} + +// GetWaiter returns an unused Waiter. PutWaiter should be called to release +// the Waiter once it is no longer needed. +// +// Where possible, users should prefer to associate each goroutine that calls +// Waiter.Wait() with a distinct pre-allocated Waiter to avoid allocation of +// Waiters in hot paths. +func GetWaiter() *Waiter { + return waiterPool.Get().(*Waiter) +} + +// PutWaiter releases an unused Waiter previously returned by GetWaiter. +func PutWaiter(w *Waiter) { + waiterPool.Put(w) +} diff --git a/pkg/syserr/host_darwin.go b/pkg/syserr/host_darwin.go new file mode 100644 index 0000000..67fa71f --- /dev/null +++ b/pkg/syserr/host_darwin.go @@ -0,0 +1,44 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin +// +build darwin + +package syserr + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +const maxErrno = 107 + +var darwinHostTranslations [maxErrno]*Error + +func getHostTranslation(err unix.Errno) *Error { + if uint64(err) >= uint64(len(darwinHostTranslations)) { + return nil + } + return darwinHostTranslations[err] +} + +// TODO(gvisor.dev/issue/1270): We currently only add translations for errors +// that exist both on Darwin and Linux. +func addHostTranslation(host unix.Errno, trans *Error) { + if darwinHostTranslations[host] != nil { + panic(fmt.Sprintf("duplicate translation for host errno %q (%d)", host.Error(), host)) + } + darwinHostTranslations[host] = trans +} diff --git a/pkg/syserr/host_linux.go b/pkg/syserr/host_linux.go new file mode 100644 index 0000000..f4434dd --- /dev/null +++ b/pkg/syserr/host_linux.go @@ -0,0 +1,95 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package syserr + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/abi/linux/errno" + "golang.org/x/sys/unix" +) + +const maxErrno = 134 + +var linuxHostTranslations [maxErrno]*Error + +func getHostTranslation(err unix.Errno) *Error { + if uint64(err) >= uint64(len(linuxHostTranslations)) { + return nil + } + return linuxHostTranslations[err] +} + +func addHostTranslation(host unix.Errno, trans *Error) { + if linuxHostTranslations[host] != nil { + panic(fmt.Sprintf("duplicate translation for host errno %q (%d)", host.Error(), host)) + } + linuxHostTranslations[host] = trans +} + +// TODO(b/34162363): Remove or replace most of these errors. +// +// Some of the errors should be replaced with package specific errors and +// others should be removed entirely. +var ( + ErrDeadlock = newWithHost("resource deadlock would occur", errno.EDEADLOCK, unix.EDEADLOCK) + ErrChannelOutOfRange = newWithHost("channel number out of range", errno.ECHRNG, unix.ECHRNG) + ErrLevelTwoNotSynced = newWithHost("level 2 not synchronized", errno.EL2NSYNC, unix.EL2NSYNC) + ErrLevelThreeHalted = newWithHost("level 3 halted", errno.EL3HLT, unix.EL3HLT) + ErrLevelThreeReset = newWithHost("level 3 reset", errno.EL3RST, unix.EL3RST) + ErrLinkNumberOutOfRange = newWithHost("link number out of range", errno.ELNRNG, unix.ELNRNG) + ErrProtocolDriverNotAttached = newWithHost("protocol driver not attached", errno.EUNATCH, unix.EUNATCH) + ErrNoCSIAvailable = newWithHost("no CSI structure available", errno.ENOCSI, unix.ENOCSI) + ErrLevelTwoHalted = newWithHost("level 2 halted", errno.EL2HLT, unix.EL2HLT) + ErrInvalidExchange = newWithHost("invalid exchange", errno.EBADE, unix.EBADE) + ErrInvalidRequestDescriptor = newWithHost("invalid request descriptor", errno.EBADR, unix.EBADR) + ErrExchangeFull = newWithHost("exchange full", errno.EXFULL, unix.EXFULL) + ErrNoAnode = newWithHost("no anode", errno.ENOANO, unix.ENOANO) + ErrInvalidRequestCode = newWithHost("invalid request code", errno.EBADRQC, unix.EBADRQC) + ErrInvalidSlot = newWithHost("invalid slot", errno.EBADSLT, unix.EBADSLT) + ErrBadFontFile = newWithHost("bad font file format", errno.EBFONT, unix.EBFONT) + ErrMachineNotOnNetwork = newWithHost("machine is not on the network", errno.ENONET, unix.ENONET) + ErrPackageNotInstalled = newWithHost("package not installed", errno.ENOPKG, unix.ENOPKG) + ErrAdvertise = newWithHost("advertise error", errno.EADV, unix.EADV) + ErrSRMount = newWithHost("srmount error", errno.ESRMNT, unix.ESRMNT) + ErrSendCommunication = newWithHost("communication error on send", errno.ECOMM, unix.ECOMM) + ErrRFS = newWithHost("RFS specific error", errno.EDOTDOT, unix.EDOTDOT) + ErrNetworkNameNotUnique = newWithHost("name not unique on network", errno.ENOTUNIQ, unix.ENOTUNIQ) + ErrFDInBadState = newWithHost("file descriptor in bad state", errno.EBADFD, unix.EBADFD) + ErrRemoteAddressChanged = newWithHost("remote address changed", errno.EREMCHG, unix.EREMCHG) + ErrSharedLibraryInaccessible = newWithHost("can not access a needed shared library", errno.ELIBACC, unix.ELIBACC) + ErrCorruptedSharedLibrary = newWithHost("accessing a corrupted shared library", errno.ELIBBAD, unix.ELIBBAD) + ErrLibSectionCorrupted = newWithHost(".lib section in a.out corrupted", errno.ELIBSCN, unix.ELIBSCN) + ErrTooManySharedLibraries = newWithHost("attempting to link in too many shared libraries", errno.ELIBMAX, unix.ELIBMAX) + ErrSharedLibraryExeced = newWithHost("cannot exec a shared library directly", errno.ELIBEXEC, unix.ELIBEXEC) + ErrShouldRestart = newWithHost("interrupted system call should be restarted", errno.ERESTART, unix.ERESTART) + ErrStreamPipe = newWithHost("streams pipe error", errno.ESTRPIPE, unix.ESTRPIPE) + ErrStructureNeedsCleaning = newWithHost("structure needs cleaning", errno.EUCLEAN, unix.EUCLEAN) + ErrIsNotNamedFile = newWithHost("not a XENIX named type file", errno.ENOTNAM, unix.ENOTNAM) + ErrNotAvailable = newWithHost("no XENIX semaphores available", errno.ENAVAIL, unix.ENAVAIL) + ErrIsNamedFile = newWithHost("is a named type file", errno.EISNAM, unix.EISNAM) + ErrRemoteIO = newWithHost("remote I/O error", errno.EREMOTEIO, unix.EREMOTEIO) + ErrNoMedium = newWithHost("no medium found", errno.ENOMEDIUM, unix.ENOMEDIUM) + ErrWrongMediumType = newWithHost("wrong medium type", errno.EMEDIUMTYPE, unix.EMEDIUMTYPE) + ErrNoKey = newWithHost("required key not available", errno.ENOKEY, unix.ENOKEY) + ErrKeyExpired = newWithHost("key has expired", errno.EKEYEXPIRED, unix.EKEYEXPIRED) + ErrKeyRevoked = newWithHost("key has been revoked", errno.EKEYREVOKED, unix.EKEYREVOKED) + ErrKeyRejected = newWithHost("key was rejected by service", errno.EKEYREJECTED, unix.EKEYREJECTED) + ErrRFKill = newWithHost("operation not possible due to RF-kill", errno.ERFKILL, unix.ERFKILL) + ErrHwPoison = newWithHost("memory page has hardware error", errno.EHWPOISON, unix.EHWPOISON) +) diff --git a/pkg/syserr/netstack.go b/pkg/syserr/netstack.go new file mode 100644 index 0000000..fd18d83 --- /dev/null +++ b/pkg/syserr/netstack.go @@ -0,0 +1,160 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package syserr + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/abi/linux/errno" + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// LINT.IfChange + +// Mapping for tcpip.Error types. +var ( + ErrUnknownProtocol = New((&tcpip.ErrUnknownProtocol{}).String(), errno.EINVAL) + ErrUnknownNICID = New((&tcpip.ErrUnknownNICID{}).String(), errno.ENODEV) + ErrUnknownDevice = New((&tcpip.ErrUnknownDevice{}).String(), errno.ENODEV) + ErrUnknownProtocolOption = New((&tcpip.ErrUnknownProtocolOption{}).String(), errno.ENOPROTOOPT) + ErrDuplicateNICID = New((&tcpip.ErrDuplicateNICID{}).String(), errno.EEXIST) + ErrDuplicateAddress = New((&tcpip.ErrDuplicateAddress{}).String(), errno.EEXIST) + ErrAlreadyBound = New((&tcpip.ErrAlreadyBound{}).String(), errno.EINVAL) + ErrInvalidEndpointState = New((&tcpip.ErrInvalidEndpointState{}).String(), errno.EINVAL) + ErrAlreadyConnecting = New((&tcpip.ErrAlreadyConnecting{}).String(), errno.EALREADY) + ErrNoPortAvailable = New((&tcpip.ErrNoPortAvailable{}).String(), errno.EAGAIN) + ErrPortInUse = New((&tcpip.ErrPortInUse{}).String(), errno.EADDRINUSE) + ErrBadLocalAddress = New((&tcpip.ErrBadLocalAddress{}).String(), errno.EADDRNOTAVAIL) + ErrClosedForSend = New((&tcpip.ErrClosedForSend{}).String(), errno.EPIPE) + ErrClosedForReceive = New((&tcpip.ErrClosedForReceive{}).String(), errno.NOERRNO) + ErrTimeout = New((&tcpip.ErrTimeout{}).String(), errno.ETIMEDOUT) + ErrAborted = New((&tcpip.ErrAborted{}).String(), errno.EPIPE) + ErrConnectStarted = New((&tcpip.ErrConnectStarted{}).String(), errno.EINPROGRESS) + ErrDestinationRequired = New((&tcpip.ErrDestinationRequired{}).String(), errno.EDESTADDRREQ) + ErrNotSupported = New((&tcpip.ErrNotSupported{}).String(), errno.EOPNOTSUPP) + ErrQueueSizeNotSupported = New((&tcpip.ErrQueueSizeNotSupported{}).String(), errno.ENOTTY) + ErrNoSuchFile = New((&tcpip.ErrNoSuchFile{}).String(), errno.ENOENT) + ErrInvalidOptionValue = New((&tcpip.ErrInvalidOptionValue{}).String(), errno.EINVAL) + ErrBroadcastDisabled = New((&tcpip.ErrBroadcastDisabled{}).String(), errno.EACCES) + ErrNotPermittedNet = New((&tcpip.ErrNotPermitted{}).String(), errno.EPERM) + ErrBadBuffer = New((&tcpip.ErrBadBuffer{}).String(), errno.EFAULT) + ErrMalformedHeader = New((&tcpip.ErrMalformedHeader{}).String(), errno.EINVAL) + ErrInvalidPortRange = New((&tcpip.ErrInvalidPortRange{}).String(), errno.EINVAL) + ErrMulticastInputCannotBeOutput = New((&tcpip.ErrMulticastInputCannotBeOutput{}).String(), errno.EINVAL) + ErrMissingRequiredFields = New((&tcpip.ErrMissingRequiredFields{}).String(), errno.EINVAL) + ErrNoNet = New((&tcpip.ErrNoNet{}).String(), errno.ENONET) + ErrEndpointBusy = New((&tcpip.ErrEndpointBusy{}).String(), errno.EBUSY) +) + +// TranslateNetstackError converts an error from the tcpip package to a sentry +// internal error. +func TranslateNetstackError(err tcpip.Error) *Error { + switch err.(type) { + case nil: + return nil + case *tcpip.ErrUnknownProtocol: + return ErrUnknownProtocol + case *tcpip.ErrUnknownNICID: + return ErrUnknownNICID + case *tcpip.ErrUnknownDevice: + return ErrUnknownDevice + case *tcpip.ErrUnknownProtocolOption: + return ErrUnknownProtocolOption + case *tcpip.ErrDuplicateNICID: + return ErrDuplicateNICID + case *tcpip.ErrDuplicateAddress: + return ErrDuplicateAddress + case *tcpip.ErrHostUnreachable: + return ErrHostUnreachable + case *tcpip.ErrHostDown: + return ErrHostDown + case *tcpip.ErrNoNet: + return ErrNoNet + case *tcpip.ErrAlreadyBound: + return ErrAlreadyBound + case *tcpip.ErrInvalidEndpointState: + return ErrInvalidEndpointState + case *tcpip.ErrAlreadyConnecting: + return ErrAlreadyConnecting + case *tcpip.ErrAlreadyConnected: + return ErrAlreadyConnected + case *tcpip.ErrNoPortAvailable: + return ErrNoPortAvailable + case *tcpip.ErrPortInUse: + return ErrPortInUse + case *tcpip.ErrBadLocalAddress: + return ErrBadLocalAddress + case *tcpip.ErrClosedForSend: + return ErrClosedForSend + case *tcpip.ErrClosedForReceive: + return ErrClosedForReceive + case *tcpip.ErrWouldBlock: + return ErrWouldBlock + case *tcpip.ErrConnectionRefused: + return ErrConnectionRefused + case *tcpip.ErrTimeout: + return ErrTimeout + case *tcpip.ErrAborted: + return ErrAborted + case *tcpip.ErrConnectStarted: + return ErrConnectStarted + case *tcpip.ErrDestinationRequired: + return ErrDestinationRequired + case *tcpip.ErrNotSupported: + return ErrNotSupported + case *tcpip.ErrQueueSizeNotSupported: + return ErrQueueSizeNotSupported + case *tcpip.ErrNotConnected: + return ErrNotConnected + case *tcpip.ErrConnectionReset: + return ErrConnectionReset + case *tcpip.ErrConnectionAborted: + return ErrConnectionAborted + case *tcpip.ErrNoSuchFile: + return ErrNoSuchFile + case *tcpip.ErrInvalidOptionValue: + return ErrInvalidOptionValue + case *tcpip.ErrBadAddress: + return ErrBadAddress + case *tcpip.ErrNetworkUnreachable: + return ErrNetworkUnreachable + case *tcpip.ErrMessageTooLong: + return ErrMessageTooLong + case *tcpip.ErrNoBufferSpace: + return ErrNoBufferSpace + case *tcpip.ErrBroadcastDisabled: + return ErrBroadcastDisabled + case *tcpip.ErrNotPermitted: + return ErrNotPermittedNet + case *tcpip.ErrAddressFamilyNotSupported: + return ErrAddressFamilyNotSupported + case *tcpip.ErrBadBuffer: + return ErrBadBuffer + case *tcpip.ErrMalformedHeader: + return ErrMalformedHeader + case *tcpip.ErrInvalidPortRange: + return ErrInvalidPortRange + case *tcpip.ErrMulticastInputCannotBeOutput: + return ErrMulticastInputCannotBeOutput + case *tcpip.ErrMissingRequiredFields: + return ErrMissingRequiredFields + case *tcpip.ErrEndpointBusy: + return ErrEndpointBusy + default: + panic(fmt.Sprintf("unknown error %T", err)) + } +} + +// LINT.ThenChange(../tcpip/errors.go) diff --git a/pkg/syserr/syserr.go b/pkg/syserr/syserr.go new file mode 100644 index 0000000..1c46961 --- /dev/null +++ b/pkg/syserr/syserr.go @@ -0,0 +1,282 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package syserr contains sandbox-internal errors. These errors are distinct +// from both the errors returned by host system calls and the errors returned +// to sandboxed applications. +package syserr + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/abi/linux/errno" + "github.com/sagernet/gvisor/pkg/errors" + "github.com/sagernet/gvisor/pkg/errors/linuxerr" + "github.com/sagernet/gvisor/pkg/safecopy" + "golang.org/x/sys/unix" +) + +// Error represents an internal error. +type Error struct { + // message is the human readable form of this Error. + message string + + // noTranslation indicates that this Error cannot be translated to a + // errno.Errno. + noTranslation bool + + // errno is the errno.Errno this Error should be translated to. + errno errno.Errno +} + +// New creates a new Error and adds a translation for it. +// +// New must only be called at init. +func New(message string, linuxTranslation errno.Errno) *Error { + err := &Error{message: message, errno: linuxTranslation} + + // TODO(b/34162363): Remove this. + if int(err.errno) >= len(linuxBackwardsTranslations) { + panic(fmt.Sprint("invalid errno: ", err.errno)) + } + + e := error(unix.Errno(err.errno)) + // linuxerr.ErrWouldBlock gets translated to linuxerr.EWOULDBLOCK and + // enables proper blocking semantics. This should temporary address the + // class of blocking bugs that keep popping up with the current state of + // the error space. + if err.errno == linuxerr.EWOULDBLOCK.Errno() { + e = linuxerr.ErrWouldBlock + } + linuxBackwardsTranslations[err.errno] = linuxBackwardsTranslation{err: e, ok: true} + + return err +} + +// NewDynamic creates a new error with a dynamic error message and an errno +// translation. +// +// NewDynamic should only be used sparingly and not be used for static error +// messages. Errors with static error messages should be declared with New as +// global variables. +func NewDynamic(message string, linuxTranslation errno.Errno) *Error { + return &Error{message: message, errno: linuxTranslation} +} + +func newWithHost(message string, linuxTranslation errno.Errno, hostErrno unix.Errno) *Error { + e := New(message, linuxTranslation) + addHostTranslation(hostErrno, e) + return e +} + +// String implements fmt.Stringer.String. +func (e *Error) String() string { + if e == nil { + return "" + } + return e.message +} + +type linuxBackwardsTranslation struct { + err error + ok bool +} + +// TODO(b/34162363): Remove this. +var linuxBackwardsTranslations [maxErrno]linuxBackwardsTranslation + +// ToError translates an Error to a corresponding error value. +// +// TODO(b/34162363): Remove this. +func (e *Error) ToError() error { + if e == nil { + return nil + } + if e.noTranslation { + panic(fmt.Sprintf("error %q does not support translation", e.message)) + } + err := int(e.errno) + if err == errno.NOERRNO { + return nil + } + if err >= len(linuxBackwardsTranslations) || !linuxBackwardsTranslations[err].ok { + panic(fmt.Sprintf("unknown error %q (%d)", e.message, err)) + } + return linuxBackwardsTranslations[err].err +} + +// ToLinux converts the Error to a Linux ABI error that can be returned to the +// application. +func (e *Error) ToLinux() errno.Errno { + if e.noTranslation { + panic(fmt.Sprintf("No Linux ABI translation available for %q", e.message)) + } + return e.errno +} + +// AnnotatedError represents an error with an additional message. +type AnnotatedError struct { + error *Error + message string +} + +// Error implements Error() for the error interface +func (e *AnnotatedError) Error() string { + return fmt.Sprintf("%s: %s", e.error.String(), e.message) +} + +// NewAnnotatedError creates a new AnnotatedError with the given error and message. +func NewAnnotatedError(error *Error, message string) *AnnotatedError { + return &AnnotatedError{error: error, message: message} +} + +// GetError returns the underlying error. +func (e *AnnotatedError) GetError() *Error { + return e.error +} + +// TODO(b/34162363): Remove or replace most of these errors. +// +// Some of the errors should be replaced with package specific errors and +// others should be removed entirely. +// +// Note that some errors are declared in platform-specific files. +var ( + ErrNotPermitted = newWithHost("operation not permitted", errno.EPERM, unix.EPERM) + ErrNoFileOrDir = newWithHost("no such file or directory", errno.ENOENT, unix.ENOENT) + ErrNoProcess = newWithHost("no such process", errno.ESRCH, unix.ESRCH) + ErrInterrupted = newWithHost("interrupted system call", errno.EINTR, unix.EINTR) + ErrIO = newWithHost("I/O error", errno.EIO, unix.EIO) + ErrDeviceOrAddress = newWithHost("no such device or address", errno.ENXIO, unix.ENXIO) + ErrTooManyArgs = newWithHost("argument list too long", errno.E2BIG, unix.E2BIG) + ErrEcec = newWithHost("exec format error", errno.ENOEXEC, unix.ENOEXEC) + ErrBadFD = newWithHost("bad file number", errno.EBADF, unix.EBADF) + ErrNoChild = newWithHost("no child processes", errno.ECHILD, unix.ECHILD) + ErrTryAgain = newWithHost("try again", errno.EAGAIN, unix.EAGAIN) + ErrNoMemory = newWithHost("out of memory", errno.ENOMEM, unix.ENOMEM) + ErrPermissionDenied = newWithHost("permission denied", errno.EACCES, unix.EACCES) + ErrBadAddress = newWithHost("bad address", errno.EFAULT, unix.EFAULT) + ErrNotBlockDevice = newWithHost("block device required", errno.ENOTBLK, unix.ENOTBLK) + ErrBusy = newWithHost("device or resource busy", errno.EBUSY, unix.EBUSY) + ErrExists = newWithHost("file exists", errno.EEXIST, unix.EEXIST) + ErrCrossDeviceLink = newWithHost("cross-device link", errno.EXDEV, unix.EXDEV) + ErrNoDevice = newWithHost("no such device", errno.ENODEV, unix.ENODEV) + ErrNotDir = newWithHost("not a directory", errno.ENOTDIR, unix.ENOTDIR) + ErrIsDir = newWithHost("is a directory", errno.EISDIR, unix.EISDIR) + ErrInvalidArgument = newWithHost("invalid argument", errno.EINVAL, unix.EINVAL) + ErrFileTableOverflow = newWithHost("file table overflow", errno.ENFILE, unix.ENFILE) + ErrTooManyOpenFiles = newWithHost("too many open files", errno.EMFILE, unix.EMFILE) + ErrNotTTY = newWithHost("not a typewriter", errno.ENOTTY, unix.ENOTTY) + ErrTestFileBusy = newWithHost("text file busy", errno.ETXTBSY, unix.ETXTBSY) + ErrFileTooBig = newWithHost("file too large", errno.EFBIG, unix.EFBIG) + ErrNoSpace = newWithHost("no space left on device", errno.ENOSPC, unix.ENOSPC) + ErrIllegalSeek = newWithHost("illegal seek", errno.ESPIPE, unix.ESPIPE) + ErrReadOnlyFS = newWithHost("read-only file system", errno.EROFS, unix.EROFS) + ErrTooManyLinks = newWithHost("too many links", errno.EMLINK, unix.EMLINK) + ErrBrokenPipe = newWithHost("broken pipe", errno.EPIPE, unix.EPIPE) + ErrDomain = newWithHost("math argument out of domain of func", errno.EDOM, unix.EDOM) + ErrRange = newWithHost("math result not representable", errno.ERANGE, unix.ERANGE) + ErrNameTooLong = newWithHost("file name too long", errno.ENAMETOOLONG, unix.ENAMETOOLONG) + ErrNoLocksAvailable = newWithHost("no record locks available", errno.ENOLCK, unix.ENOLCK) + ErrInvalidSyscall = newWithHost("invalid system call number", errno.ENOSYS, unix.ENOSYS) + ErrDirNotEmpty = newWithHost("directory not empty", errno.ENOTEMPTY, unix.ENOTEMPTY) + ErrLinkLoop = newWithHost("too many symbolic links encountered", errno.ELOOP, unix.ELOOP) + ErrNoMessage = newWithHost("no message of desired type", errno.ENOMSG, unix.ENOMSG) + ErrIdentifierRemoved = newWithHost("identifier removed", errno.EIDRM, unix.EIDRM) + ErrNotStream = newWithHost("device not a stream", errno.ENOSTR, unix.ENOSTR) + ErrNoDataAvailable = newWithHost("no data available", errno.ENODATA, unix.ENODATA) + ErrTimerExpired = newWithHost("timer expired", errno.ETIME, unix.ETIME) + ErrStreamsResourceDepleted = newWithHost("out of streams resources", errno.ENOSR, unix.ENOSR) + ErrIsRemote = newWithHost("object is remote", errno.EREMOTE, unix.EREMOTE) + ErrNoLink = newWithHost("link has been severed", errno.ENOLINK, unix.ENOLINK) + ErrProtocol = newWithHost("protocol error", errno.EPROTO, unix.EPROTO) + ErrMultihopAttempted = newWithHost("multihop attempted", errno.EMULTIHOP, unix.EMULTIHOP) + ErrInvalidDataMessage = newWithHost("not a data message", errno.EBADMSG, unix.EBADMSG) + ErrOverflow = newWithHost("value too large for defined data type", errno.EOVERFLOW, unix.EOVERFLOW) + ErrIllegalByteSequence = newWithHost("illegal byte sequence", errno.EILSEQ, unix.EILSEQ) + ErrTooManyUsers = newWithHost("too many users", errno.EUSERS, unix.EUSERS) + ErrNotASocket = newWithHost("socket operation on non-socket", errno.ENOTSOCK, unix.ENOTSOCK) + ErrDestinationAddressRequired = newWithHost("destination address required", errno.EDESTADDRREQ, unix.EDESTADDRREQ) + ErrMessageTooLong = newWithHost("message too long", errno.EMSGSIZE, unix.EMSGSIZE) + ErrWrongProtocolForSocket = newWithHost("protocol wrong type for socket", errno.EPROTOTYPE, unix.EPROTOTYPE) + ErrProtocolNotAvailable = newWithHost("protocol not available", errno.ENOPROTOOPT, unix.ENOPROTOOPT) + ErrProtocolNotSupported = newWithHost("protocol not supported", errno.EPROTONOSUPPORT, unix.EPROTONOSUPPORT) + ErrSocketNotSupported = newWithHost("socket type not supported", errno.ESOCKTNOSUPPORT, unix.ESOCKTNOSUPPORT) + ErrEndpointOperation = newWithHost("operation not supported on transport endpoint", errno.EOPNOTSUPP, unix.EOPNOTSUPP) + ErrProtocolFamilyNotSupported = newWithHost("protocol family not supported", errno.EPFNOSUPPORT, unix.EPFNOSUPPORT) + ErrAddressFamilyNotSupported = newWithHost("address family not supported by protocol", errno.EAFNOSUPPORT, unix.EAFNOSUPPORT) + ErrAddressInUse = newWithHost("address already in use", errno.EADDRINUSE, unix.EADDRINUSE) + ErrAddressNotAvailable = newWithHost("cannot assign requested address", errno.EADDRNOTAVAIL, unix.EADDRNOTAVAIL) + ErrNetworkDown = newWithHost("network is down", errno.ENETDOWN, unix.ENETDOWN) + ErrNetworkUnreachable = newWithHost("network is unreachable", errno.ENETUNREACH, unix.ENETUNREACH) + ErrNetworkReset = newWithHost("network dropped connection because of reset", errno.ENETRESET, unix.ENETRESET) + ErrConnectionAborted = newWithHost("software caused connection abort", errno.ECONNABORTED, unix.ECONNABORTED) + ErrConnectionReset = newWithHost("connection reset by peer", errno.ECONNRESET, unix.ECONNRESET) + ErrNoBufferSpace = newWithHost("no buffer space available", errno.ENOBUFS, unix.ENOBUFS) + ErrAlreadyConnected = newWithHost("transport endpoint is already connected", errno.EISCONN, unix.EISCONN) + ErrNotConnected = newWithHost("transport endpoint is not connected", errno.ENOTCONN, unix.ENOTCONN) + ErrShutdown = newWithHost("cannot send after transport endpoint shutdown", errno.ESHUTDOWN, unix.ESHUTDOWN) + ErrTooManyRefs = newWithHost("too many references: cannot splice", errno.ETOOMANYREFS, unix.ETOOMANYREFS) + ErrTimedOut = newWithHost("connection timed out", errno.ETIMEDOUT, unix.ETIMEDOUT) + ErrConnectionRefused = newWithHost("connection refused", errno.ECONNREFUSED, unix.ECONNREFUSED) + ErrHostDown = newWithHost("host is down", errno.EHOSTDOWN, unix.EHOSTDOWN) + ErrHostUnreachable = newWithHost("no route to host", errno.EHOSTUNREACH, unix.EHOSTUNREACH) + ErrAlreadyInProgress = newWithHost("operation already in progress", errno.EALREADY, unix.EALREADY) + ErrInProgress = newWithHost("operation now in progress", errno.EINPROGRESS, unix.EINPROGRESS) + ErrStaleFileHandle = newWithHost("stale file handle", errno.ESTALE, unix.ESTALE) + ErrQuotaExceeded = newWithHost("quota exceeded", errno.EDQUOT, unix.EDQUOT) + ErrCanceled = newWithHost("operation canceled", errno.ECANCELED, unix.ECANCELED) + ErrOwnerDied = newWithHost("owner died", errno.EOWNERDEAD, unix.EOWNERDEAD) + ErrNotRecoverable = newWithHost("state not recoverable", errno.ENOTRECOVERABLE, unix.ENOTRECOVERABLE) + + // ErrWouldBlock translates to EWOULDBLOCK which is the same as EAGAIN + // on Linux. + ErrWouldBlock = New("operation would block", errno.EWOULDBLOCK) +) + +// FromHost translates a unix.Errno to a corresponding Error value. +func FromHost(err unix.Errno) *Error { + got := getHostTranslation(err) + if got == nil { + panic(fmt.Sprintf("unknown host errno %q (%d)", err.Error(), err)) + } + return got +} + +// IsValid checks if the given errno is a valid errno which can be translated +// to an Error. +func IsValid(err unix.Errno) bool { + return getHostTranslation(err) != nil +} + +// FromError converts a generic error to an *Error. +// +// TODO(b/34162363): Remove this function. +func FromError(err error) *Error { + if err == nil { + return nil + } + + switch e := err.(type) { + case unix.Errno: + return FromHost(e) + case *errors.Error: + return FromHost(unix.Errno(e.Errno())) + case safecopy.SegvError, safecopy.BusError, safecopy.AlignmentError: + return FromHost(unix.EFAULT) + } + + msg := fmt.Sprintf("err: %s type: %T", err.Error(), err) + panic(msg) +} diff --git a/pkg/syserr/syserr_linux_state_autogen.go b/pkg/syserr/syserr_linux_state_autogen.go new file mode 100644 index 0000000..90e7961 --- /dev/null +++ b/pkg/syserr/syserr_linux_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package syserr diff --git a/pkg/syserr/syserr_state_autogen.go b/pkg/syserr/syserr_state_autogen.go new file mode 100644 index 0000000..b2a6f3a --- /dev/null +++ b/pkg/syserr/syserr_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build darwin +// +build darwin + +package syserr diff --git a/pkg/tcpip/adapters/gonet/err.go b/pkg/tcpip/adapters/gonet/err.go new file mode 100644 index 0000000..f5f6256 --- /dev/null +++ b/pkg/tcpip/adapters/gonet/err.go @@ -0,0 +1,103 @@ +package gonet + +import ( + "fmt" + "syscall" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +func TranslateNetstackError(err tcpip.Error) error { + switch err.(type) { + case nil: + return nil + case *tcpip.ErrUnknownProtocol: + return syscall.EINVAL + case *tcpip.ErrUnknownNICID: + return syscall.ENODEV + case *tcpip.ErrUnknownDevice: + return syscall.ENODEV + case *tcpip.ErrUnknownProtocolOption: + return syscall.ENOPROTOOPT + case *tcpip.ErrDuplicateNICID: + return syscall.EEXIST + case *tcpip.ErrDuplicateAddress: + return syscall.EEXIST + case *tcpip.ErrHostUnreachable: + return syscall.EHOSTUNREACH + case *tcpip.ErrHostDown: + return syscall.EHOSTDOWN + case *tcpip.ErrNoNet: + return errNoNet + case *tcpip.ErrAlreadyBound: + return syscall.EINVAL + case *tcpip.ErrInvalidEndpointState: + return syscall.EINVAL + case *tcpip.ErrAlreadyConnecting: + return syscall.EALREADY + case *tcpip.ErrAlreadyConnected: + return syscall.EISCONN + case *tcpip.ErrNoPortAvailable: + return syscall.EAGAIN + case *tcpip.ErrPortInUse: + return syscall.EADDRINUSE + case *tcpip.ErrBadLocalAddress: + return syscall.EADDRNOTAVAIL + case *tcpip.ErrClosedForSend: + return syscall.EPIPE + case *tcpip.ErrClosedForReceive: + return syscall.ENOTCONN + case *tcpip.ErrWouldBlock: + return syscall.EWOULDBLOCK + case *tcpip.ErrConnectionRefused: + return syscall.ECONNREFUSED + case *tcpip.ErrTimeout: + return syscall.ETIMEDOUT + case *tcpip.ErrAborted: + return syscall.EPIPE + case *tcpip.ErrConnectStarted: + return syscall.EINPROGRESS + case *tcpip.ErrDestinationRequired: + return syscall.EDESTADDRREQ + case *tcpip.ErrNotSupported: + return syscall.EOPNOTSUPP + case *tcpip.ErrQueueSizeNotSupported: + return syscall.ENOTTY + case *tcpip.ErrNotConnected: + return syscall.ENOTCONN + case *tcpip.ErrConnectionReset: + return syscall.ECONNRESET + case *tcpip.ErrConnectionAborted: + return syscall.ECONNABORTED + case *tcpip.ErrNoSuchFile: + return syscall.ENOENT + case *tcpip.ErrInvalidOptionValue: + return syscall.EINVAL + case *tcpip.ErrBadAddress: + return syscall.EFAULT + case *tcpip.ErrNetworkUnreachable: + return syscall.ENETUNREACH + case *tcpip.ErrMessageTooLong: + return syscall.EMSGSIZE + case *tcpip.ErrNoBufferSpace: + return syscall.ENOBUFS + case *tcpip.ErrBroadcastDisabled: + return syscall.EACCES + case *tcpip.ErrNotPermitted: + return syscall.EPERM + case *tcpip.ErrAddressFamilyNotSupported: + return syscall.EAFNOSUPPORT + case *tcpip.ErrBadBuffer: + return syscall.EFAULT + case *tcpip.ErrMalformedHeader: + return syscall.EINVAL + case *tcpip.ErrInvalidPortRange: + return syscall.EINVAL + case *tcpip.ErrMulticastInputCannotBeOutput: + return syscall.EINVAL + case *tcpip.ErrMissingRequiredFields: + return syscall.EINVAL + default: + panic(fmt.Sprintf("unknown error %T", err)) + } +} diff --git a/pkg/tcpip/adapters/gonet/err_linuxwindows.go b/pkg/tcpip/adapters/gonet/err_linuxwindows.go new file mode 100644 index 0000000..e96b78e --- /dev/null +++ b/pkg/tcpip/adapters/gonet/err_linuxwindows.go @@ -0,0 +1,7 @@ +//go:build linux || windows + +package gonet + +import "syscall" + +var errNoNet = syscall.ENONET diff --git a/pkg/tcpip/adapters/gonet/err_stub.go b/pkg/tcpip/adapters/gonet/err_stub.go new file mode 100644 index 0000000..e23f4ca --- /dev/null +++ b/pkg/tcpip/adapters/gonet/err_stub.go @@ -0,0 +1,7 @@ +//go:build !(linux || windows) + +package gonet + +import "errors" + +var errNoNet = errors.New("machine is not on the network") diff --git a/pkg/tcpip/adapters/gonet/gonet.go b/pkg/tcpip/adapters/gonet/gonet.go new file mode 100644 index 0000000..519f34c --- /dev/null +++ b/pkg/tcpip/adapters/gonet/gonet.go @@ -0,0 +1,714 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gonet provides a Go net package compatible wrapper for a tcpip stack. +package gonet + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport/tcp" + "github.com/sagernet/gvisor/pkg/tcpip/transport/udp" + "github.com/sagernet/gvisor/pkg/waiter" +) + +var ( + errCanceled = errors.New("operation canceled") + errWouldBlock = errors.New("operation would block") +) + +// timeoutError is how the net package reports timeouts. +type timeoutError struct{} + +func (e *timeoutError) Error() string { return "i/o timeout" } +func (e *timeoutError) Timeout() bool { return true } +func (e *timeoutError) Temporary() bool { return true } + +// A TCPListener is a wrapper around a TCP tcpip.Endpoint that implements +// net.Listener. +type TCPListener struct { + stack *stack.Stack + ep tcpip.Endpoint + wq *waiter.Queue + cancelOnce sync.Once + cancel chan struct{} +} + +// NewTCPListener creates a new TCPListener from a listening tcpip.Endpoint. +func NewTCPListener(s *stack.Stack, wq *waiter.Queue, ep tcpip.Endpoint) *TCPListener { + return &TCPListener{ + stack: s, + ep: ep, + wq: wq, + cancel: make(chan struct{}), + } +} + +// maxListenBacklog is set to be reasonably high for most uses of gonet. Go net +// package uses the value in /proc/sys/net/core/somaxconn file in Linux as the +// default listen backlog. The value below matches the default in common linux +// distros. +// +// See: https://cs.opensource.google/go/go/+/refs/tags/go1.18.1:src/net/sock_linux.go;drc=refs%2Ftags%2Fgo1.18.1;l=66 +const maxListenBacklog = 4096 + +// ListenTCP creates a new TCPListener. +func ListenTCP(s *stack.Stack, addr tcpip.FullAddress, network tcpip.NetworkProtocolNumber) (*TCPListener, error) { + // Create a TCP endpoint, bind it, then start listening. + var wq waiter.Queue + ep, err := s.NewEndpoint(tcp.ProtocolNumber, network, &wq) + if err != nil { + return nil, TranslateNetstackError(err) + } + + if err := ep.Bind(addr); err != nil { + ep.Close() + return nil, &net.OpError{ + Op: "bind", + Net: "tcp", + Addr: fullToTCPAddr(addr), + Err: TranslateNetstackError(err), + } + } + + if err := ep.Listen(maxListenBacklog); err != nil { + ep.Close() + return nil, &net.OpError{ + Op: "listen", + Net: "tcp", + Addr: fullToTCPAddr(addr), + Err: TranslateNetstackError(err), + } + } + + return NewTCPListener(s, &wq, ep), nil +} + +// Close implements net.Listener.Close. +func (l *TCPListener) Close() error { + l.ep.Close() + return nil +} + +// Shutdown stops the HTTP server. +func (l *TCPListener) Shutdown() { + l.ep.Shutdown(tcpip.ShutdownWrite | tcpip.ShutdownRead) + l.cancelOnce.Do(func() { + close(l.cancel) // broadcast cancellation + }) +} + +// Addr implements net.Listener.Addr. +func (l *TCPListener) Addr() net.Addr { + a, err := l.ep.GetLocalAddress() + if err != nil { + return nil + } + return fullToTCPAddr(a) +} + +type deadlineTimer struct { + // mu protects the fields below. + mu sync.Mutex + + readTimer *time.Timer + readCancelCh chan struct{} + writeTimer *time.Timer + writeCancelCh chan struct{} +} + +func (d *deadlineTimer) init() { + d.readCancelCh = make(chan struct{}) + d.writeCancelCh = make(chan struct{}) +} + +func (d *deadlineTimer) readCancel() <-chan struct{} { + d.mu.Lock() + c := d.readCancelCh + d.mu.Unlock() + return c +} + +func (d *deadlineTimer) writeCancel() <-chan struct{} { + d.mu.Lock() + c := d.writeCancelCh + d.mu.Unlock() + return c +} + +// setDeadline contains the shared logic for setting a deadline. +// +// cancelCh and timer must be pointers to deadlineTimer.readCancelCh and +// deadlineTimer.readTimer or deadlineTimer.writeCancelCh and +// deadlineTimer.writeTimer. +// +// setDeadline must only be called while holding d.mu. +func (d *deadlineTimer) setDeadline(cancelCh *chan struct{}, timer **time.Timer, t time.Time) { + if *timer != nil && !(*timer).Stop() { + *cancelCh = make(chan struct{}) + } + + // Create a new channel if we already closed it due to setting an already + // expired time. We won't race with the timer because we already handled + // that above. + select { + case <-*cancelCh: + *cancelCh = make(chan struct{}) + default: + } + + // "A zero value for t means I/O operations will not time out." + // - net.Conn.SetDeadline + if t.IsZero() { + *timer = nil + return + } + + timeout := t.Sub(time.Now()) + if timeout <= 0 { + close(*cancelCh) + return + } + + // Timer.Stop returns whether or not the AfterFunc has started, but + // does not indicate whether or not it has completed. Make a copy of + // the cancel channel to prevent this code from racing with the next + // call of setDeadline replacing *cancelCh. + ch := *cancelCh + *timer = time.AfterFunc(timeout, func() { + close(ch) + }) +} + +// SetReadDeadline implements net.Conn.SetReadDeadline and +// net.PacketConn.SetReadDeadline. +func (d *deadlineTimer) SetReadDeadline(t time.Time) error { + d.mu.Lock() + d.setDeadline(&d.readCancelCh, &d.readTimer, t) + d.mu.Unlock() + return nil +} + +// SetWriteDeadline implements net.Conn.SetWriteDeadline and +// net.PacketConn.SetWriteDeadline. +func (d *deadlineTimer) SetWriteDeadline(t time.Time) error { + d.mu.Lock() + d.setDeadline(&d.writeCancelCh, &d.writeTimer, t) + d.mu.Unlock() + return nil +} + +// SetDeadline implements net.Conn.SetDeadline and net.PacketConn.SetDeadline. +func (d *deadlineTimer) SetDeadline(t time.Time) error { + d.mu.Lock() + d.setDeadline(&d.readCancelCh, &d.readTimer, t) + d.setDeadline(&d.writeCancelCh, &d.writeTimer, t) + d.mu.Unlock() + return nil +} + +// A TCPConn is a wrapper around a TCP tcpip.Endpoint that implements the net.Conn +// interface. +type TCPConn struct { + deadlineTimer + + wq *waiter.Queue + ep tcpip.Endpoint + + // readMu serializes reads and implicitly protects read. + // + // Lock ordering: + // If both readMu and deadlineTimer.mu are to be used in a single + // request, readMu must be acquired before deadlineTimer.mu. + readMu sync.Mutex + + // read contains bytes that have been read from the endpoint, + // but haven't yet been returned. + read []byte +} + +// NewTCPConn creates a new TCPConn. +func NewTCPConn(wq *waiter.Queue, ep tcpip.Endpoint) *TCPConn { + c := &TCPConn{ + wq: wq, + ep: ep, + } + c.deadlineTimer.init() + return c +} + +// Accept implements net.Conn.Accept. +func (l *TCPListener) Accept() (net.Conn, error) { + n, wq, err := l.ep.Accept(nil) + + if _, ok := err.(*tcpip.ErrWouldBlock); ok { + // Create wait queue entry that notifies a channel. + waitEntry, notifyCh := waiter.NewChannelEntry(waiter.ReadableEvents) + l.wq.EventRegister(&waitEntry) + defer l.wq.EventUnregister(&waitEntry) + + for { + n, wq, err = l.ep.Accept(nil) + + if _, ok := err.(*tcpip.ErrWouldBlock); !ok { + break + } + + select { + case <-l.cancel: + return nil, errCanceled + case <-notifyCh: + } + } + } + + if err != nil { + return nil, &net.OpError{ + Op: "accept", + Net: "tcp", + Addr: l.Addr(), + Err: TranslateNetstackError(err), + } + } + + return NewTCPConn(wq, n), nil +} + +type opErrorer interface { + newOpError(op string, err error) *net.OpError +} + +// commonRead implements the common logic between net.Conn.Read and +// net.PacketConn.ReadFrom. +func commonRead(b []byte, ep tcpip.Endpoint, wq *waiter.Queue, deadline <-chan struct{}, addr *tcpip.FullAddress, errorer opErrorer) (int, error) { + select { + case <-deadline: + return 0, errorer.newOpError("read", &timeoutError{}) + default: + } + + w := tcpip.SliceWriter(b) + opts := tcpip.ReadOptions{NeedRemoteAddr: addr != nil} + res, err := ep.Read(&w, opts) + + if _, ok := err.(*tcpip.ErrWouldBlock); ok { + // Create wait queue entry that notifies a channel. + waitEntry, notifyCh := waiter.NewChannelEntry(waiter.ReadableEvents) + wq.EventRegister(&waitEntry) + defer wq.EventUnregister(&waitEntry) + for { + res, err = ep.Read(&w, opts) + if _, ok := err.(*tcpip.ErrWouldBlock); !ok { + break + } + select { + case <-deadline: + return 0, errorer.newOpError("read", &timeoutError{}) + case <-notifyCh: + } + } + } + + if _, ok := err.(*tcpip.ErrClosedForReceive); ok { + return 0, io.EOF + } + + if err != nil { + return 0, errorer.newOpError("read", TranslateNetstackError(err)) + } + + if addr != nil { + *addr = res.RemoteAddr + } + return res.Count, nil +} + +// Read implements net.Conn.Read. +func (c *TCPConn) Read(b []byte) (int, error) { + c.readMu.Lock() + defer c.readMu.Unlock() + + deadline := c.readCancel() + + n, err := commonRead(b, c.ep, c.wq, deadline, nil, c) + if n != 0 { + c.ep.ModerateRecvBuf(n) + } + return n, err +} + +// Write implements net.Conn.Write. +func (c *TCPConn) Write(b []byte) (int, error) { + deadline := c.writeCancel() + + // Check if deadlineTimer has already expired. + select { + case <-deadline: + return 0, c.newOpError("write", &timeoutError{}) + default: + } + + // We must handle two soft failure conditions simultaneously: + // 1. Write may write nothing and return *tcpip.ErrWouldBlock. + // If this happens, we need to register for notifications if we have + // not already and wait to try again. + // 2. Write may write fewer than the full number of bytes and return + // without error. In this case we need to try writing the remaining + // bytes again. I do not need to register for notifications. + // + // What is more, these two soft failure conditions can be interspersed. + // There is no guarantee that all of the condition #1s will occur before + // all of the condition #2s or visa-versa. + var ( + r bytes.Reader + nbytes int + entry waiter.Entry + ch <-chan struct{} + ) + for nbytes != len(b) { + r.Reset(b[nbytes:]) + n, err := c.ep.Write(&r, tcpip.WriteOptions{}) + nbytes += int(n) + switch err.(type) { + case nil: + case *tcpip.ErrWouldBlock: + if ch == nil { + entry, ch = waiter.NewChannelEntry(waiter.WritableEvents) + c.wq.EventRegister(&entry) + defer c.wq.EventUnregister(&entry) + } else { + // Don't wait immediately after registration in case more data + // became available between when we last checked and when we setup + // the notification. + select { + case <-deadline: + return nbytes, c.newOpError("write", &timeoutError{}) + case <-ch: + continue + } + } + default: + return nbytes, c.newOpError("write", TranslateNetstackError(err)) + } + } + return nbytes, nil +} + +// Close implements net.Conn.Close. +func (c *TCPConn) Close() error { + c.ep.Close() + return nil +} + +// CloseRead shuts down the reading side of the TCP connection. Most callers +// should just use Close. +// +// A TCP Half-Close is performed the same as CloseRead for *net.TCPConn. +func (c *TCPConn) CloseRead() error { + if terr := c.ep.Shutdown(tcpip.ShutdownRead); terr != nil { + return c.newOpError("close", errors.New(terr.String())) + } + return nil +} + +// CloseWrite shuts down the writing side of the TCP connection. Most callers +// should just use Close. +// +// A TCP Half-Close is performed the same as CloseWrite for *net.TCPConn. +func (c *TCPConn) CloseWrite() error { + if terr := c.ep.Shutdown(tcpip.ShutdownWrite); terr != nil { + return c.newOpError("close", errors.New(terr.String())) + } + return nil +} + +// LocalAddr implements net.Conn.LocalAddr. +func (c *TCPConn) LocalAddr() net.Addr { + a, err := c.ep.GetLocalAddress() + if err != nil { + return nil + } + return fullToTCPAddr(a) +} + +// RemoteAddr implements net.Conn.RemoteAddr. +func (c *TCPConn) RemoteAddr() net.Addr { + a, err := c.ep.GetRemoteAddress() + if err != nil { + return nil + } + return fullToTCPAddr(a) +} + +func (c *TCPConn) newOpError(op string, err error) *net.OpError { + return &net.OpError{ + Op: op, + Net: "tcp", + Source: c.LocalAddr(), + Addr: c.RemoteAddr(), + Err: err, + } +} + +func fullToTCPAddr(addr tcpip.FullAddress) *net.TCPAddr { + return &net.TCPAddr{IP: net.IP(addr.Addr.AsSlice()), Port: int(addr.Port)} +} + +func fullToUDPAddr(addr tcpip.FullAddress) *net.UDPAddr { + return &net.UDPAddr{IP: net.IP(addr.Addr.AsSlice()), Port: int(addr.Port)} +} + +// DialTCP creates a new TCPConn connected to the specified address. +func DialTCP(s *stack.Stack, addr tcpip.FullAddress, network tcpip.NetworkProtocolNumber) (*TCPConn, error) { + return DialContextTCP(context.Background(), s, addr, network) +} + +// DialTCPWithBind creates a new TCPConn connected to the specified +// remoteAddress with its local address bound to localAddr. +func DialTCPWithBind(ctx context.Context, s *stack.Stack, localAddr, remoteAddr tcpip.FullAddress, network tcpip.NetworkProtocolNumber) (*TCPConn, error) { + // Create TCP endpoint, then connect. + var wq waiter.Queue + ep, err := s.NewEndpoint(tcp.ProtocolNumber, network, &wq) + if err != nil { + return nil, TranslateNetstackError(err) + } + + // Create wait queue entry that notifies a channel. + // + // We do this unconditionally as Connect will always return an error. + waitEntry, notifyCh := waiter.NewChannelEntry(waiter.WritableEvents) + wq.EventRegister(&waitEntry) + defer wq.EventUnregister(&waitEntry) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + // Bind before connect if requested. + if localAddr != (tcpip.FullAddress{}) { + if err = ep.Bind(localAddr); err != nil { + return nil, fmt.Errorf("ep.Bind(%+v) = %s", localAddr, err) + } + } + + err = ep.Connect(remoteAddr) + if _, ok := err.(*tcpip.ErrConnectStarted); ok { + select { + case <-ctx.Done(): + ep.Close() + return nil, ctx.Err() + case <-notifyCh: + } + + err = ep.LastError() + } + if err != nil { + ep.Close() + return nil, &net.OpError{ + Op: "connect", + Net: "tcp", + Addr: fullToTCPAddr(remoteAddr), + Err: TranslateNetstackError(err), + } + } + + return NewTCPConn(&wq, ep), nil +} + +// DialContextTCP creates a new TCPConn connected to the specified address +// with the option of adding cancellation and timeouts. +func DialContextTCP(ctx context.Context, s *stack.Stack, addr tcpip.FullAddress, network tcpip.NetworkProtocolNumber) (*TCPConn, error) { + return DialTCPWithBind(ctx, s, tcpip.FullAddress{} /* localAddr */, addr /* remoteAddr */, network) +} + +// A UDPConn is a wrapper around a UDP tcpip.Endpoint that implements +// net.Conn and net.PacketConn. +type UDPConn struct { + deadlineTimer + + ep tcpip.Endpoint + wq *waiter.Queue +} + +// NewUDPConn creates a new UDPConn. +func NewUDPConn(wq *waiter.Queue, ep tcpip.Endpoint) *UDPConn { + c := &UDPConn{ + ep: ep, + wq: wq, + } + c.deadlineTimer.init() + return c +} + +// DialUDP creates a new UDPConn. +// +// If laddr is nil, a local address is automatically chosen. +// +// If raddr is nil, the UDPConn is left unconnected. +func DialUDP(s *stack.Stack, laddr, raddr *tcpip.FullAddress, network tcpip.NetworkProtocolNumber) (*UDPConn, error) { + var wq waiter.Queue + ep, err := s.NewEndpoint(udp.ProtocolNumber, network, &wq) + if err != nil { + return nil, TranslateNetstackError(err) + } + + if laddr != nil { + if err := ep.Bind(*laddr); err != nil { + ep.Close() + return nil, &net.OpError{ + Op: "bind", + Net: "udp", + Addr: fullToUDPAddr(*laddr), + Err: TranslateNetstackError(err), + } + } + } + + c := NewUDPConn(&wq, ep) + + if raddr != nil { + if err := c.ep.Connect(*raddr); err != nil { + c.ep.Close() + return nil, &net.OpError{ + Op: "connect", + Net: "udp", + Addr: fullToUDPAddr(*raddr), + Err: TranslateNetstackError(err), + } + } + } + + return c, nil +} + +func (c *UDPConn) newOpError(op string, err error) *net.OpError { + return c.newRemoteOpError(op, nil, err) +} + +func (c *UDPConn) newRemoteOpError(op string, remote net.Addr, err error) *net.OpError { + return &net.OpError{ + Op: op, + Net: "udp", + Source: c.LocalAddr(), + Addr: remote, + Err: err, + } +} + +// RemoteAddr implements net.Conn.RemoteAddr. +func (c *UDPConn) RemoteAddr() net.Addr { + a, err := c.ep.GetRemoteAddress() + if err != nil { + return nil + } + return fullToUDPAddr(a) +} + +// Read implements net.Conn.Read +func (c *UDPConn) Read(b []byte) (int, error) { + bytesRead, _, err := c.ReadFrom(b) + return bytesRead, err +} + +// ReadFrom implements net.PacketConn.ReadFrom. +func (c *UDPConn) ReadFrom(b []byte) (int, net.Addr, error) { + deadline := c.readCancel() + + var addr tcpip.FullAddress + n, err := commonRead(b, c.ep, c.wq, deadline, &addr, c) + if err != nil { + return 0, nil, err + } + return n, fullToUDPAddr(addr), nil +} + +func (c *UDPConn) Write(b []byte) (int, error) { + return c.WriteTo(b, nil) +} + +// WriteTo implements net.PacketConn.WriteTo. +func (c *UDPConn) WriteTo(b []byte, addr net.Addr) (int, error) { + deadline := c.writeCancel() + + // Check if deadline has already expired. + select { + case <-deadline: + return 0, c.newRemoteOpError("write", addr, &timeoutError{}) + default: + } + + // If we're being called by Write, there is no addr + writeOptions := tcpip.WriteOptions{} + if addr != nil { + ua := addr.(*net.UDPAddr) + writeOptions.To = &tcpip.FullAddress{ + Addr: tcpip.AddrFromSlice(ua.IP), + Port: uint16(ua.Port), + } + } + + var r bytes.Reader + r.Reset(b) + n, err := c.ep.Write(&r, writeOptions) + if _, ok := err.(*tcpip.ErrWouldBlock); ok { + // Create wait queue entry that notifies a channel. + waitEntry, notifyCh := waiter.NewChannelEntry(waiter.WritableEvents) + c.wq.EventRegister(&waitEntry) + defer c.wq.EventUnregister(&waitEntry) + for { + select { + case <-deadline: + return int(n), c.newRemoteOpError("write", addr, &timeoutError{}) + case <-notifyCh: + } + + n, err = c.ep.Write(&r, writeOptions) + if _, ok := err.(*tcpip.ErrWouldBlock); !ok { + break + } + } + } + + if err == nil { + return int(n), nil + } + + return int(n), c.newRemoteOpError("write", addr, TranslateNetstackError(err)) +} + +// Close implements net.PacketConn.Close. +func (c *UDPConn) Close() error { + c.ep.Close() + return nil +} + +// LocalAddr implements net.PacketConn.LocalAddr. +func (c *UDPConn) LocalAddr() net.Addr { + a, err := c.ep.GetLocalAddress() + if err != nil { + return nil + } + return fullToUDPAddr(a) +} diff --git a/pkg/tcpip/adapters/gonet/gonet_state_autogen.go b/pkg/tcpip/adapters/gonet/gonet_state_autogen.go new file mode 100644 index 0000000..7a5c541 --- /dev/null +++ b/pkg/tcpip/adapters/gonet/gonet_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package gonet diff --git a/pkg/tcpip/checksum/checksum.go b/pkg/tcpip/checksum/checksum.go new file mode 100644 index 0000000..5d4e117 --- /dev/null +++ b/pkg/tcpip/checksum/checksum.go @@ -0,0 +1,68 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package checksum provides the implementation of the encoding and decoding of +// network protocol headers. +package checksum + +import ( + "encoding/binary" +) + +// Size is the size of a checksum. +// +// The checksum is held in a uint16 which is 2 bytes. +const Size = 2 + +// Put puts the checksum in the provided byte slice. +func Put(b []byte, xsum uint16) { + binary.BigEndian.PutUint16(b, xsum) +} + +// Checksum calculates the checksum (as defined in RFC 1071) of the bytes in the +// given byte array. This function uses an optimized version of the checksum +// algorithm. +// +// The initial checksum must have been computed on an even number of bytes. +func Checksum(buf []byte, initial uint16) uint16 { + s, _ := calculateChecksum(buf, false, initial) + return s +} + +// Checksumer calculates checksum defined in RFC 1071. +type Checksumer struct { + sum uint16 + odd bool +} + +// Add adds b to checksum. +func (c *Checksumer) Add(b []byte) { + if len(b) > 0 { + c.sum, c.odd = calculateChecksum(b, c.odd, c.sum) + } +} + +// Checksum returns the latest checksum value. +func (c *Checksumer) Checksum() uint16 { + return c.sum +} + +// Combine combines the two uint16 to form their checksum. This is done +// by adding them and the carry. +// +// Note that checksum a must have been computed on an even number of bytes. +func Combine(a, b uint16) uint16 { + v := uint32(a) + uint32(b) + return uint16(v + v>>16) +} diff --git a/pkg/tcpip/checksum/checksum_state_autogen.go b/pkg/tcpip/checksum/checksum_state_autogen.go new file mode 100644 index 0000000..936aef7 --- /dev/null +++ b/pkg/tcpip/checksum/checksum_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package checksum diff --git a/pkg/tcpip/checksum/checksum_unsafe.go b/pkg/tcpip/checksum/checksum_unsafe.go new file mode 100644 index 0000000..66b7ab6 --- /dev/null +++ b/pkg/tcpip/checksum/checksum_unsafe.go @@ -0,0 +1,182 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checksum + +import ( + "encoding/binary" + "math/bits" + "unsafe" +) + +// Note: odd indicates whether initial is a partial checksum over an odd number +// of bytes. +func calculateChecksum(buf []byte, odd bool, initial uint16) (uint16, bool) { + // Use a larger-than-uint16 accumulator to benefit from parallel summation + // as described in RFC 1071 1.2.C. + acc := uint64(initial) + + // Handle an odd number of previously-summed bytes, and get the return + // value for odd. + if odd { + acc += uint64(buf[0]) + buf = buf[1:] + } + odd = len(buf)&1 != 0 + + // Aligning &buf[0] below is much simpler if len(buf) >= 8; special-case + // smaller bufs. + if len(buf) < 8 { + if len(buf) >= 4 { + acc += (uint64(buf[0]) << 8) + uint64(buf[1]) + acc += (uint64(buf[2]) << 8) + uint64(buf[3]) + buf = buf[4:] + } + if len(buf) >= 2 { + acc += (uint64(buf[0]) << 8) + uint64(buf[1]) + buf = buf[2:] + } + if len(buf) >= 1 { + acc += uint64(buf[0]) << 8 + // buf = buf[1:] is skipped because it's unused and nogo will + // complain. + } + return reduce(acc), odd + } + + // On little-endian architectures, multi-byte loads from buf will load + // bytes in the wrong order. Rather than byte-swap after each load (slow), + // we byte-swap the accumulator before summing any bytes and byte-swap it + // back before returning, which still produces the correct result as + // described in RFC 1071 1.2.B "Byte Order Independence". + // + // acc is at most a uint16 + a uint8, so its upper 32 bits must be 0s. We + // preserve this property by byte-swapping only the lower 32 bits of acc, + // so that additions to acc performed during alignment can't overflow. + acc = uint64(bswapIfLittleEndian32(uint32(acc))) + + // Align &buf[0] to an 8-byte boundary. + bswapped := false + if sliceAddr(buf)&1 != 0 { + // Compute the rest of the partial checksum with bytes swapped, and + // swap back before returning; see the last paragraph of + // RFC 1071 1.2.B. + acc = uint64(bits.ReverseBytes32(uint32(acc))) + bswapped = true + // No `<< 8` here due to the byte swap we just did. + acc += uint64(bswapIfLittleEndian16(uint16(buf[0]))) + buf = buf[1:] + } + if sliceAddr(buf)&2 != 0 { + acc += uint64(*(*uint16)(unsafe.Pointer(&buf[0]))) + buf = buf[2:] + } + if sliceAddr(buf)&4 != 0 { + acc += uint64(*(*uint32)(unsafe.Pointer(&buf[0]))) + buf = buf[4:] + } + + // Sum 64 bytes at a time. Beyond this point, additions to acc may + // overflow, so we have to handle carrying. + for len(buf) >= 64 { + var carry uint64 + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[0])), 0) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[8])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[16])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[24])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[32])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[40])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[48])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[56])), carry) + acc, _ = bits.Add64(acc, 0, carry) + buf = buf[64:] + } + + // Sum the remaining 0-63 bytes. + if len(buf) >= 32 { + var carry uint64 + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[0])), 0) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[8])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[16])), carry) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[24])), carry) + acc, _ = bits.Add64(acc, 0, carry) + buf = buf[32:] + } + if len(buf) >= 16 { + var carry uint64 + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[0])), 0) + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[8])), carry) + acc, _ = bits.Add64(acc, 0, carry) + buf = buf[16:] + } + if len(buf) >= 8 { + var carry uint64 + acc, carry = bits.Add64(acc, *(*uint64)(unsafe.Pointer(&buf[0])), 0) + acc, _ = bits.Add64(acc, 0, carry) + buf = buf[8:] + } + if len(buf) >= 4 { + var carry uint64 + acc, carry = bits.Add64(acc, uint64(*(*uint32)(unsafe.Pointer(&buf[0]))), 0) + acc, _ = bits.Add64(acc, 0, carry) + buf = buf[4:] + } + if len(buf) >= 2 { + var carry uint64 + acc, carry = bits.Add64(acc, uint64(*(*uint16)(unsafe.Pointer(&buf[0]))), 0) + acc, _ = bits.Add64(acc, 0, carry) + buf = buf[2:] + } + if len(buf) >= 1 { + // bswapIfBigEndian16(buf[0]) == bswapIfLittleEndian16(buf[0]<<8). + var carry uint64 + acc, carry = bits.Add64(acc, uint64(bswapIfBigEndian16(uint16(buf[0]))), 0) + acc, _ = bits.Add64(acc, 0, carry) + // buf = buf[1:] is skipped because it's unused and nogo will complain. + } + + // Reduce the checksum to 16 bits and undo byte swaps before returning. + acc16 := bswapIfLittleEndian16(reduce(acc)) + if bswapped { + acc16 = bits.ReverseBytes16(acc16) + } + return acc16, odd +} + +func reduce(acc uint64) uint16 { + // Ideally we would do: + // return uint16(acc>>48) +' uint16(acc>>32) +' uint16(acc>>16) +' uint16(acc) + // for more instruction-level parallelism; however, there is no + // bits.Add16(). + acc = (acc >> 32) + (acc & 0xffff_ffff) // at most 0x1_ffff_fffe + acc32 := uint32(acc>>32 + acc) // at most 0xffff_ffff + acc32 = (acc32 >> 16) + (acc32 & 0xffff) // at most 0x1_fffe + return uint16(acc32>>16 + acc32) // at most 0xffff +} + +func bswapIfLittleEndian32(val uint32) uint32 { + return binary.BigEndian.Uint32((*[4]byte)(unsafe.Pointer(&val))[:]) +} + +func bswapIfLittleEndian16(val uint16) uint16 { + return binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&val))[:]) +} + +func bswapIfBigEndian16(val uint16) uint16 { + return binary.LittleEndian.Uint16((*[2]byte)(unsafe.Pointer(&val))[:]) +} + +func sliceAddr(buf []byte) uintptr { + return uintptr(unsafe.Pointer(unsafe.SliceData(buf))) +} diff --git a/pkg/tcpip/checksum/checksum_unsafe_state_autogen.go b/pkg/tcpip/checksum/checksum_unsafe_state_autogen.go new file mode 100644 index 0000000..936aef7 --- /dev/null +++ b/pkg/tcpip/checksum/checksum_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package checksum diff --git a/pkg/tcpip/errors.go b/pkg/tcpip/errors.go new file mode 100644 index 0000000..9f6e63e --- /dev/null +++ b/pkg/tcpip/errors.go @@ -0,0 +1,643 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpip + +import ( + "fmt" +) + +// Error represents an error in the netstack error space. +// +// The error interface is intentionally omitted to avoid loss of type +// information that would occur if these errors were passed as error. +type Error interface { + isError() + + // IgnoreStats indicates whether this error should be included in failure + // counts in tcpip.Stats structs. + IgnoreStats() bool + + fmt.Stringer +} + +const maxErrno = 134 + +// LINT.IfChange + +// ErrAborted indicates the operation was aborted. +// +// +stateify savable +type ErrAborted struct{} + +func (*ErrAborted) isError() {} + +// IgnoreStats implements Error. +func (*ErrAborted) IgnoreStats() bool { + return false +} + +func (*ErrAborted) String() string { + return "operation aborted" +} + +// ErrAddressFamilyNotSupported indicates the operation does not support the +// given address family. +// +// +stateify savable +type ErrAddressFamilyNotSupported struct{} + +func (*ErrAddressFamilyNotSupported) isError() {} + +// IgnoreStats implements Error. +func (*ErrAddressFamilyNotSupported) IgnoreStats() bool { + return false +} + +func (*ErrAddressFamilyNotSupported) String() string { + return "address family not supported by protocol" +} + +// ErrAlreadyBound indicates the endpoint is already bound. +// +// +stateify savable +type ErrAlreadyBound struct{} + +func (*ErrAlreadyBound) isError() {} + +// IgnoreStats implements Error. +func (*ErrAlreadyBound) IgnoreStats() bool { + return true +} +func (*ErrAlreadyBound) String() string { return "endpoint already bound" } + +// ErrAlreadyConnected indicates the endpoint is already connected. +// +// +stateify savable +type ErrAlreadyConnected struct{} + +func (*ErrAlreadyConnected) isError() {} + +// IgnoreStats implements Error. +func (*ErrAlreadyConnected) IgnoreStats() bool { + return true +} +func (*ErrAlreadyConnected) String() string { return "endpoint is already connected" } + +// ErrAlreadyConnecting indicates the endpoint is already connecting. +// +// +stateify savable +type ErrAlreadyConnecting struct{} + +func (*ErrAlreadyConnecting) isError() {} + +// IgnoreStats implements Error. +func (*ErrAlreadyConnecting) IgnoreStats() bool { + return true +} +func (*ErrAlreadyConnecting) String() string { return "endpoint is already connecting" } + +// ErrBadAddress indicates a bad address was provided. +// +// +stateify savable +type ErrBadAddress struct{} + +func (*ErrBadAddress) isError() {} + +// IgnoreStats implements Error. +func (*ErrBadAddress) IgnoreStats() bool { + return false +} +func (*ErrBadAddress) String() string { return "bad address" } + +// ErrBadBuffer indicates a bad buffer was provided. +// +// +stateify savable +type ErrBadBuffer struct{} + +func (*ErrBadBuffer) isError() {} + +// IgnoreStats implements Error. +func (*ErrBadBuffer) IgnoreStats() bool { + return false +} +func (*ErrBadBuffer) String() string { return "bad buffer" } + +// ErrBadLocalAddress indicates a bad local address was provided. +// +// +stateify savable +type ErrBadLocalAddress struct{} + +func (*ErrBadLocalAddress) isError() {} + +// IgnoreStats implements Error. +func (*ErrBadLocalAddress) IgnoreStats() bool { + return false +} +func (*ErrBadLocalAddress) String() string { return "bad local address" } + +// ErrBroadcastDisabled indicates broadcast is not enabled on the endpoint. +// +// +stateify savable +type ErrBroadcastDisabled struct{} + +func (*ErrBroadcastDisabled) isError() {} + +// IgnoreStats implements Error. +func (*ErrBroadcastDisabled) IgnoreStats() bool { + return false +} +func (*ErrBroadcastDisabled) String() string { return "broadcast socket option disabled" } + +// ErrClosedForReceive indicates the endpoint is closed for incoming data. +// +// +stateify savable +type ErrClosedForReceive struct{} + +func (*ErrClosedForReceive) isError() {} + +// IgnoreStats implements Error. +func (*ErrClosedForReceive) IgnoreStats() bool { + return false +} +func (*ErrClosedForReceive) String() string { return "endpoint is closed for receive" } + +// ErrClosedForSend indicates the endpoint is closed for outgoing data. +// +// +stateify savable +type ErrClosedForSend struct{} + +func (*ErrClosedForSend) isError() {} + +// IgnoreStats implements Error. +func (*ErrClosedForSend) IgnoreStats() bool { + return false +} +func (*ErrClosedForSend) String() string { return "endpoint is closed for send" } + +// ErrConnectStarted indicates the endpoint is connecting asynchronously. +// +// +stateify savable +type ErrConnectStarted struct{} + +func (*ErrConnectStarted) isError() {} + +// IgnoreStats implements Error. +func (*ErrConnectStarted) IgnoreStats() bool { + return true +} +func (*ErrConnectStarted) String() string { return "connection attempt started" } + +// ErrConnectionAborted indicates the connection was aborted. +// +// +stateify savable +type ErrConnectionAborted struct{} + +func (*ErrConnectionAborted) isError() {} + +// IgnoreStats implements Error. +func (*ErrConnectionAborted) IgnoreStats() bool { + return false +} +func (*ErrConnectionAborted) String() string { return "connection aborted" } + +// ErrConnectionRefused indicates the connection was refused. +// +// +stateify savable +type ErrConnectionRefused struct{} + +func (*ErrConnectionRefused) isError() {} + +// IgnoreStats implements Error. +func (*ErrConnectionRefused) IgnoreStats() bool { + return false +} +func (*ErrConnectionRefused) String() string { return "connection was refused" } + +// ErrConnectionReset indicates the connection was reset. +// +// +stateify savable +type ErrConnectionReset struct{} + +func (*ErrConnectionReset) isError() {} + +// IgnoreStats implements Error. +func (*ErrConnectionReset) IgnoreStats() bool { + return false +} +func (*ErrConnectionReset) String() string { return "connection reset by peer" } + +// ErrDestinationRequired indicates the operation requires a destination +// address, and one was not provided. +// +// +stateify savable +type ErrDestinationRequired struct{} + +func (*ErrDestinationRequired) isError() {} + +// IgnoreStats implements Error. +func (*ErrDestinationRequired) IgnoreStats() bool { + return false +} +func (*ErrDestinationRequired) String() string { return "destination address is required" } + +// ErrDuplicateAddress indicates the operation encountered a duplicate address. +// +// +stateify savable +type ErrDuplicateAddress struct{} + +func (*ErrDuplicateAddress) isError() {} + +// IgnoreStats implements Error. +func (*ErrDuplicateAddress) IgnoreStats() bool { + return false +} +func (*ErrDuplicateAddress) String() string { return "duplicate address" } + +// ErrDuplicateNICID indicates the operation encountered a duplicate NIC ID. +// +// +stateify savable +type ErrDuplicateNICID struct{} + +func (*ErrDuplicateNICID) isError() {} + +// IgnoreStats implements Error. +func (*ErrDuplicateNICID) IgnoreStats() bool { + return false +} +func (*ErrDuplicateNICID) String() string { return "duplicate nic id" } + +// ErrInvalidNICID indicates the operation used an invalid NIC ID. +// +// +stateify savable +type ErrInvalidNICID struct{} + +func (*ErrInvalidNICID) isError() {} + +// IgnoreStats implements Error. +func (*ErrInvalidNICID) IgnoreStats() bool { + return false +} +func (*ErrInvalidNICID) String() string { return "invalid nic id" } + +// ErrInvalidEndpointState indicates the endpoint is in an invalid state. +// +// +stateify savable +type ErrInvalidEndpointState struct{} + +func (*ErrInvalidEndpointState) isError() {} + +// IgnoreStats implements Error. +func (*ErrInvalidEndpointState) IgnoreStats() bool { + return false +} +func (*ErrInvalidEndpointState) String() string { return "endpoint is in invalid state" } + +// ErrInvalidOptionValue indicates an invalid option value was provided. +// +// +stateify savable +type ErrInvalidOptionValue struct{} + +func (*ErrInvalidOptionValue) isError() {} + +// IgnoreStats implements Error. +func (*ErrInvalidOptionValue) IgnoreStats() bool { + return false +} +func (*ErrInvalidOptionValue) String() string { return "invalid option value specified" } + +// ErrInvalidPortRange indicates an attempt to set an invalid port range. +// +// +stateify savable +type ErrInvalidPortRange struct{} + +func (*ErrInvalidPortRange) isError() {} + +// IgnoreStats implements Error. +func (*ErrInvalidPortRange) IgnoreStats() bool { + return true +} +func (*ErrInvalidPortRange) String() string { return "invalid port range" } + +// ErrMalformedHeader indicates the operation encountered a malformed header. +// +// +stateify savable +type ErrMalformedHeader struct{} + +func (*ErrMalformedHeader) isError() {} + +// IgnoreStats implements Error. +func (*ErrMalformedHeader) IgnoreStats() bool { + return false +} +func (*ErrMalformedHeader) String() string { return "header is malformed" } + +// ErrMessageTooLong indicates the operation encountered a message whose length +// exceeds the maximum permitted. +// +// +stateify savable +type ErrMessageTooLong struct{} + +func (*ErrMessageTooLong) isError() {} + +// IgnoreStats implements Error. +func (*ErrMessageTooLong) IgnoreStats() bool { + return false +} +func (*ErrMessageTooLong) String() string { return "message too long" } + +// ErrNetworkUnreachable indicates the operation is not able to reach the +// destination network. +// +// +stateify savable +type ErrNetworkUnreachable struct{} + +func (*ErrNetworkUnreachable) isError() {} + +// IgnoreStats implements Error. +func (*ErrNetworkUnreachable) IgnoreStats() bool { + return false +} +func (*ErrNetworkUnreachable) String() string { return "network is unreachable" } + +// ErrNoBufferSpace indicates no buffer space is available. +// +// +stateify savable +type ErrNoBufferSpace struct{} + +func (*ErrNoBufferSpace) isError() {} + +// IgnoreStats implements Error. +func (*ErrNoBufferSpace) IgnoreStats() bool { + return false +} +func (*ErrNoBufferSpace) String() string { return "no buffer space available" } + +// ErrNoPortAvailable indicates no port could be allocated for the operation. +// +// +stateify savable +type ErrNoPortAvailable struct{} + +func (*ErrNoPortAvailable) isError() {} + +// IgnoreStats implements Error. +func (*ErrNoPortAvailable) IgnoreStats() bool { + return false +} +func (*ErrNoPortAvailable) String() string { return "no ports are available" } + +// ErrHostUnreachable indicates that a destination host could not be +// reached. +// +// +stateify savable +type ErrHostUnreachable struct{} + +func (*ErrHostUnreachable) isError() {} + +// IgnoreStats implements Error. +func (*ErrHostUnreachable) IgnoreStats() bool { + return false +} +func (*ErrHostUnreachable) String() string { return "no route to host" } + +// ErrHostDown indicates that a destination host is down. +// +// +stateify savable +type ErrHostDown struct{} + +func (*ErrHostDown) isError() {} + +// IgnoreStats implements Error. +func (*ErrHostDown) IgnoreStats() bool { + return false +} +func (*ErrHostDown) String() string { return "host is down" } + +// ErrNoNet indicates that the host is not on the network. +// +// +stateify savable +type ErrNoNet struct{} + +func (*ErrNoNet) isError() {} + +// IgnoreStats implements Error. +func (*ErrNoNet) IgnoreStats() bool { + return false +} +func (*ErrNoNet) String() string { return "machine is not on the network" } + +// ErrNoSuchFile is used to indicate that ENOENT should be returned the to +// calling application. +// +// +stateify savable +type ErrNoSuchFile struct{} + +func (*ErrNoSuchFile) isError() {} + +// IgnoreStats implements Error. +func (*ErrNoSuchFile) IgnoreStats() bool { + return false +} +func (*ErrNoSuchFile) String() string { return "no such file" } + +// ErrNotConnected indicates the endpoint is not connected. +// +// +stateify savable +type ErrNotConnected struct{} + +func (*ErrNotConnected) isError() {} + +// IgnoreStats implements Error. +func (*ErrNotConnected) IgnoreStats() bool { + return false +} +func (*ErrNotConnected) String() string { return "endpoint not connected" } + +// ErrNotPermitted indicates the operation is not permitted. +// +// +stateify savable +type ErrNotPermitted struct{} + +func (*ErrNotPermitted) isError() {} + +// IgnoreStats implements Error. +func (*ErrNotPermitted) IgnoreStats() bool { + return false +} +func (*ErrNotPermitted) String() string { return "operation not permitted" } + +// ErrNotSupported indicates the operation is not supported. +// +// +stateify savable +type ErrNotSupported struct{} + +func (*ErrNotSupported) isError() {} + +// IgnoreStats implements Error. +func (*ErrNotSupported) IgnoreStats() bool { + return false +} +func (*ErrNotSupported) String() string { return "operation not supported" } + +// ErrPortInUse indicates the provided port is in use. +// +// +stateify savable +type ErrPortInUse struct{} + +func (*ErrPortInUse) isError() {} + +// IgnoreStats implements Error. +func (*ErrPortInUse) IgnoreStats() bool { + return false +} +func (*ErrPortInUse) String() string { return "port is in use" } + +// ErrQueueSizeNotSupported indicates the endpoint does not allow queue size +// operation. +// +// +stateify savable +type ErrQueueSizeNotSupported struct{} + +func (*ErrQueueSizeNotSupported) isError() {} + +// IgnoreStats implements Error. +func (*ErrQueueSizeNotSupported) IgnoreStats() bool { + return false +} +func (*ErrQueueSizeNotSupported) String() string { return "queue size querying not supported" } + +// ErrTimeout indicates the operation timed out. +// +// +stateify savable +type ErrTimeout struct{} + +func (*ErrTimeout) isError() {} + +// IgnoreStats implements Error. +func (*ErrTimeout) IgnoreStats() bool { + return false +} +func (*ErrTimeout) String() string { return "operation timed out" } + +// ErrUnknownDevice indicates an unknown device identifier was provided. +// +// +stateify savable +type ErrUnknownDevice struct{} + +func (*ErrUnknownDevice) isError() {} + +// IgnoreStats implements Error. +func (*ErrUnknownDevice) IgnoreStats() bool { + return false +} +func (*ErrUnknownDevice) String() string { return "unknown device" } + +// ErrUnknownNICID indicates an unknown NIC ID was provided. +// +// +stateify savable +type ErrUnknownNICID struct{} + +func (*ErrUnknownNICID) isError() {} + +// IgnoreStats implements Error. +func (*ErrUnknownNICID) IgnoreStats() bool { + return false +} +func (*ErrUnknownNICID) String() string { return "unknown nic id" } + +// ErrUnknownProtocol indicates an unknown protocol was requested. +// +// +stateify savable +type ErrUnknownProtocol struct{} + +func (*ErrUnknownProtocol) isError() {} + +// IgnoreStats implements Error. +func (*ErrUnknownProtocol) IgnoreStats() bool { + return false +} +func (*ErrUnknownProtocol) String() string { return "unknown protocol" } + +// ErrUnknownProtocolOption indicates an unknown protocol option was provided. +// +// +stateify savable +type ErrUnknownProtocolOption struct{} + +func (*ErrUnknownProtocolOption) isError() {} + +// IgnoreStats implements Error. +func (*ErrUnknownProtocolOption) IgnoreStats() bool { + return false +} +func (*ErrUnknownProtocolOption) String() string { return "unknown option for protocol" } + +// ErrWouldBlock indicates the operation would block. +// +// +stateify savable +type ErrWouldBlock struct{} + +func (*ErrWouldBlock) isError() {} + +// IgnoreStats implements Error. +func (*ErrWouldBlock) IgnoreStats() bool { + return true +} +func (*ErrWouldBlock) String() string { return "operation would block" } + +// ErrMissingRequiredFields indicates that a required field is missing. +// +// +stateify savable +type ErrMissingRequiredFields struct{} + +func (*ErrMissingRequiredFields) isError() {} + +// IgnoreStats implements Error. +func (*ErrMissingRequiredFields) IgnoreStats() bool { + return true +} +func (*ErrMissingRequiredFields) String() string { return "missing required fields" } + +// ErrMulticastInputCannotBeOutput indicates that an input interface matches an +// output interface in the same multicast route. +// +// +stateify savable +type ErrMulticastInputCannotBeOutput struct{} + +func (*ErrMulticastInputCannotBeOutput) isError() {} + +// IgnoreStats implements Error. +func (*ErrMulticastInputCannotBeOutput) IgnoreStats() bool { + return true +} +func (*ErrMulticastInputCannotBeOutput) String() string { return "output cannot contain input" } + +// ErrEndpointBusy indicates that the operation cannot be completed because the +// endpoint is busy. +// +// +stateify savable +type ErrEndpointBusy struct{} + +// isError implements Error. +func (*ErrEndpointBusy) isError() {} + +// IgnoreStats implements Error. +func (*ErrEndpointBusy) IgnoreStats() bool { + return true +} + +func (*ErrEndpointBusy) String() string { + return "operation cannot be completed because the endpoint is busy" +} + +// LINT.ThenChange(../syserr/netstack.go) diff --git a/pkg/tcpip/errors_linux.go b/pkg/tcpip/errors_linux.go new file mode 100644 index 0000000..0073568 --- /dev/null +++ b/pkg/tcpip/errors_linux.go @@ -0,0 +1,74 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package tcpip + +import ( + "golang.org/x/sys/unix" +) + +// TranslateErrno translate an errno from the syscall package into a +// tcpip Error. +// +// Valid, but unrecognized errnos will be translated to +// *ErrInvalidEndpointState (EINVAL). This includes the "zero" value. +func TranslateErrno(e unix.Errno) Error { + switch e { + case unix.EEXIST: + return &ErrDuplicateAddress{} + case unix.ENETUNREACH: + return &ErrHostUnreachable{} + case unix.EINVAL: + return &ErrInvalidEndpointState{} + case unix.EALREADY: + return &ErrAlreadyConnecting{} + case unix.EISCONN: + return &ErrAlreadyConnected{} + case unix.EADDRINUSE: + return &ErrPortInUse{} + case unix.EADDRNOTAVAIL: + return &ErrBadLocalAddress{} + case unix.EPIPE: + return &ErrClosedForSend{} + case unix.EWOULDBLOCK: + return &ErrWouldBlock{} + case unix.ECONNREFUSED: + return &ErrConnectionRefused{} + case unix.ETIMEDOUT: + return &ErrTimeout{} + case unix.EINPROGRESS: + return &ErrConnectStarted{} + case unix.EDESTADDRREQ: + return &ErrDestinationRequired{} + case unix.ENOTSUP: + return &ErrNotSupported{} + case unix.ENOTTY: + return &ErrQueueSizeNotSupported{} + case unix.ENOTCONN: + return &ErrNotConnected{} + case unix.ECONNRESET: + return &ErrConnectionReset{} + case unix.ECONNABORTED: + return &ErrConnectionAborted{} + case unix.EMSGSIZE: + return &ErrMessageTooLong{} + case unix.ENOBUFS: + return &ErrNoBufferSpace{} + default: + return &ErrInvalidEndpointState{} + } +} diff --git a/pkg/tcpip/faketime/faketime.go b/pkg/tcpip/faketime/faketime.go new file mode 100644 index 0000000..c28cffa --- /dev/null +++ b/pkg/tcpip/faketime/faketime.go @@ -0,0 +1,392 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package faketime provides a fake clock that implements tcpip.Clock interface. +package faketime + +import ( + "container/heap" + "fmt" + "sync" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// NullClock implements a clock that never advances. +// +// +stateify savable +type NullClock struct{} + +var _ tcpip.Clock = (*NullClock)(nil) + +// Now implements tcpip.Clock.Now. +func (*NullClock) Now() time.Time { + return time.Time{} +} + +// NowMonotonic implements tcpip.Clock.NowMonotonic. +func (*NullClock) NowMonotonic() tcpip.MonotonicTime { + return tcpip.MonotonicTime{} +} + +// nullTimer implements a timer that never fires. +// +// +stateify savable +type nullTimer struct{} + +var _ tcpip.Timer = (*nullTimer)(nil) + +// Stop implements tcpip.Timer. +func (*nullTimer) Stop() bool { + return true +} + +// Reset implements tcpip.Timer. +func (*nullTimer) Reset(time.Duration) {} + +// AfterFunc implements tcpip.Clock.AfterFunc. +func (*NullClock) AfterFunc(time.Duration, func()) tcpip.Timer { + return &nullTimer{} +} + +type notificationChannels struct { + mu struct { + sync.Mutex + + ch []<-chan struct{} + } +} + +func (n *notificationChannels) add(ch <-chan struct{}) { + n.mu.Lock() + defer n.mu.Unlock() + n.mu.ch = append(n.mu.ch, ch) +} + +// wait returns once all the notification channels are readable. +// +// Channels that are added while waiting on existing channels will be waited on +// as well. +func (n *notificationChannels) wait() { + for { + n.mu.Lock() + ch := n.mu.ch + n.mu.ch = nil + n.mu.Unlock() + + if len(ch) == 0 { + break + } + + for _, c := range ch { + <-c + } + } +} + +// +stateify savable +type manualClockMutex struct { + sync.RWMutex `state:"nosave"` + + // now is the current (fake) time of the clock. + now time.Time + + // times is min-heap of times. + times timeHeap + + // timers holds the timers scheduled for each time. + timers map[time.Time]map[*manualTimer]struct{} +} + +// ManualClock implements tcpip.Clock and only advances manually with Advance +// method. +// +// +stateify savable +type ManualClock struct { + // runningTimers tracks the completion of timer callbacks that began running + // immediately upon their scheduling. It is used to ensure the proper ordering + // of timer callback dispatch. + runningTimers notificationChannels + + mu manualClockMutex +} + +// NewManualClock creates a new ManualClock instance. +func NewManualClock() *ManualClock { + c := &ManualClock{} + + c.mu.Lock() + defer c.mu.Unlock() + + // Set the initial time to a non-zero value since the zero value is used to + // detect inactive timers. + c.mu.now = time.Unix(0, 0) + c.mu.timers = make(map[time.Time]map[*manualTimer]struct{}) + + return c +} + +var _ tcpip.Clock = (*ManualClock)(nil) + +// Now implements tcpip.Clock.Now. +func (mc *ManualClock) Now() time.Time { + mc.mu.RLock() + defer mc.mu.RUnlock() + return mc.mu.now +} + +// NowMonotonic implements tcpip.Clock.NowMonotonic. +func (mc *ManualClock) NowMonotonic() tcpip.MonotonicTime { + var mt tcpip.MonotonicTime + return mt.Add(mc.Now().Sub(time.Unix(0, 0))) +} + +// AfterFunc implements tcpip.Clock.AfterFunc. +func (mc *ManualClock) AfterFunc(d time.Duration, f func()) tcpip.Timer { + mt := &manualTimer{ + clock: mc, + f: f, + } + + mc.mu.Lock() + defer mc.mu.Unlock() + + mt.mu.Lock() + defer mt.mu.Unlock() + + mc.resetTimerLocked(mt, d) + return mt +} + +// resetTimerLocked schedules a timer to be fired after the given duration. +// +// Precondition: mc.mu and mt.mu must be locked. +func (mc *ManualClock) resetTimerLocked(mt *manualTimer, d time.Duration) { + if !mt.mu.firesAt.IsZero() { + panic("tried to reset an active timer") + } + + t := mc.mu.now.Add(d) + + if !mc.mu.now.Before(t) { + // If the timer is scheduled to fire immediately, call its callback + // in a new goroutine immediately. + // + // It needs to be called in its own goroutine to escape its current + // execution context - like an actual timer. + ch := make(chan struct{}) + mc.runningTimers.add(ch) + + go func() { + defer close(ch) + + mt.f() + }() + + return + } + + mt.mu.firesAt = t + + timers, ok := mc.mu.timers[t] + if !ok { + timers = make(map[*manualTimer]struct{}) + mc.mu.timers[t] = timers + heap.Push(&mc.mu.times, t) + } + + timers[mt] = struct{}{} +} + +// stopTimerLocked stops a timer from firing. +// +// Precondition: mc.mu and mt.mu must be locked. +func (mc *ManualClock) stopTimerLocked(mt *manualTimer) { + t := mt.mu.firesAt + mt.mu.firesAt = time.Time{} + + if t.IsZero() { + panic("tried to stop an inactive timer") + } + + timers, ok := mc.mu.timers[t] + if !ok { + err := fmt.Sprintf("tried to stop an active timer but the clock does not have anything scheduled for the timer @ t = %s %p\nScheduled timers @:", t.UTC(), mt) + for t := range mc.mu.timers { + err += fmt.Sprintf("%s\n", t.UTC()) + } + panic(err) + } + + if _, ok := timers[mt]; !ok { + panic(fmt.Sprintf("did not have an entry in timers for an active timer @ t = %s", t.UTC())) + } + + delete(timers, mt) + + if len(timers) == 0 { + delete(mc.mu.timers, t) + } +} + +// RunImmediatelyScheduledJobs runs all jobs scheduled to run at the current +// time. +func (mc *ManualClock) RunImmediatelyScheduledJobs() { + mc.Advance(0) +} + +// Advance executes all work that have been scheduled to execute within d from +// the current time. Blocks until all work has completed execution. +func (mc *ManualClock) Advance(d time.Duration) { + // We spawn goroutines for timers that were scheduled to fire at the time of + // being reset. Wait for those goroutines to complete before proceeding so + // that timer callbacks are called in the right order. + mc.runningTimers.wait() + + mc.mu.Lock() + defer mc.mu.Unlock() + + until := mc.mu.now.Add(d) + for mc.mu.times.Len() > 0 { + t := heap.Pop(&mc.mu.times).(time.Time) + if t.After(until) { + // No work to do + heap.Push(&mc.mu.times, t) + break + } + + timers := mc.mu.timers[t] + delete(mc.mu.timers, t) + + mc.mu.now = t + + // Mark the timers as inactive since they will be fired. + // + // This needs to be done while holding mc's lock because we remove the entry + // in the map of timers for the current time. If an attempt to stop a + // timer is made after mc's lock was dropped but before the timer is + // marked inactive, we would panic since no entry exists for the time when + // the timer was expected to fire. + for mt := range timers { + mt.mu.Lock() + mt.mu.firesAt = time.Time{} + mt.mu.Unlock() + } + + // Release the lock before calling the timer's callback fn since the + // callback fn might try to schedule a timer which requires obtaining + // mc's lock. + mc.mu.Unlock() + + for mt := range timers { + mt.f() + } + + // The timer callbacks may have scheduled a timer to fire immediately. + // We spawn goroutines for these timers and need to wait for them to + // finish before proceeding so that timer callbacks are called in the + // right order. + mc.runningTimers.wait() + mc.mu.Lock() + } + + mc.mu.now = until +} + +func (mc *ManualClock) resetTimer(mt *manualTimer, d time.Duration) { + mc.mu.Lock() + defer mc.mu.Unlock() + + mt.mu.Lock() + defer mt.mu.Unlock() + + if !mt.mu.firesAt.IsZero() { + mc.stopTimerLocked(mt) + } + + mc.resetTimerLocked(mt, d) +} + +func (mc *ManualClock) stopTimer(mt *manualTimer) bool { + mc.mu.Lock() + defer mc.mu.Unlock() + + mt.mu.Lock() + defer mt.mu.Unlock() + + if mt.mu.firesAt.IsZero() { + return false + } + + mc.stopTimerLocked(mt) + return true +} + +// +stateify savable +type manualTimerMu struct { + sync.Mutex `state:"nosave"` + + // firesAt is the time when the timer will fire. + // + // Zero only when the timer is not active. + firesAt time.Time +} + +// +stateify savable +type manualTimer struct { + clock *ManualClock + // TODO(b/341946753): Restore when netstack is savable. + f func() `state:"nosave"` + + mu manualTimerMu +} + +var _ tcpip.Timer = (*manualTimer)(nil) + +// Reset implements tcpip.Timer.Reset. +func (mt *manualTimer) Reset(d time.Duration) { + mt.clock.resetTimer(mt, d) +} + +// Stop implements tcpip.Timer.Stop. +func (mt *manualTimer) Stop() bool { + return mt.clock.stopTimer(mt) +} + +type timeHeap []time.Time + +var _ heap.Interface = (*timeHeap)(nil) + +func (h timeHeap) Len() int { + return len(h) +} + +func (h timeHeap) Less(i, j int) bool { + return h[i].Before(h[j]) +} + +func (h timeHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *timeHeap) Push(x any) { + *h = append(*h, x.(time.Time)) +} + +func (h *timeHeap) Pop() any { + last := (*h)[len(*h)-1] + *h = (*h)[:len(*h)-1] + return last +} diff --git a/pkg/tcpip/faketime/faketime_state_autogen.go b/pkg/tcpip/faketime/faketime_state_autogen.go new file mode 100644 index 0000000..eb5e6d2 --- /dev/null +++ b/pkg/tcpip/faketime/faketime_state_autogen.go @@ -0,0 +1,172 @@ +// automatically generated by stateify. + +package faketime + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (n *NullClock) StateTypeName() string { + return "pkg/tcpip/faketime.NullClock" +} + +func (n *NullClock) StateFields() []string { + return []string{} +} + +func (n *NullClock) beforeSave() {} + +// +checklocksignore +func (n *NullClock) StateSave(stateSinkObject state.Sink) { + n.beforeSave() +} + +func (n *NullClock) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NullClock) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (n *nullTimer) StateTypeName() string { + return "pkg/tcpip/faketime.nullTimer" +} + +func (n *nullTimer) StateFields() []string { + return []string{} +} + +func (n *nullTimer) beforeSave() {} + +// +checklocksignore +func (n *nullTimer) StateSave(stateSinkObject state.Sink) { + n.beforeSave() +} + +func (n *nullTimer) afterLoad(context.Context) {} + +// +checklocksignore +func (n *nullTimer) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (m *manualClockMutex) StateTypeName() string { + return "pkg/tcpip/faketime.manualClockMutex" +} + +func (m *manualClockMutex) StateFields() []string { + return []string{ + "now", + "times", + "timers", + } +} + +func (m *manualClockMutex) beforeSave() {} + +// +checklocksignore +func (m *manualClockMutex) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.now) + stateSinkObject.Save(1, &m.times) + stateSinkObject.Save(2, &m.timers) +} + +func (m *manualClockMutex) afterLoad(context.Context) {} + +// +checklocksignore +func (m *manualClockMutex) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.now) + stateSourceObject.Load(1, &m.times) + stateSourceObject.Load(2, &m.timers) +} + +func (mc *ManualClock) StateTypeName() string { + return "pkg/tcpip/faketime.ManualClock" +} + +func (mc *ManualClock) StateFields() []string { + return []string{ + "runningTimers", + "mu", + } +} + +func (mc *ManualClock) beforeSave() {} + +// +checklocksignore +func (mc *ManualClock) StateSave(stateSinkObject state.Sink) { + mc.beforeSave() + stateSinkObject.Save(0, &mc.runningTimers) + stateSinkObject.Save(1, &mc.mu) +} + +func (mc *ManualClock) afterLoad(context.Context) {} + +// +checklocksignore +func (mc *ManualClock) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &mc.runningTimers) + stateSourceObject.Load(1, &mc.mu) +} + +func (m *manualTimerMu) StateTypeName() string { + return "pkg/tcpip/faketime.manualTimerMu" +} + +func (m *manualTimerMu) StateFields() []string { + return []string{ + "firesAt", + } +} + +func (m *manualTimerMu) beforeSave() {} + +// +checklocksignore +func (m *manualTimerMu) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.firesAt) +} + +func (m *manualTimerMu) afterLoad(context.Context) {} + +// +checklocksignore +func (m *manualTimerMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.firesAt) +} + +func (mt *manualTimer) StateTypeName() string { + return "pkg/tcpip/faketime.manualTimer" +} + +func (mt *manualTimer) StateFields() []string { + return []string{ + "clock", + "mu", + } +} + +func (mt *manualTimer) beforeSave() {} + +// +checklocksignore +func (mt *manualTimer) StateSave(stateSinkObject state.Sink) { + mt.beforeSave() + stateSinkObject.Save(0, &mt.clock) + stateSinkObject.Save(1, &mt.mu) +} + +func (mt *manualTimer) afterLoad(context.Context) {} + +// +checklocksignore +func (mt *manualTimer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &mt.clock) + stateSourceObject.Load(1, &mt.mu) +} + +func init() { + state.Register((*NullClock)(nil)) + state.Register((*nullTimer)(nil)) + state.Register((*manualClockMutex)(nil)) + state.Register((*ManualClock)(nil)) + state.Register((*manualTimerMu)(nil)) + state.Register((*manualTimer)(nil)) +} diff --git a/pkg/tcpip/hash/jenkins/jenkins.go b/pkg/tcpip/hash/jenkins/jenkins.go new file mode 100644 index 0000000..89b20f0 --- /dev/null +++ b/pkg/tcpip/hash/jenkins/jenkins.go @@ -0,0 +1,79 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package jenkins implements Jenkins's one_at_a_time, non-cryptographic hash +// functions created by by Bob Jenkins. +// +// See https://en.wikipedia.org/wiki/Jenkins_hash_function#cite_note-dobbsx-1 +package jenkins + +import ( + "hash" +) + +// Sum32 represents Jenkins's one_at_a_time hash. +// +// Use the Sum32 type directly (as opposed to New32 below) +// to avoid allocations. +type Sum32 uint32 + +// New32 returns a new 32-bit Jenkins's one_at_a_time hash.Hash. +// +// Its Sum method will lay the value out in big-endian byte order. +func New32() hash.Hash32 { + var s Sum32 + return &s +} + +// Reset resets the hash to its initial state. +func (s *Sum32) Reset() { *s = 0 } + +// Sum32 returns the hash value +func (s *Sum32) Sum32() uint32 { + sCopy := *s + + sCopy += sCopy << 3 + sCopy ^= sCopy >> 11 + sCopy += sCopy << 15 + + return uint32(sCopy) +} + +// Write adds more data to the running hash. +// +// It never returns an error. +func (s *Sum32) Write(data []byte) (int, error) { + sCopy := *s + for _, b := range data { + sCopy += Sum32(b) + sCopy += sCopy << 10 + sCopy ^= sCopy >> 6 + } + *s = sCopy + return len(data), nil +} + +// Size returns the number of bytes Sum will return. +func (s *Sum32) Size() int { return 4 } + +// BlockSize returns the hash's underlying block size. +func (s *Sum32) BlockSize() int { return 1 } + +// Sum appends the current hash to in and returns the resulting slice. +// +// It does not change the underlying hash state. +func (s *Sum32) Sum(in []byte) []byte { + v := s.Sum32() + return append(in, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} diff --git a/pkg/tcpip/hash/jenkins/jenkins_state_autogen.go b/pkg/tcpip/hash/jenkins/jenkins_state_autogen.go new file mode 100644 index 0000000..216cc5a --- /dev/null +++ b/pkg/tcpip/hash/jenkins/jenkins_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package jenkins diff --git a/pkg/tcpip/header/arp.go b/pkg/tcpip/header/arp.go new file mode 100644 index 0000000..457a22d --- /dev/null +++ b/pkg/tcpip/header/arp.go @@ -0,0 +1,127 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // ARPProtocolNumber is the ARP network protocol number. + ARPProtocolNumber tcpip.NetworkProtocolNumber = 0x0806 + + // ARPSize is the size of an IPv4-over-Ethernet ARP packet. + ARPSize = 28 +) + +// ARPHardwareType is the hardware type for LinkEndpoint in an ARP header. +type ARPHardwareType uint16 + +// Typical ARP HardwareType values. Some of the constants have to be specific +// values as they are egressed on the wire in the HTYPE field of an ARP header. +const ( + ARPHardwareNone ARPHardwareType = 0 + // ARPHardwareEther specifically is the HTYPE for Ethernet as specified + // in the IANA list here: + // + // https://www.iana.org/assignments/arp-parameters/arp-parameters.xhtml#arp-parameters-2 + ARPHardwareEther ARPHardwareType = 1 + ARPHardwareLoopback ARPHardwareType = 2 +) + +// ARPOp is an ARP opcode. +type ARPOp uint16 + +// Typical ARP opcodes defined in RFC 826. +const ( + ARPRequest ARPOp = 1 + ARPReply ARPOp = 2 +) + +// ARP is an ARP packet stored in a byte array as described in RFC 826. +type ARP []byte + +const ( + hTypeOffset = 0 + protocolOffset = 2 + haAddressSizeOffset = 4 + protoAddressSizeOffset = 5 + opCodeOffset = 6 + senderHAAddressOffset = 8 + senderProtocolAddressOffset = senderHAAddressOffset + EthernetAddressSize + targetHAAddressOffset = senderProtocolAddressOffset + IPv4AddressSize + targetProtocolAddressOffset = targetHAAddressOffset + EthernetAddressSize +) + +func (a ARP) hardwareAddressType() ARPHardwareType { + return ARPHardwareType(binary.BigEndian.Uint16(a[hTypeOffset:])) +} + +func (a ARP) protocolAddressSpace() uint16 { return binary.BigEndian.Uint16(a[protocolOffset:]) } +func (a ARP) hardwareAddressSize() int { return int(a[haAddressSizeOffset]) } +func (a ARP) protocolAddressSize() int { return int(a[protoAddressSizeOffset]) } + +// Op is the ARP opcode. +func (a ARP) Op() ARPOp { return ARPOp(binary.BigEndian.Uint16(a[opCodeOffset:])) } + +// SetOp sets the ARP opcode. +func (a ARP) SetOp(op ARPOp) { + binary.BigEndian.PutUint16(a[opCodeOffset:], uint16(op)) +} + +// SetIPv4OverEthernet configures the ARP packet for IPv4-over-Ethernet. +func (a ARP) SetIPv4OverEthernet() { + binary.BigEndian.PutUint16(a[hTypeOffset:], uint16(ARPHardwareEther)) + binary.BigEndian.PutUint16(a[protocolOffset:], uint16(IPv4ProtocolNumber)) + a[haAddressSizeOffset] = EthernetAddressSize + a[protoAddressSizeOffset] = uint8(IPv4AddressSize) +} + +// HardwareAddressSender is the link address of the sender. +// It is a view on to the ARP packet so it can be used to set the value. +func (a ARP) HardwareAddressSender() []byte { + return a[senderHAAddressOffset : senderHAAddressOffset+EthernetAddressSize] +} + +// ProtocolAddressSender is the protocol address of the sender. +// It is a view on to the ARP packet so it can be used to set the value. +func (a ARP) ProtocolAddressSender() []byte { + return a[senderProtocolAddressOffset : senderProtocolAddressOffset+IPv4AddressSize] +} + +// HardwareAddressTarget is the link address of the target. +// It is a view on to the ARP packet so it can be used to set the value. +func (a ARP) HardwareAddressTarget() []byte { + return a[targetHAAddressOffset : targetHAAddressOffset+EthernetAddressSize] +} + +// ProtocolAddressTarget is the protocol address of the target. +// It is a view on to the ARP packet so it can be used to set the value. +func (a ARP) ProtocolAddressTarget() []byte { + return a[targetProtocolAddressOffset : targetProtocolAddressOffset+IPv4AddressSize] +} + +// IsValid reports whether this is an ARP packet for IPv4 over Ethernet. +func (a ARP) IsValid() bool { + if len(a) < ARPSize { + return false + } + return a.hardwareAddressType() == ARPHardwareEther && + a.protocolAddressSpace() == uint16(IPv4ProtocolNumber) && + a.hardwareAddressSize() == EthernetAddressSize && + a.protocolAddressSize() == IPv4AddressSize +} diff --git a/pkg/tcpip/header/checksum.go b/pkg/tcpip/header/checksum.go new file mode 100644 index 0000000..c6fd4bf --- /dev/null +++ b/pkg/tcpip/header/checksum.go @@ -0,0 +1,107 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package header provides the implementation of the encoding and decoding of +// network protocol headers. +package header + +import ( + "encoding/binary" + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +// PseudoHeaderChecksum calculates the pseudo-header checksum for the given +// destination protocol and network address. Pseudo-headers are needed by +// transport layers when calculating their own checksum. +func PseudoHeaderChecksum(protocol tcpip.TransportProtocolNumber, srcAddr tcpip.Address, dstAddr tcpip.Address, totalLen uint16) uint16 { + xsum := checksum.Checksum(srcAddr.AsSlice(), 0) + xsum = checksum.Checksum(dstAddr.AsSlice(), xsum) + + // Add the length portion of the checksum to the pseudo-checksum. + var tmp [2]byte + binary.BigEndian.PutUint16(tmp[:], totalLen) + xsum = checksum.Checksum(tmp[:], xsum) + + return checksum.Checksum([]byte{0, uint8(protocol)}, xsum) +} + +// checksumUpdate2ByteAlignedUint16 updates a uint16 value in a calculated +// checksum. +// +// The value MUST begin at a 2-byte boundary in the original buffer. +func checksumUpdate2ByteAlignedUint16(xsum, old, new uint16) uint16 { + // As per RFC 1071 page 4, + // (4) Incremental Update + // + // ... + // + // To update the checksum, simply add the differences of the + // sixteen bit integers that have been changed. To see why this + // works, observe that every 16-bit integer has an additive inverse + // and that addition is associative. From this it follows that + // given the original value m, the new value m', and the old + // checksum C, the new checksum C' is: + // + // C' = C + (-m) + m' = C + (m' - m) + if old == new { + return xsum + } + return checksum.Combine(xsum, checksum.Combine(new, ^old)) +} + +// checksumUpdate2ByteAlignedAddress updates an address in a calculated +// checksum. +// +// The addresses must have the same length and must contain an even number +// of bytes. The address MUST begin at a 2-byte boundary in the original buffer. +func checksumUpdate2ByteAlignedAddress(xsum uint16, old, new tcpip.Address) uint16 { + const uint16Bytes = 2 + + if old.BitLen() != new.BitLen() { + panic(fmt.Sprintf("buffer lengths are different; old = %d, new = %d", old.BitLen()/8, new.BitLen()/8)) + } + + if oldBytes := old.BitLen() % 16; oldBytes != 0 { + panic(fmt.Sprintf("buffer has an odd number of bytes; got = %d", oldBytes)) + } + + oldAddr := old.AsSlice() + newAddr := new.AsSlice() + + // As per RFC 1071 page 4, + // (4) Incremental Update + // + // ... + // + // To update the checksum, simply add the differences of the + // sixteen bit integers that have been changed. To see why this + // works, observe that every 16-bit integer has an additive inverse + // and that addition is associative. From this it follows that + // given the original value m, the new value m', and the old + // checksum C, the new checksum C' is: + // + // C' = C + (-m) + m' = C + (m' - m) + for len(oldAddr) != 0 { + // Convert the 2 byte sequences to uint16 values then apply the increment + // update. + xsum = checksumUpdate2ByteAlignedUint16(xsum, (uint16(oldAddr[0])<<8)+uint16(oldAddr[1]), (uint16(newAddr[0])<<8)+uint16(newAddr[1])) + oldAddr = oldAddr[uint16Bytes:] + newAddr = newAddr[uint16Bytes:] + } + + return xsum +} diff --git a/pkg/tcpip/header/datagram.go b/pkg/tcpip/header/datagram.go new file mode 100644 index 0000000..7569091 --- /dev/null +++ b/pkg/tcpip/header/datagram.go @@ -0,0 +1,18 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +// DatagramMaximumSize is the maximum supported size of a single datagram. +const DatagramMaximumSize = 0xffff // 65KB. diff --git a/pkg/tcpip/header/eth.go b/pkg/tcpip/header/eth.go new file mode 100644 index 0000000..626ee53 --- /dev/null +++ b/pkg/tcpip/header/eth.go @@ -0,0 +1,192 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + dstMAC = 0 + srcMAC = 6 + ethType = 12 +) + +// EthernetFields contains the fields of an ethernet frame header. It is used to +// describe the fields of a frame that needs to be encoded. +type EthernetFields struct { + // SrcAddr is the "MAC source" field of an ethernet frame header. + SrcAddr tcpip.LinkAddress + + // DstAddr is the "MAC destination" field of an ethernet frame header. + DstAddr tcpip.LinkAddress + + // Type is the "ethertype" field of an ethernet frame header. + Type tcpip.NetworkProtocolNumber +} + +// Ethernet represents an ethernet frame header stored in a byte array. +type Ethernet []byte + +const ( + // EthernetMinimumSize is the minimum size of a valid ethernet frame. + EthernetMinimumSize = 14 + + // EthernetMaximumSize is the maximum size of a valid ethernet frame. + EthernetMaximumSize = 18 + + // EthernetAddressSize is the size, in bytes, of an ethernet address. + EthernetAddressSize = 6 + + // UnspecifiedEthernetAddress is the unspecified ethernet address + // (all bits set to 0). + UnspecifiedEthernetAddress = tcpip.LinkAddress("\x00\x00\x00\x00\x00\x00") + + // EthernetBroadcastAddress is an ethernet address that addresses every node + // on a local link. + EthernetBroadcastAddress = tcpip.LinkAddress("\xff\xff\xff\xff\xff\xff") + + // unicastMulticastFlagMask is the mask of the least significant bit in + // the first octet (in network byte order) of an ethernet address that + // determines whether the ethernet address is a unicast or multicast. If + // the masked bit is a 1, then the address is a multicast, unicast + // otherwise. + // + // See the IEEE Std 802-2001 document for more details. Specifically, + // section 9.2.1 of http://ieee802.org/secmail/pdfocSP2xXA6d.pdf: + // "A 48-bit universal address consists of two parts. The first 24 bits + // correspond to the OUI as assigned by the IEEE, expect that the + // assignee may set the LSB of the first octet to 1 for group addresses + // or set it to 0 for individual addresses." + unicastMulticastFlagMask = 1 + + // unicastMulticastFlagByteIdx is the byte that holds the + // unicast/multicast flag. See unicastMulticastFlagMask. + unicastMulticastFlagByteIdx = 0 +) + +const ( + // EthernetProtocolAll is a catch-all for all protocols carried inside + // an ethernet frame. It is mainly used to create packet sockets that + // capture all traffic. + EthernetProtocolAll tcpip.NetworkProtocolNumber = 0x0003 + + // EthernetProtocolPUP is the PARC Universal Packet protocol ethertype. + EthernetProtocolPUP tcpip.NetworkProtocolNumber = 0x0200 +) + +// Ethertypes holds the protocol numbers describing the payload of an ethernet +// frame. These types aren't necessarily supported by netstack, but can be used +// to catch all traffic of a type via packet endpoints. +var Ethertypes = []tcpip.NetworkProtocolNumber{ + EthernetProtocolAll, + EthernetProtocolPUP, +} + +// SourceAddress returns the "MAC source" field of the ethernet frame header. +func (b Ethernet) SourceAddress() tcpip.LinkAddress { + return tcpip.LinkAddress(b[srcMAC:][:EthernetAddressSize]) +} + +// DestinationAddress returns the "MAC destination" field of the ethernet frame +// header. +func (b Ethernet) DestinationAddress() tcpip.LinkAddress { + return tcpip.LinkAddress(b[dstMAC:][:EthernetAddressSize]) +} + +// Type returns the "ethertype" field of the ethernet frame header. +func (b Ethernet) Type() tcpip.NetworkProtocolNumber { + return tcpip.NetworkProtocolNumber(binary.BigEndian.Uint16(b[ethType:])) +} + +// Encode encodes all the fields of the ethernet frame header. +func (b Ethernet) Encode(e *EthernetFields) { + binary.BigEndian.PutUint16(b[ethType:], uint16(e.Type)) + copy(b[srcMAC:][:EthernetAddressSize], e.SrcAddr) + copy(b[dstMAC:][:EthernetAddressSize], e.DstAddr) +} + +// IsMulticastEthernetAddress returns true if the address is a multicast +// ethernet address. +func IsMulticastEthernetAddress(addr tcpip.LinkAddress) bool { + if len(addr) != EthernetAddressSize { + return false + } + + return addr[unicastMulticastFlagByteIdx]&unicastMulticastFlagMask != 0 +} + +// IsValidUnicastEthernetAddress returns true if the address is a unicast +// ethernet address. +func IsValidUnicastEthernetAddress(addr tcpip.LinkAddress) bool { + if len(addr) != EthernetAddressSize { + return false + } + + if addr == UnspecifiedEthernetAddress { + return false + } + + if addr[unicastMulticastFlagByteIdx]&unicastMulticastFlagMask != 0 { + return false + } + + return true +} + +// EthernetAddressFromMulticastIPv4Address returns a multicast Ethernet address +// for a multicast IPv4 address. +// +// addr MUST be a multicast IPv4 address. +func EthernetAddressFromMulticastIPv4Address(addr tcpip.Address) tcpip.LinkAddress { + var linkAddrBytes [EthernetAddressSize]byte + // RFC 1112 Host Extensions for IP Multicasting + // + // 6.4. Extensions to an Ethernet Local Network Module: + // + // An IP host group address is mapped to an Ethernet multicast + // address by placing the low-order 23-bits of the IP address + // into the low-order 23 bits of the Ethernet multicast address + // 01-00-5E-00-00-00 (hex). + addrBytes := addr.As4() + linkAddrBytes[0] = 0x1 + linkAddrBytes[2] = 0x5e + linkAddrBytes[3] = addrBytes[1] & 0x7F + copy(linkAddrBytes[4:], addrBytes[IPv4AddressSize-2:]) + return tcpip.LinkAddress(linkAddrBytes[:]) +} + +// EthernetAddressFromMulticastIPv6Address returns a multicast Ethernet address +// for a multicast IPv6 address. +// +// addr MUST be a multicast IPv6 address. +func EthernetAddressFromMulticastIPv6Address(addr tcpip.Address) tcpip.LinkAddress { + // RFC 2464 Transmission of IPv6 Packets over Ethernet Networks + // + // 7. Address Mapping -- Multicast + // + // An IPv6 packet with a multicast destination address DST, + // consisting of the sixteen octets DST[1] through DST[16], is + // transmitted to the Ethernet multicast address whose first + // two octets are the value 3333 hexadecimal and whose last + // four octets are the last four octets of DST. + addrBytes := addr.As16() + linkAddrBytes := []byte(addrBytes[IPv6AddressSize-EthernetAddressSize:]) + linkAddrBytes[0] = 0x33 + linkAddrBytes[1] = 0x33 + return tcpip.LinkAddress(linkAddrBytes[:]) +} diff --git a/pkg/tcpip/header/gue.go b/pkg/tcpip/header/gue.go new file mode 100644 index 0000000..10d358c --- /dev/null +++ b/pkg/tcpip/header/gue.go @@ -0,0 +1,73 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +const ( + typeHLen = 0 + encapProto = 1 +) + +// GUEFields contains the fields of a GUE packet. It is used to describe the +// fields of a packet that needs to be encoded. +type GUEFields struct { + // Type is the "type" field of the GUE header. + Type uint8 + + // Control is the "control" field of the GUE header. + Control bool + + // HeaderLength is the "header length" field of the GUE header. It must + // be at least 4 octets, and a multiple of 4 as well. + HeaderLength uint8 + + // Protocol is the "protocol" field of the GUE header. This is one of + // the IPPROTO_* values. + Protocol uint8 +} + +// GUE represents a Generic UDP Encapsulation header stored in a byte array, the +// fields are described in https://tools.ietf.org/html/draft-ietf-nvo3-gue-01. +type GUE []byte + +const ( + // GUEMinimumSize is the minimum size of a valid GUE packet. + GUEMinimumSize = 4 +) + +// TypeAndControl returns the GUE packet type (top 3 bits of the first byte, +// which includes the control bit). +func (b GUE) TypeAndControl() uint8 { + return b[typeHLen] >> 5 +} + +// HeaderLength returns the total length of the GUE header. +func (b GUE) HeaderLength() uint8 { + return 4 + 4*(b[typeHLen]&0x1f) +} + +// Protocol returns the protocol field of the GUE header. +func (b GUE) Protocol() uint8 { + return b[encapProto] +} + +// Encode encodes all the fields of the GUE header. +func (b GUE) Encode(i *GUEFields) { + ctl := uint8(0) + if i.Control { + ctl = 1 << 5 + } + b[typeHLen] = ctl | i.Type<<6 | (i.HeaderLength-4)/4 + b[encapProto] = i.Protocol +} diff --git a/pkg/tcpip/header/header_state_autogen.go b/pkg/tcpip/header/header_state_autogen.go new file mode 100644 index 0000000..aaa3fbb --- /dev/null +++ b/pkg/tcpip/header/header_state_autogen.go @@ -0,0 +1,120 @@ +// automatically generated by stateify. + +package header + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (t *TCPSynOptions) StateTypeName() string { + return "pkg/tcpip/header.TCPSynOptions" +} + +func (t *TCPSynOptions) StateFields() []string { + return []string{ + "MSS", + "WS", + "TS", + "TSVal", + "TSEcr", + "SACKPermitted", + "Flags", + } +} + +func (t *TCPSynOptions) beforeSave() {} + +// +checklocksignore +func (t *TCPSynOptions) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.MSS) + stateSinkObject.Save(1, &t.WS) + stateSinkObject.Save(2, &t.TS) + stateSinkObject.Save(3, &t.TSVal) + stateSinkObject.Save(4, &t.TSEcr) + stateSinkObject.Save(5, &t.SACKPermitted) + stateSinkObject.Save(6, &t.Flags) +} + +func (t *TCPSynOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPSynOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.MSS) + stateSourceObject.Load(1, &t.WS) + stateSourceObject.Load(2, &t.TS) + stateSourceObject.Load(3, &t.TSVal) + stateSourceObject.Load(4, &t.TSEcr) + stateSourceObject.Load(5, &t.SACKPermitted) + stateSourceObject.Load(6, &t.Flags) +} + +func (r *SACKBlock) StateTypeName() string { + return "pkg/tcpip/header.SACKBlock" +} + +func (r *SACKBlock) StateFields() []string { + return []string{ + "Start", + "End", + } +} + +func (r *SACKBlock) beforeSave() {} + +// +checklocksignore +func (r *SACKBlock) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.Start) + stateSinkObject.Save(1, &r.End) +} + +func (r *SACKBlock) afterLoad(context.Context) {} + +// +checklocksignore +func (r *SACKBlock) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.Start) + stateSourceObject.Load(1, &r.End) +} + +func (t *TCPOptions) StateTypeName() string { + return "pkg/tcpip/header.TCPOptions" +} + +func (t *TCPOptions) StateFields() []string { + return []string{ + "TS", + "TSVal", + "TSEcr", + "SACKBlocks", + } +} + +func (t *TCPOptions) beforeSave() {} + +// +checklocksignore +func (t *TCPOptions) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.TS) + stateSinkObject.Save(1, &t.TSVal) + stateSinkObject.Save(2, &t.TSEcr) + stateSinkObject.Save(3, &t.SACKBlocks) +} + +func (t *TCPOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.TS) + stateSourceObject.Load(1, &t.TSVal) + stateSourceObject.Load(2, &t.TSEcr) + stateSourceObject.Load(3, &t.SACKBlocks) +} + +func init() { + state.Register((*TCPSynOptions)(nil)) + state.Register((*SACKBlock)(nil)) + state.Register((*TCPOptions)(nil)) +} diff --git a/pkg/tcpip/header/icmpv4.go b/pkg/tcpip/header/icmpv4.go new file mode 100644 index 0000000..cc27af0 --- /dev/null +++ b/pkg/tcpip/header/icmpv4.go @@ -0,0 +1,228 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +// ICMPv4 represents an ICMPv4 header stored in a byte array. +type ICMPv4 []byte + +const ( + // ICMPv4PayloadOffset defines the start of ICMP payload. + ICMPv4PayloadOffset = 8 + + // ICMPv4MinimumSize is the minimum size of a valid ICMP packet. + ICMPv4MinimumSize = 8 + + // ICMPv4MinimumErrorPayloadSize Is the smallest number of bytes of an + // errant packet's transport layer that an ICMP error type packet should + // attempt to send as per RFC 792 (see each type) and RFC 1122 + // section 3.2.2 which states: + // Every ICMP error message includes the Internet header and at + // least the first 8 data octets of the datagram that triggered + // the error; more than 8 octets MAY be sent; this header and data + // MUST be unchanged from the received datagram. + // + // RFC 792 shows: + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | Type | Code | Checksum | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | unused | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | Internet Header + 64 bits of Original Data Datagram | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ICMPv4MinimumErrorPayloadSize = 8 + + // ICMPv4ProtocolNumber is the ICMP transport protocol number. + ICMPv4ProtocolNumber tcpip.TransportProtocolNumber = 1 + + // icmpv4ChecksumOffset is the offset of the checksum field + // in an ICMPv4 message. + icmpv4ChecksumOffset = 2 + + // icmpv4MTUOffset is the offset of the MTU field + // in an ICMPv4FragmentationNeeded message. + icmpv4MTUOffset = 6 + + // icmpv4IdentOffset is the offset of the ident field + // in an ICMPv4EchoRequest/Reply message. + icmpv4IdentOffset = 4 + + // icmpv4PointerOffset is the offset of the pointer field + // in an ICMPv4ParamProblem message. + icmpv4PointerOffset = 4 + + // icmpv4SequenceOffset is the offset of the sequence field + // in an ICMPv4EchoRequest/Reply message. + icmpv4SequenceOffset = 6 +) + +// ICMPv4Type is the ICMP type field described in RFC 792. +type ICMPv4Type byte + +// ICMPv4Code is the ICMP code field described in RFC 792. +type ICMPv4Code byte + +// Typical values of ICMPv4Type defined in RFC 792. +const ( + ICMPv4EchoReply ICMPv4Type = 0 + ICMPv4DstUnreachable ICMPv4Type = 3 + ICMPv4SrcQuench ICMPv4Type = 4 + ICMPv4Redirect ICMPv4Type = 5 + ICMPv4Echo ICMPv4Type = 8 + ICMPv4TimeExceeded ICMPv4Type = 11 + ICMPv4ParamProblem ICMPv4Type = 12 + ICMPv4Timestamp ICMPv4Type = 13 + ICMPv4TimestampReply ICMPv4Type = 14 + ICMPv4InfoRequest ICMPv4Type = 15 + ICMPv4InfoReply ICMPv4Type = 16 +) + +// ICMP codes for ICMPv4 Time Exceeded messages as defined in RFC 792. +const ( + ICMPv4TTLExceeded ICMPv4Code = 0 + ICMPv4ReassemblyTimeout ICMPv4Code = 1 +) + +// ICMP codes for ICMPv4 Destination Unreachable messages as defined in RFC 792, +// RFC 1122 section 3.2.2.1 and RFC 1812 section 5.2.7.1. +const ( + ICMPv4NetUnreachable ICMPv4Code = 0 + ICMPv4HostUnreachable ICMPv4Code = 1 + ICMPv4ProtoUnreachable ICMPv4Code = 2 + ICMPv4PortUnreachable ICMPv4Code = 3 + ICMPv4FragmentationNeeded ICMPv4Code = 4 + ICMPv4SourceRouteFailed ICMPv4Code = 5 + ICMPv4DestinationNetworkUnknown ICMPv4Code = 6 + ICMPv4DestinationHostUnknown ICMPv4Code = 7 + ICMPv4SourceHostIsolated ICMPv4Code = 8 + ICMPv4NetProhibited ICMPv4Code = 9 + ICMPv4HostProhibited ICMPv4Code = 10 + ICMPv4NetUnreachableForTos ICMPv4Code = 11 + ICMPv4HostUnreachableForTos ICMPv4Code = 12 + ICMPv4AdminProhibited ICMPv4Code = 13 + ICMPv4HostPrecedenceViolation ICMPv4Code = 14 + ICMPv4PrecedenceCutInEffect ICMPv4Code = 15 +) + +// ICMPv4UnusedCode is a code to use in ICMP messages where no code is needed. +const ICMPv4UnusedCode ICMPv4Code = 0 + +// Type is the ICMP type field. +func (b ICMPv4) Type() ICMPv4Type { return ICMPv4Type(b[0]) } + +// SetType sets the ICMP type field. +func (b ICMPv4) SetType(t ICMPv4Type) { b[0] = byte(t) } + +// Code is the ICMP code field. Its meaning depends on the value of Type. +func (b ICMPv4) Code() ICMPv4Code { return ICMPv4Code(b[1]) } + +// SetCode sets the ICMP code field. +func (b ICMPv4) SetCode(c ICMPv4Code) { b[1] = byte(c) } + +// Pointer returns the pointer field in a Parameter Problem packet. +func (b ICMPv4) Pointer() byte { return b[icmpv4PointerOffset] } + +// SetPointer sets the pointer field in a Parameter Problem packet. +func (b ICMPv4) SetPointer(c byte) { b[icmpv4PointerOffset] = c } + +// Checksum is the ICMP checksum field. +func (b ICMPv4) Checksum() uint16 { + return binary.BigEndian.Uint16(b[icmpv4ChecksumOffset:]) +} + +// SetChecksum sets the ICMP checksum field. +func (b ICMPv4) SetChecksum(cs uint16) { + checksum.Put(b[icmpv4ChecksumOffset:], cs) +} + +// SourcePort implements Transport.SourcePort. +func (ICMPv4) SourcePort() uint16 { + return 0 +} + +// DestinationPort implements Transport.DestinationPort. +func (ICMPv4) DestinationPort() uint16 { + return 0 +} + +// SetSourcePort implements Transport.SetSourcePort. +func (ICMPv4) SetSourcePort(uint16) { +} + +// SetDestinationPort implements Transport.SetDestinationPort. +func (ICMPv4) SetDestinationPort(uint16) { +} + +// Payload implements Transport.Payload. +func (b ICMPv4) Payload() []byte { + return b[ICMPv4PayloadOffset:] +} + +// MTU retrieves the MTU field from an ICMPv4 message. +func (b ICMPv4) MTU() uint16 { + return binary.BigEndian.Uint16(b[icmpv4MTUOffset:]) +} + +// SetMTU sets the MTU field from an ICMPv4 message. +func (b ICMPv4) SetMTU(mtu uint16) { + binary.BigEndian.PutUint16(b[icmpv4MTUOffset:], mtu) +} + +// Ident retrieves the Ident field from an ICMPv4 message. +func (b ICMPv4) Ident() uint16 { + return binary.BigEndian.Uint16(b[icmpv4IdentOffset:]) +} + +// SetIdent sets the Ident field from an ICMPv4 message. +func (b ICMPv4) SetIdent(ident uint16) { + binary.BigEndian.PutUint16(b[icmpv4IdentOffset:], ident) +} + +// SetIdentWithChecksumUpdate sets the Ident field and updates the checksum. +func (b ICMPv4) SetIdentWithChecksumUpdate(new uint16) { + old := b.Ident() + b.SetIdent(new) + b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) +} + +// Sequence retrieves the Sequence field from an ICMPv4 message. +func (b ICMPv4) Sequence() uint16 { + return binary.BigEndian.Uint16(b[icmpv4SequenceOffset:]) +} + +// SetSequence sets the Sequence field from an ICMPv4 message. +func (b ICMPv4) SetSequence(sequence uint16) { + binary.BigEndian.PutUint16(b[icmpv4SequenceOffset:], sequence) +} + +// ICMPv4Checksum calculates the ICMP checksum over the provided ICMP header, +// and payload. +func ICMPv4Checksum(h ICMPv4, payloadCsum uint16) uint16 { + xsum := payloadCsum + + // h[2:4] is the checksum itself, skip it to avoid checksumming the checksum. + xsum = checksum.Checksum(h[:2], xsum) + xsum = checksum.Checksum(h[4:], xsum) + + return ^xsum +} diff --git a/pkg/tcpip/header/icmpv6.go b/pkg/tcpip/header/icmpv6.go new file mode 100644 index 0000000..40f26ca --- /dev/null +++ b/pkg/tcpip/header/icmpv6.go @@ -0,0 +1,304 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +// ICMPv6 represents an ICMPv6 header stored in a byte array. +type ICMPv6 []byte + +const ( + // ICMPv6HeaderSize is the size of the ICMPv6 header. That is, the + // sum of the size of the ICMPv6 Type, Code and Checksum fields, as + // per RFC 4443 section 2.1. After the ICMPv6 header, the ICMPv6 + // message body begins. + ICMPv6HeaderSize = 4 + + // ICMPv6MinimumSize is the minimum size of a valid ICMP packet. + ICMPv6MinimumSize = 8 + + // ICMPv6PayloadOffset is the offset of the payload in an + // ICMP packet. + ICMPv6PayloadOffset = 8 + + // ICMPv6ProtocolNumber is the ICMP transport protocol number. + ICMPv6ProtocolNumber tcpip.TransportProtocolNumber = 58 + + // ICMPv6NeighborSolicitMinimumSize is the minimum size of a + // neighbor solicitation packet. + ICMPv6NeighborSolicitMinimumSize = ICMPv6HeaderSize + NDPNSMinimumSize + + // ICMPv6NeighborAdvertMinimumSize is the minimum size of a + // neighbor advertisement packet. + ICMPv6NeighborAdvertMinimumSize = ICMPv6HeaderSize + NDPNAMinimumSize + + // ICMPv6EchoMinimumSize is the minimum size of a valid echo packet. + ICMPv6EchoMinimumSize = 8 + + // ICMPv6ErrorHeaderSize is the size of an ICMP error packet header, + // as per RFC 4443, Appendix A, item 4 and the errata. + // ... all ICMP error messages shall have exactly + // 32 bits of type-specific data, so that receivers can reliably find + // the embedded invoking packet even when they don't recognize the + // ICMP message Type. + ICMPv6ErrorHeaderSize = 8 + + // ICMPv6DstUnreachableMinimumSize is the minimum size of a valid ICMP + // destination unreachable packet. + ICMPv6DstUnreachableMinimumSize = ICMPv6MinimumSize + + // ICMPv6PacketTooBigMinimumSize is the minimum size of a valid ICMP + // packet-too-big packet. + ICMPv6PacketTooBigMinimumSize = ICMPv6MinimumSize + + // ICMPv6ChecksumOffset is the offset of the checksum field + // in an ICMPv6 message. + ICMPv6ChecksumOffset = 2 + + // icmpv6PointerOffset is the offset of the pointer + // in an ICMPv6 Parameter problem message. + icmpv6PointerOffset = 4 + + // icmpv6MTUOffset is the offset of the MTU field in an ICMPv6 + // PacketTooBig message. + icmpv6MTUOffset = 4 + + // icmpv6IdentOffset is the offset of the ident field + // in a ICMPv6 Echo Request/Reply message. + icmpv6IdentOffset = 4 + + // icmpv6SequenceOffset is the offset of the sequence field + // in a ICMPv6 Echo Request/Reply message. + icmpv6SequenceOffset = 6 + + // NDPHopLimit is the expected IP hop limit value of 255 for received + // NDP packets, as per RFC 4861 sections 4.1 - 4.5, 6.1.1, 6.1.2, 7.1.1, + // 7.1.2 and 8.1. If the hop limit value is not 255, nodes MUST silently + // drop the NDP packet. All outgoing NDP packets must use this value for + // its IP hop limit field. + NDPHopLimit = 255 +) + +// ICMPv6Type is the ICMP type field described in RFC 4443. +type ICMPv6Type byte + +// Values for use in the Type field of ICMPv6 packet from RFC 4433. +const ( + ICMPv6DstUnreachable ICMPv6Type = 1 + ICMPv6PacketTooBig ICMPv6Type = 2 + ICMPv6TimeExceeded ICMPv6Type = 3 + ICMPv6ParamProblem ICMPv6Type = 4 + ICMPv6EchoRequest ICMPv6Type = 128 + ICMPv6EchoReply ICMPv6Type = 129 + + // Neighbor Discovery Protocol (NDP) messages, see RFC 4861. + + ICMPv6RouterSolicit ICMPv6Type = 133 + ICMPv6RouterAdvert ICMPv6Type = 134 + ICMPv6NeighborSolicit ICMPv6Type = 135 + ICMPv6NeighborAdvert ICMPv6Type = 136 + ICMPv6RedirectMsg ICMPv6Type = 137 + + // Multicast Listener Discovery (MLD) messages, see RFC 2710. + + ICMPv6MulticastListenerQuery ICMPv6Type = 130 + ICMPv6MulticastListenerReport ICMPv6Type = 131 + ICMPv6MulticastListenerDone ICMPv6Type = 132 + + // Multicast Listener Discovert Version 2 (MLDv2) messages, see RFC 3810. + + ICMPv6MulticastListenerV2Report ICMPv6Type = 143 +) + +// IsErrorType returns true if the receiver is an ICMP error type. +func (typ ICMPv6Type) IsErrorType() bool { + // Per RFC 4443 section 2.1: + // ICMPv6 messages are grouped into two classes: error messages and + // informational messages. Error messages are identified as such by a + // zero in the high-order bit of their message Type field values. Thus, + // error messages have message types from 0 to 127; informational + // messages have message types from 128 to 255. + return typ&0x80 == 0 +} + +// ICMPv6Code is the ICMP Code field described in RFC 4443. +type ICMPv6Code byte + +// ICMP codes used with Destination Unreachable (Type 1). As per RFC 4443 +// section 3.1. +const ( + ICMPv6NetworkUnreachable ICMPv6Code = 0 + ICMPv6Prohibited ICMPv6Code = 1 + ICMPv6BeyondScope ICMPv6Code = 2 + ICMPv6AddressUnreachable ICMPv6Code = 3 + ICMPv6PortUnreachable ICMPv6Code = 4 + ICMPv6Policy ICMPv6Code = 5 + ICMPv6RejectRoute ICMPv6Code = 6 +) + +// ICMP codes used with Time Exceeded (Type 3). As per RFC 4443 section 3.3. +const ( + ICMPv6HopLimitExceeded ICMPv6Code = 0 + ICMPv6ReassemblyTimeout ICMPv6Code = 1 +) + +// ICMP codes used with Parameter Problem (Type 4). As per RFC 4443 section 3.4. +const ( + // ICMPv6ErroneousHeader indicates an erroneous header field was encountered. + ICMPv6ErroneousHeader ICMPv6Code = 0 + + // ICMPv6UnknownHeader indicates an unrecognized Next Header type encountered. + ICMPv6UnknownHeader ICMPv6Code = 1 + + // ICMPv6UnknownOption indicates an unrecognized IPv6 option was encountered. + ICMPv6UnknownOption ICMPv6Code = 2 +) + +// ICMPv6UnusedCode is the code value used with ICMPv6 messages which don't use +// the code field. (Types not mentioned above.) +const ICMPv6UnusedCode ICMPv6Code = 0 + +// Type is the ICMP type field. +func (b ICMPv6) Type() ICMPv6Type { return ICMPv6Type(b[0]) } + +// SetType sets the ICMP type field. +func (b ICMPv6) SetType(t ICMPv6Type) { b[0] = byte(t) } + +// Code is the ICMP code field. Its meaning depends on the value of Type. +func (b ICMPv6) Code() ICMPv6Code { return ICMPv6Code(b[1]) } + +// SetCode sets the ICMP code field. +func (b ICMPv6) SetCode(c ICMPv6Code) { b[1] = byte(c) } + +// TypeSpecific returns the type specific data field. +func (b ICMPv6) TypeSpecific() uint32 { + return binary.BigEndian.Uint32(b[icmpv6PointerOffset:]) +} + +// SetTypeSpecific sets the type specific data field. +func (b ICMPv6) SetTypeSpecific(val uint32) { + binary.BigEndian.PutUint32(b[icmpv6PointerOffset:], val) +} + +// Checksum is the ICMP checksum field. +func (b ICMPv6) Checksum() uint16 { + return binary.BigEndian.Uint16(b[ICMPv6ChecksumOffset:]) +} + +// SetChecksum sets the ICMP checksum field. +func (b ICMPv6) SetChecksum(cs uint16) { + checksum.Put(b[ICMPv6ChecksumOffset:], cs) +} + +// SourcePort implements Transport.SourcePort. +func (ICMPv6) SourcePort() uint16 { + return 0 +} + +// DestinationPort implements Transport.DestinationPort. +func (ICMPv6) DestinationPort() uint16 { + return 0 +} + +// SetSourcePort implements Transport.SetSourcePort. +func (ICMPv6) SetSourcePort(uint16) { +} + +// SetDestinationPort implements Transport.SetDestinationPort. +func (ICMPv6) SetDestinationPort(uint16) { +} + +// MTU retrieves the MTU field from an ICMPv6 message. +func (b ICMPv6) MTU() uint32 { + return binary.BigEndian.Uint32(b[icmpv6MTUOffset:]) +} + +// SetMTU sets the MTU field from an ICMPv6 message. +func (b ICMPv6) SetMTU(mtu uint32) { + binary.BigEndian.PutUint32(b[icmpv6MTUOffset:], mtu) +} + +// Ident retrieves the Ident field from an ICMPv6 message. +func (b ICMPv6) Ident() uint16 { + return binary.BigEndian.Uint16(b[icmpv6IdentOffset:]) +} + +// SetIdent sets the Ident field from an ICMPv6 message. +func (b ICMPv6) SetIdent(ident uint16) { + binary.BigEndian.PutUint16(b[icmpv6IdentOffset:], ident) +} + +// SetIdentWithChecksumUpdate sets the Ident field and updates the checksum. +func (b ICMPv6) SetIdentWithChecksumUpdate(new uint16) { + old := b.Ident() + b.SetIdent(new) + b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) +} + +// Sequence retrieves the Sequence field from an ICMPv6 message. +func (b ICMPv6) Sequence() uint16 { + return binary.BigEndian.Uint16(b[icmpv6SequenceOffset:]) +} + +// SetSequence sets the Sequence field from an ICMPv6 message. +func (b ICMPv6) SetSequence(sequence uint16) { + binary.BigEndian.PutUint16(b[icmpv6SequenceOffset:], sequence) +} + +// MessageBody returns the message body as defined by RFC 4443 section 2.1; the +// portion of the ICMPv6 buffer after the first ICMPv6HeaderSize bytes. +func (b ICMPv6) MessageBody() []byte { + return b[ICMPv6HeaderSize:] +} + +// Payload implements Transport.Payload. +func (b ICMPv6) Payload() []byte { + return b[ICMPv6PayloadOffset:] +} + +// ICMPv6ChecksumParams contains parameters to calculate ICMPv6 checksum. +type ICMPv6ChecksumParams struct { + Header ICMPv6 + Src tcpip.Address + Dst tcpip.Address + PayloadCsum uint16 + PayloadLen int +} + +// ICMPv6Checksum calculates the ICMP checksum over the provided ICMPv6 header, +// IPv6 src/dst addresses and the payload. +func ICMPv6Checksum(params ICMPv6ChecksumParams) uint16 { + h := params.Header + + xsum := PseudoHeaderChecksum(ICMPv6ProtocolNumber, params.Src, params.Dst, uint16(len(h)+params.PayloadLen)) + xsum = checksum.Combine(xsum, params.PayloadCsum) + + // h[2:4] is the checksum itself, skip it to avoid checksumming the checksum. + xsum = checksum.Checksum(h[:2], xsum) + xsum = checksum.Checksum(h[4:], xsum) + + return ^xsum +} + +// UpdateChecksumPseudoHeaderAddress updates the checksum to reflect an +// updated address in the pseudo header. +func (b ICMPv6) UpdateChecksumPseudoHeaderAddress(old, new tcpip.Address) { + b.SetChecksum(^checksumUpdate2ByteAlignedAddress(^b.Checksum(), old, new)) +} diff --git a/pkg/tcpip/header/igmp.go b/pkg/tcpip/header/igmp.go new file mode 100644 index 0000000..b4f4561 --- /dev/null +++ b/pkg/tcpip/header/igmp.go @@ -0,0 +1,185 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +// IGMP represents an IGMP header stored in a byte array. +type IGMP []byte + +// IGMP implements `Transport`. +var _ Transport = (*IGMP)(nil) + +const ( + // IGMPMinimumSize is the minimum size of a valid IGMP packet in bytes, + // as per RFC 2236, Section 2, Page 2. + IGMPMinimumSize = 8 + + // IGMPQueryMinimumSize is the minimum size of a valid Membership Query + // Message in bytes, as per RFC 2236, Section 2, Page 2. + IGMPQueryMinimumSize = 8 + + // IGMPReportMinimumSize is the minimum size of a valid Report Message in + // bytes, as per RFC 2236, Section 2, Page 2. + IGMPReportMinimumSize = 8 + + // IGMPLeaveMessageMinimumSize is the minimum size of a valid Leave Message + // in bytes, as per RFC 2236, Section 2, Page 2. + IGMPLeaveMessageMinimumSize = 8 + + // IGMPTTL is the TTL for all IGMP messages, as per RFC 2236, Section 3, Page + // 3. + IGMPTTL = 1 + + // igmpTypeOffset defines the offset of the type field in an IGMP message. + igmpTypeOffset = 0 + + // igmpMaxRespTimeOffset defines the offset of the MaxRespTime field in an + // IGMP message. + igmpMaxRespTimeOffset = 1 + + // igmpChecksumOffset defines the offset of the checksum field in an IGMP + // message. + igmpChecksumOffset = 2 + + // igmpGroupAddressOffset defines the offset of the Group Address field in an + // IGMP message. + igmpGroupAddressOffset = 4 + + // IGMPProtocolNumber is IGMP's transport protocol number. + IGMPProtocolNumber tcpip.TransportProtocolNumber = 2 +) + +// IGMPType is the IGMP type field as per RFC 2236. +type IGMPType byte + +// Values for the IGMP Type described in RFC 2236 Section 2.1, Page 2. +// Descriptions below come from there. +const ( + // IGMPMembershipQuery indicates that the message type is Membership Query. + // "There are two sub-types of Membership Query messages: + // - General Query, used to learn which groups have members on an + // attached network. + // - Group-Specific Query, used to learn if a particular group + // has any members on an attached network. + // These two messages are differentiated by the Group Address, as + // described in section 1.4 ." + IGMPMembershipQuery IGMPType = 0x11 + // IGMPv1MembershipReport indicates that the message is a Membership Report + // generated by a host using the IGMPv1 protocol: "an additional type of + // message, for backwards-compatibility with IGMPv1" + IGMPv1MembershipReport IGMPType = 0x12 + // IGMPv2MembershipReport indicates that the Message type is a Membership + // Report generated by a host using the IGMPv2 protocol. + IGMPv2MembershipReport IGMPType = 0x16 + // IGMPLeaveGroup indicates that the message type is a Leave Group + // notification message. + IGMPLeaveGroup IGMPType = 0x17 + // IGMPv3MembershipReport indicates that the message type is a IGMPv3 report. + IGMPv3MembershipReport IGMPType = 0x22 +) + +// Type is the IGMP type field. +func (b IGMP) Type() IGMPType { return IGMPType(b[igmpTypeOffset]) } + +// SetType sets the IGMP type field. +func (b IGMP) SetType(t IGMPType) { b[igmpTypeOffset] = byte(t) } + +// MaxRespTime gets the MaxRespTimeField. This is meaningful only in Membership +// Query messages, in other cases it is set to 0 by the sender and ignored by +// the receiver. +func (b IGMP) MaxRespTime() time.Duration { + // As per RFC 2236 section 2.2, + // + // The Max Response Time field is meaningful only in Membership Query + // messages, and specifies the maximum allowed time before sending a + // responding report in units of 1/10 second. In all other messages, it + // is set to zero by the sender and ignored by receivers. + return DecisecondToDuration(uint16(b[igmpMaxRespTimeOffset])) +} + +// SetMaxRespTime sets the MaxRespTimeField. +func (b IGMP) SetMaxRespTime(m byte) { b[igmpMaxRespTimeOffset] = m } + +// Checksum is the IGMP checksum field. +func (b IGMP) Checksum() uint16 { + return binary.BigEndian.Uint16(b[igmpChecksumOffset:]) +} + +// SetChecksum sets the IGMP checksum field. +func (b IGMP) SetChecksum(checksum uint16) { + binary.BigEndian.PutUint16(b[igmpChecksumOffset:], checksum) +} + +// GroupAddress gets the Group Address field. +func (b IGMP) GroupAddress() tcpip.Address { + return tcpip.AddrFrom4([4]byte(b[igmpGroupAddressOffset:][:IPv4AddressSize])) +} + +// SetGroupAddress sets the Group Address field. +func (b IGMP) SetGroupAddress(address tcpip.Address) { + addrBytes := address.As4() + if n := copy(b[igmpGroupAddressOffset:], addrBytes[:]); n != IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d", n, IPv4AddressSize)) + } +} + +// SourcePort implements Transport.SourcePort. +func (IGMP) SourcePort() uint16 { + return 0 +} + +// DestinationPort implements Transport.DestinationPort. +func (IGMP) DestinationPort() uint16 { + return 0 +} + +// SetSourcePort implements Transport.SetSourcePort. +func (IGMP) SetSourcePort(uint16) { +} + +// SetDestinationPort implements Transport.SetDestinationPort. +func (IGMP) SetDestinationPort(uint16) { +} + +// Payload implements Transport.Payload. +func (IGMP) Payload() []byte { + return nil +} + +// IGMPCalculateChecksum calculates the IGMP checksum over the provided IGMP +// header. +func IGMPCalculateChecksum(h IGMP) uint16 { + // The header contains a checksum itself, set it aside to avoid checksumming + // the checksum and replace it afterwards. + existingXsum := h.Checksum() + h.SetChecksum(0) + xsum := ^checksum.Checksum(h, 0) + h.SetChecksum(existingXsum) + return xsum +} + +// DecisecondToDuration converts a value representing deci-seconds to a +// time.Duration. +func DecisecondToDuration(ds uint16) time.Duration { + return time.Duration(ds) * time.Second / 10 +} diff --git a/pkg/tcpip/header/igmpv3.go b/pkg/tcpip/header/igmpv3.go new file mode 100644 index 0000000..1d2edc6 --- /dev/null +++ b/pkg/tcpip/header/igmpv3.go @@ -0,0 +1,500 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "bytes" + "encoding/binary" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// IGMPv3RoutersAddress is the address to send IGMPv3 reports to. +// +// As per RFC 3376 section 4.2.14, +// +// Version 3 Reports are sent with an IP destination address of +// 224.0.0.22, to which all IGMPv3-capable multicast routers listen. +var IGMPv3RoutersAddress = tcpip.AddrFrom4([4]byte{0xe0, 0x00, 0x00, 0x16}) + +const ( + // IGMPv3QueryMinimumSize is the mimum size of a valid IGMPv3 query, + // as per RFC 3376 section 4.1. + IGMPv3QueryMinimumSize = 12 + + igmpv3QueryMaxRespCodeOffset = 1 + igmpv3QueryGroupAddressOffset = 4 + igmpv3QueryResvSQRVOffset = 8 + igmpv3QueryQRVMask = 0b111 + igmpv3QueryQQICOffset = 9 + igmpv3QueryNumberOfSourcesOffset = 10 + igmpv3QuerySourcesOffset = 12 +) + +// IGMPv3Query is an IGMPv3 query message. +// +// As per RFC 3376 section 4.1, +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type = 0x11 | Max Resp Code | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Group Address | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Resv |S| QRV | QQIC | Number of Sources (N) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Source Address [1] | +// +- -+ +// | Source Address [2] | +// +- . -+ +// . . . +// . . . +// +- -+ +// | Source Address [N] | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type IGMPv3Query IGMP + +// MaximumResponseCode returns the Maximum Response Code. +func (i IGMPv3Query) MaximumResponseCode() uint8 { + return i[igmpv3QueryMaxRespCodeOffset] +} + +// IGMPv3MaximumResponseDelay returns the Maximum Response Delay in an IGMPv3 +// Maximum Response Code. +// +// As per RFC 3376 section 4.1.1, +// +// The Max Resp Code field specifies the maximum time allowed before +// sending a responding report. The actual time allowed, called the Max +// Resp Time, is represented in units of 1/10 second and is derived from +// the Max Resp Code as follows: +// +// If Max Resp Code < 128, Max Resp Time = Max Resp Code +// +// If Max Resp Code >= 128, Max Resp Code represents a floating-point +// value as follows: +// +// 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +// |1| exp | mant | +// +-+-+-+-+-+-+-+-+ +// +// Max Resp Time = (mant | 0x10) << (exp + 3) +// +// Small values of Max Resp Time allow IGMPv3 routers to tune the "leave +// latency" (the time between the moment the last host leaves a group +// and the moment the routing protocol is notified that there are no +// more members). Larger values, especially in the exponential range, +// allow tuning of the burstiness of IGMP traffic on a network. +func IGMPv3MaximumResponseDelay(codeRaw uint8) time.Duration { + code := uint16(codeRaw) + if code < 128 { + return DecisecondToDuration(code) + } + + const mantBits = 4 + const expMask = 0b111 + exp := (code >> mantBits) & expMask + mant := code & ((1 << mantBits) - 1) + return DecisecondToDuration((mant | 0x10) << (exp + 3)) +} + +// GroupAddress returns the group address. +func (i IGMPv3Query) GroupAddress() tcpip.Address { + return tcpip.AddrFrom4([4]byte(i[igmpv3QueryGroupAddressOffset:][:IPv4AddressSize])) +} + +// QuerierRobustnessVariable returns the querier's robustness variable. +func (i IGMPv3Query) QuerierRobustnessVariable() uint8 { + return i[igmpv3QueryResvSQRVOffset] & igmpv3QueryQRVMask +} + +// QuerierQueryInterval returns the querier's query interval. +func (i IGMPv3Query) QuerierQueryInterval() time.Duration { + return mldv2AndIGMPv3QuerierQueryCodeToInterval(i[igmpv3QueryQQICOffset]) +} + +// Sources returns an iterator over source addresses in the query. +// +// Returns false if the message cannot hold the expected number of sources. +func (i IGMPv3Query) Sources() (AddressIterator, bool) { + return makeAddressIterator( + i[igmpv3QuerySourcesOffset:], + binary.BigEndian.Uint16(i[igmpv3QueryNumberOfSourcesOffset:]), + IPv4AddressSize, + ) +} + +// IGMPv3ReportRecordType is the type of an IGMPv3 multicast address record +// found in an IGMPv3 report, as per RFC 3810 section 5.2.12. +type IGMPv3ReportRecordType int + +// IGMPv3 multicast address record types, as per RFC 3810 section 5.2.12. +const ( + IGMPv3ReportRecordModeIsInclude IGMPv3ReportRecordType = 1 + IGMPv3ReportRecordModeIsExclude IGMPv3ReportRecordType = 2 + IGMPv3ReportRecordChangeToIncludeMode IGMPv3ReportRecordType = 3 + IGMPv3ReportRecordChangeToExcludeMode IGMPv3ReportRecordType = 4 + IGMPv3ReportRecordAllowNewSources IGMPv3ReportRecordType = 5 + IGMPv3ReportRecordBlockOldSources IGMPv3ReportRecordType = 6 +) + +const ( + igmpv3ReportGroupAddressRecordMinimumSize = 8 + igmpv3ReportGroupAddressRecordTypeOffset = 0 + igmpv3ReportGroupAddressRecordAuxDataLenOffset = 1 + igmpv3ReportGroupAddressRecordAuxDataLenUnits = 4 + igmpv3ReportGroupAddressRecordNumberOfSourcesOffset = 2 + igmpv3ReportGroupAddressRecordGroupAddressOffset = 4 + igmpv3ReportGroupAddressRecordSourcesOffset = 8 +) + +// IGMPv3ReportGroupAddressRecordSerializer is an IGMPv3 Multicast Address +// Record serializer. +// +// As per RFC 3810 section 5.2, a Multicast Address Record has the following +// internal format: +// +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Record Type | Aux Data Len | Number of Sources (N) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Multicast Address * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Source Address [1] * +// | | +// * * +// | | +// +- -+ +// | | +// * * +// | | +// * Source Address [2] * +// | | +// * * +// | | +// +- -+ +// . . . +// . . . +// . . . +// +- -+ +// | | +// * * +// | | +// * Source Address [N] * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Auxiliary Data . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type IGMPv3ReportGroupAddressRecordSerializer struct { + RecordType IGMPv3ReportRecordType + GroupAddress tcpip.Address + Sources []tcpip.Address +} + +// Length returns the number of bytes this serializer would occupy. +func (s *IGMPv3ReportGroupAddressRecordSerializer) Length() int { + return igmpv3ReportGroupAddressRecordSourcesOffset + len(s.Sources)*IPv4AddressSize +} + +func copyIPv4Address(dst []byte, src tcpip.Address) { + srcBytes := src.As4() + if n := copy(dst, srcBytes[:]); n != IPv4AddressSize { + panic(fmt.Sprintf("got copy(...) = %d, want = %d", n, IPv4AddressSize)) + } +} + +// SerializeInto serializes the record into the buffer. +// +// Panics if the buffer does not have enough space to fit the record. +func (s *IGMPv3ReportGroupAddressRecordSerializer) SerializeInto(b []byte) { + b[igmpv3ReportGroupAddressRecordTypeOffset] = byte(s.RecordType) + b[igmpv3ReportGroupAddressRecordAuxDataLenOffset] = 0 + binary.BigEndian.PutUint16(b[igmpv3ReportGroupAddressRecordNumberOfSourcesOffset:], uint16(len(s.Sources))) + copyIPv4Address(b[igmpv3ReportGroupAddressRecordGroupAddressOffset:], s.GroupAddress) + b = b[igmpv3ReportGroupAddressRecordSourcesOffset:] + for _, source := range s.Sources { + copyIPv4Address(b, source) + b = b[IPv4AddressSize:] + } +} + +const ( + igmpv3ReportTypeOffset = 0 + igmpv3ReportReserved1Offset = 1 + igmpv3ReportReserved2Offset = 4 + igmpv3ReportNumberOfGroupAddressRecordsOffset = 6 + igmpv3ReportGroupAddressRecordsOffset = 8 +) + +// IGMPv3ReportSerializer is an MLD Version 2 Report serializer. +// +// As per RFC 3810 section 5.2, +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type = 143 | Reserved | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reserved |Nr of Mcast Address Records (M)| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [1] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [2] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | . | +// . . . +// | . | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [M] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type IGMPv3ReportSerializer struct { + Records []IGMPv3ReportGroupAddressRecordSerializer +} + +// Length returns the number of bytes this serializer would occupy. +func (s *IGMPv3ReportSerializer) Length() int { + ret := igmpv3ReportGroupAddressRecordsOffset + for _, record := range s.Records { + ret += record.Length() + } + return ret +} + +// SerializeInto serializes the report into the buffer. +// +// Panics if the buffer does not have enough space to fit the report. +func (s *IGMPv3ReportSerializer) SerializeInto(b []byte) { + b[igmpv3ReportTypeOffset] = byte(IGMPv3MembershipReport) + b[igmpv3ReportReserved1Offset] = 0 + binary.BigEndian.PutUint16(b[igmpv3ReportReserved2Offset:], 0) + binary.BigEndian.PutUint16(b[igmpv3ReportNumberOfGroupAddressRecordsOffset:], uint16(len(s.Records))) + recordsBytes := b[igmpv3ReportGroupAddressRecordsOffset:] + for _, record := range s.Records { + len := record.Length() + record.SerializeInto(recordsBytes[:len]) + recordsBytes = recordsBytes[len:] + } + binary.BigEndian.PutUint16(b[igmpChecksumOffset:], IGMPCalculateChecksum(b)) +} + +// IGMPv3ReportGroupAddressRecord is an IGMPv3 record. +// +// As per RFC 3810 section 5.2, a Multicast Address Record has the following +// internal format: +// +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Record Type | Aux Data Len | Number of Sources (N) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Multicast Address * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Source Address [1] * +// | | +// * * +// | | +// +- -+ +// | | +// * * +// | | +// * Source Address [2] * +// | | +// * * +// | | +// +- -+ +// . . . +// . . . +// . . . +// +- -+ +// | | +// * * +// | | +// * Source Address [N] * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Auxiliary Data . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type IGMPv3ReportGroupAddressRecord []byte + +// RecordType returns the type of this record. +func (r IGMPv3ReportGroupAddressRecord) RecordType() IGMPv3ReportRecordType { + return IGMPv3ReportRecordType(r[igmpv3ReportGroupAddressRecordTypeOffset]) +} + +// AuxDataLen returns the length of the auxiliary data in this record. +func (r IGMPv3ReportGroupAddressRecord) AuxDataLen() int { + return int(r[igmpv3ReportGroupAddressRecordAuxDataLenOffset]) * igmpv3ReportGroupAddressRecordAuxDataLenUnits +} + +// numberOfSources returns the number of sources in this record. +func (r IGMPv3ReportGroupAddressRecord) numberOfSources() uint16 { + return binary.BigEndian.Uint16(r[igmpv3ReportGroupAddressRecordNumberOfSourcesOffset:]) +} + +// GroupAddress returns the multicast address this record targets. +func (r IGMPv3ReportGroupAddressRecord) GroupAddress() tcpip.Address { + return tcpip.AddrFrom4([4]byte(r[igmpv3ReportGroupAddressRecordGroupAddressOffset:][:IPv4AddressSize])) +} + +// Sources returns an iterator over source addresses in the query. +// +// Returns false if the message cannot hold the expected number of sources. +func (r IGMPv3ReportGroupAddressRecord) Sources() (AddressIterator, bool) { + expectedLen := int(r.numberOfSources()) * IPv4AddressSize + b := r[igmpv3ReportGroupAddressRecordSourcesOffset:] + if len(b) < expectedLen { + return AddressIterator{}, false + } + return AddressIterator{addressSize: IPv4AddressSize, buf: bytes.NewBuffer(b[:expectedLen])}, true +} + +// IGMPv3Report is an IGMPv3 Report. +// +// As per RFC 3810 section 5.2, +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type = 143 | Reserved | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reserved |Nr of Mcast Address Records (M)| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [1] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [2] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | . | +// . . . +// | . | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [M] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type IGMPv3Report []byte + +// Checksum returns the checksum. +func (i IGMPv3Report) Checksum() uint16 { + return binary.BigEndian.Uint16(i[igmpChecksumOffset:]) +} + +// IGMPv3ReportGroupAddressRecordIterator is an iterator over IGMPv3 Multicast +// Address Records. +type IGMPv3ReportGroupAddressRecordIterator struct { + recordsLeft uint16 + buf *bytes.Buffer +} + +// IGMPv3ReportGroupAddressRecordIteratorNextDisposition is the possible +// return values from IGMPv3ReportGroupAddressRecordIterator.Next. +type IGMPv3ReportGroupAddressRecordIteratorNextDisposition int + +const ( + // IGMPv3ReportGroupAddressRecordIteratorNextOk indicates that a multicast + // address record was yielded. + IGMPv3ReportGroupAddressRecordIteratorNextOk IGMPv3ReportGroupAddressRecordIteratorNextDisposition = iota + + // IGMPv3ReportGroupAddressRecordIteratorNextDone indicates that the iterator + // has been exhausted. + IGMPv3ReportGroupAddressRecordIteratorNextDone + + // IGMPv3ReportGroupAddressRecordIteratorNextErrBufferTooShort indicates + // that the iterator expected another record, but the buffer ended + // prematurely. + IGMPv3ReportGroupAddressRecordIteratorNextErrBufferTooShort +) + +// Next returns the next IGMPv3 Multicast Address Record. +func (it *IGMPv3ReportGroupAddressRecordIterator) Next() (IGMPv3ReportGroupAddressRecord, IGMPv3ReportGroupAddressRecordIteratorNextDisposition) { + if it.recordsLeft == 0 { + return IGMPv3ReportGroupAddressRecord{}, IGMPv3ReportGroupAddressRecordIteratorNextDone + } + if it.buf.Len() < igmpv3ReportGroupAddressRecordMinimumSize { + return IGMPv3ReportGroupAddressRecord{}, IGMPv3ReportGroupAddressRecordIteratorNextErrBufferTooShort + } + + hdr := IGMPv3ReportGroupAddressRecord(it.buf.Bytes()) + expectedLen := igmpv3ReportGroupAddressRecordMinimumSize + + int(hdr.AuxDataLen()) + int(hdr.numberOfSources())*IPv4AddressSize + + bytes := it.buf.Next(expectedLen) + if len(bytes) < expectedLen { + return IGMPv3ReportGroupAddressRecord{}, IGMPv3ReportGroupAddressRecordIteratorNextErrBufferTooShort + } + it.recordsLeft-- + return IGMPv3ReportGroupAddressRecord(bytes), IGMPv3ReportGroupAddressRecordIteratorNextOk +} + +// GroupAddressRecords returns an iterator of IGMPv3 Multicast Address +// Records. +func (i IGMPv3Report) GroupAddressRecords() IGMPv3ReportGroupAddressRecordIterator { + return IGMPv3ReportGroupAddressRecordIterator{ + recordsLeft: binary.BigEndian.Uint16(i[igmpv3ReportNumberOfGroupAddressRecordsOffset:]), + buf: bytes.NewBuffer(i[igmpv3ReportGroupAddressRecordsOffset:]), + } +} diff --git a/pkg/tcpip/header/interfaces.go b/pkg/tcpip/header/interfaces.go new file mode 100644 index 0000000..b039b72 --- /dev/null +++ b/pkg/tcpip/header/interfaces.go @@ -0,0 +1,130 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // MaxIPPacketSize is the maximum supported IP packet size, excluding + // jumbograms. The maximum IPv4 packet size is 64k-1 (total size must fit + // in 16 bits). For IPv6, the payload max size (excluding jumbograms) is + // 64k-1 (also needs to fit in 16 bits). So we use 64k - 1 + 2 * m, where + // m is the minimum IPv6 header size; we leave room for some potential + // IP options. + MaxIPPacketSize = 0xffff + 2*IPv6MinimumSize +) + +// Transport offers generic methods to query and/or update the fields of the +// header of a transport protocol buffer. +type Transport interface { + // SourcePort returns the value of the "source port" field. + SourcePort() uint16 + + // Destination returns the value of the "destination port" field. + DestinationPort() uint16 + + // Checksum returns the value of the "checksum" field. + Checksum() uint16 + + // SetSourcePort sets the value of the "source port" field. + SetSourcePort(uint16) + + // SetDestinationPort sets the value of the "destination port" field. + SetDestinationPort(uint16) + + // SetChecksum sets the value of the "checksum" field. + SetChecksum(uint16) + + // Payload returns the data carried in the transport buffer. + Payload() []byte +} + +// ChecksummableTransport is a Transport that supports checksumming. +type ChecksummableTransport interface { + Transport + + // SetSourcePortWithChecksumUpdate sets the source port and updates + // the checksum. + // + // The receiver's checksum must be a fully calculated checksum. + SetSourcePortWithChecksumUpdate(port uint16) + + // SetDestinationPortWithChecksumUpdate sets the destination port and updates + // the checksum. + // + // The receiver's checksum must be a fully calculated checksum. + SetDestinationPortWithChecksumUpdate(port uint16) + + // UpdateChecksumPseudoHeaderAddress updates the checksum to reflect an + // updated address in the pseudo header. + // + // If fullChecksum is true, the receiver's checksum field is assumed to hold a + // fully calculated checksum. Otherwise, it is assumed to hold a partially + // calculated checksum which only reflects the pseudo header. + UpdateChecksumPseudoHeaderAddress(old, new tcpip.Address, fullChecksum bool) +} + +// Network offers generic methods to query and/or update the fields of the +// header of a network protocol buffer. +type Network interface { + // SourceAddress returns the value of the "source address" field. + SourceAddress() tcpip.Address + + // DestinationAddress returns the value of the "destination address" + // field. + DestinationAddress() tcpip.Address + + // Checksum returns the value of the "checksum" field. + Checksum() uint16 + + // SetSourceAddress sets the value of the "source address" field. + SetSourceAddress(tcpip.Address) + + // SetDestinationAddress sets the value of the "destination address" + // field. + SetDestinationAddress(tcpip.Address) + + // SetChecksum sets the value of the "checksum" field. + SetChecksum(uint16) + + // TransportProtocol returns the number of the transport protocol + // stored in the payload. + TransportProtocol() tcpip.TransportProtocolNumber + + // Payload returns a byte slice containing the payload of the network + // packet. + Payload() []byte + + // TOS returns the values of the "type of service" and "flow label" fields. + TOS() (uint8, uint32) + + // SetTOS sets the values of the "type of service" and "flow label" fields. + SetTOS(t uint8, l uint32) +} + +// ChecksummableNetwork is a Network that supports checksumming. +type ChecksummableNetwork interface { + Network + + // SetSourceAddressAndChecksum sets the source address and updates the + // checksum to reflect the new address. + SetSourceAddressWithChecksumUpdate(tcpip.Address) + + // SetDestinationAddressAndChecksum sets the destination address and + // updates the checksum to reflect the new address. + SetDestinationAddressWithChecksumUpdate(tcpip.Address) +} diff --git a/pkg/tcpip/header/ipv4.go b/pkg/tcpip/header/ipv4.go new file mode 100644 index 0000000..f9eeaef --- /dev/null +++ b/pkg/tcpip/header/ipv4.go @@ -0,0 +1,1274 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +// RFC 971 defines the fields of the IPv4 header on page 11 using the following +// diagram: ("Figure 4") +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |Version| IHL |Type of Service| Total Length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Identification |Flags| Fragment Offset | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Time to Live | Protocol | Header Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Source Address | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Destination Address | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Options | Padding | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +const ( + versIHL = 0 + tos = 1 + // IPv4TotalLenOffset is the offset of the total length field in the + // IPv4 header. + IPv4TotalLenOffset = 2 + id = 4 + flagsFO = 6 + ttl = 8 + protocol = 9 + xsum = 10 + srcAddr = 12 + dstAddr = 16 + options = 20 +) + +// IPv4Fields contains the fields of an IPv4 packet. It is used to describe the +// fields of a packet that needs to be encoded. The IHL field is not here as +// it is totally defined by the size of the options. +type IPv4Fields struct { + // TOS is the "type of service" field of an IPv4 packet. + TOS uint8 + + // TotalLength is the "total length" field of an IPv4 packet. + TotalLength uint16 + + // ID is the "identification" field of an IPv4 packet. + ID uint16 + + // Flags is the "flags" field of an IPv4 packet. + Flags uint8 + + // FragmentOffset is the "fragment offset" field of an IPv4 packet. + FragmentOffset uint16 + + // TTL is the "time to live" field of an IPv4 packet. + TTL uint8 + + // Protocol is the "protocol" field of an IPv4 packet. + Protocol uint8 + + // Checksum is the "checksum" field of an IPv4 packet. + Checksum uint16 + + // SrcAddr is the "source ip address" of an IPv4 packet. + SrcAddr tcpip.Address + + // DstAddr is the "destination ip address" of an IPv4 packet. + DstAddr tcpip.Address + + // Options must be 40 bytes or less as they must fit along with the + // rest of the IPv4 header into the maximum size describable in the + // IHL field. RFC 791 section 3.1 says: + // IHL: 4 bits + // + // Internet Header Length is the length of the internet header in 32 + // bit words, and thus points to the beginning of the data. Note that + // the minimum value for a correct header is 5. + // + // That leaves ten 32 bit (4 byte) fields for options. An attempt to encode + // more will fail. + Options IPv4OptionsSerializer +} + +// IPv4 is an IPv4 header. +// Most of the methods of IPv4 access to the underlying slice without +// checking the boundaries and could panic because of 'index out of range'. +// Always call IsValid() to validate an instance of IPv4 before using other +// methods. +type IPv4 []byte + +const ( + // IPv4MinimumSize is the minimum size of a valid IPv4 packet; + // i.e. a packet header with no options. + IPv4MinimumSize = 20 + + // IPv4MaximumHeaderSize is the maximum size of an IPv4 header. Given + // that there are only 4 bits (max 0xF (15)) to represent the header length + // in 32-bit (4 byte) units, the header cannot exceed 15*4 = 60 bytes. + IPv4MaximumHeaderSize = 60 + + // IPv4MaximumOptionsSize is the largest size the IPv4 options can be. + IPv4MaximumOptionsSize = IPv4MaximumHeaderSize - IPv4MinimumSize + + // IPv4MaximumPayloadSize is the maximum size of a valid IPv4 payload. + // + // Linux limits this to 65,515 octets (the max IP datagram size - the IPv4 + // header size). But RFC 791 section 3.2 discusses the design of the IPv4 + // fragment "allows 2**13 = 8192 fragments of 8 octets each for a total of + // 65,536 octets. Note that this is consistent with the datagram total + // length field (of course, the header is counted in the total length and not + // in the fragments)." + IPv4MaximumPayloadSize = 65536 + + // MinIPFragmentPayloadSize is the minimum number of payload bytes that + // the first fragment must carry when an IPv4 packet is fragmented. + MinIPFragmentPayloadSize = 8 + + // IPv4AddressSize is the size, in bytes, of an IPv4 address. + IPv4AddressSize = 4 + + // IPv4AddressSizeBits is the size, in bits, of an IPv4 address. + IPv4AddressSizeBits = 32 + + // IPv4ProtocolNumber is IPv4's network protocol number. + IPv4ProtocolNumber tcpip.NetworkProtocolNumber = 0x0800 + + // IPv4Version is the version of the IPv4 protocol. + IPv4Version = 4 + + // IPv4MinimumProcessableDatagramSize is the minimum size of an IP + // packet that every IPv4 capable host must be able to + // process/reassemble. + IPv4MinimumProcessableDatagramSize = 576 + + // IPv4MinimumMTU is the minimum MTU required by IPv4, per RFC 791, + // section 3.2: + // Every internet module must be able to forward a datagram of 68 octets + // without further fragmentation. This is because an internet header may be + // up to 60 octets, and the minimum fragment is 8 octets. + IPv4MinimumMTU = 68 +) + +var ( + // IPv4AllSystems is the all systems IPv4 multicast address as per + // IANA's IPv4 Multicast Address Space Registry. See + // https://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml. + IPv4AllSystems = tcpip.AddrFrom4([4]byte{0xe0, 0x00, 0x00, 0x01}) + + // IPv4Broadcast is the broadcast address of the IPv4 procotol. + IPv4Broadcast = tcpip.AddrFrom4([4]byte{0xff, 0xff, 0xff, 0xff}) + + // IPv4Any is the non-routable IPv4 "any" meta address. + IPv4Any = tcpip.AddrFrom4([4]byte{0x00, 0x00, 0x00, 0x00}) + + // IPv4AllRoutersGroup is a multicast address for all routers. + IPv4AllRoutersGroup = tcpip.AddrFrom4([4]byte{0xe0, 0x00, 0x00, 0x02}) + + // IPv4Loopback is the loopback IPv4 address. + IPv4Loopback = tcpip.AddrFrom4([4]byte{0x7f, 0x00, 0x00, 0x01}) +) + +// Flags that may be set in an IPv4 packet. +const ( + IPv4FlagMoreFragments = 1 << iota + IPv4FlagDontFragment +) + +// ipv4LinkLocalUnicastSubnet is the IPv4 link local unicast subnet as defined +// by RFC 3927 section 1. +var ipv4LinkLocalUnicastSubnet = func() tcpip.Subnet { + subnet, err := tcpip.NewSubnet(tcpip.AddrFrom4([4]byte{0xa9, 0xfe, 0x00, 0x00}), tcpip.MaskFrom("\xff\xff\x00\x00")) + if err != nil { + panic(err) + } + return subnet +}() + +// ipv4LinkLocalMulticastSubnet is the IPv4 link local multicast subnet as +// defined by RFC 5771 section 4. +var ipv4LinkLocalMulticastSubnet = func() tcpip.Subnet { + subnet, err := tcpip.NewSubnet(tcpip.AddrFrom4([4]byte{0xe0, 0x00, 0x00, 0x00}), tcpip.MaskFrom("\xff\xff\xff\x00")) + if err != nil { + panic(err) + } + return subnet +}() + +// IPv4EmptySubnet is the empty IPv4 subnet. +var IPv4EmptySubnet = func() tcpip.Subnet { + subnet, err := tcpip.NewSubnet(IPv4Any, tcpip.MaskFrom("\x00\x00\x00\x00")) + if err != nil { + panic(err) + } + return subnet +}() + +// IPv4CurrentNetworkSubnet is the subnet of addresses for the current network, +// per RFC 6890 section 2.2.2, +// +// +----------------------+----------------------------+ +// | Attribute | Value | +// +----------------------+----------------------------+ +// | Address Block | 0.0.0.0/8 | +// | Name | "This host on this network"| +// | RFC | [RFC1122], Section 3.2.1.3 | +// | Allocation Date | September 1981 | +// | Termination Date | N/A | +// | Source | True | +// | Destination | False | +// | Forwardable | False | +// | Global | False | +// | Reserved-by-Protocol | True | +// +----------------------+----------------------------+ +var IPv4CurrentNetworkSubnet = func() tcpip.Subnet { + subnet, err := tcpip.NewSubnet(IPv4Any, tcpip.MaskFrom("\xff\x00\x00\x00")) + if err != nil { + panic(err) + } + return subnet +}() + +// IPv4LoopbackSubnet is the loopback subnet for IPv4. +var IPv4LoopbackSubnet = func() tcpip.Subnet { + subnet, err := tcpip.NewSubnet(tcpip.AddrFrom4([4]byte{0x7f, 0x00, 0x00, 0x00}), tcpip.MaskFrom("\xff\x00\x00\x00")) + if err != nil { + panic(err) + } + return subnet +}() + +// IPVersion returns the version of IP used in the given packet. It returns -1 +// if the packet is not large enough to contain the version field. +func IPVersion(b []byte) int { + // Length must be at least offset+length of version field. + if len(b) < versIHL+1 { + return -1 + } + return int(b[versIHL] >> ipVersionShift) +} + +// RFC 791 page 11 shows the header length (IHL) is in the lower 4 bits +// of the first byte, and is counted in multiples of 4 bytes. +// +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |Version| IHL |Type of Service| Total Length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// (...) +// Version: 4 bits +// The Version field indicates the format of the internet header. This +// document describes version 4. +// +// IHL: 4 bits +// Internet Header Length is the length of the internet header in 32 +// bit words, and thus points to the beginning of the data. Note that +// the minimum value for a correct header is 5. +const ( + ipVersionShift = 4 + ipIHLMask = 0x0f + IPv4IHLStride = 4 +) + +// HeaderLength returns the value of the "header length" field of the IPv4 +// header. The length returned is in bytes. +func (b IPv4) HeaderLength() uint8 { + return (b[versIHL] & ipIHLMask) * IPv4IHLStride +} + +// SetHeaderLength sets the value of the "Internet Header Length" field. +func (b IPv4) SetHeaderLength(hdrLen uint8) { + if hdrLen > IPv4MaximumHeaderSize { + panic(fmt.Sprintf("got IPv4 Header size = %d, want <= %d", hdrLen, IPv4MaximumHeaderSize)) + } + b[versIHL] = (IPv4Version << ipVersionShift) | ((hdrLen / IPv4IHLStride) & ipIHLMask) +} + +// ID returns the value of the identifier field of the IPv4 header. +func (b IPv4) ID() uint16 { + return binary.BigEndian.Uint16(b[id:]) +} + +// Protocol returns the value of the protocol field of the IPv4 header. +func (b IPv4) Protocol() uint8 { + return b[protocol] +} + +// Flags returns the "flags" field of the IPv4 header. +func (b IPv4) Flags() uint8 { + return uint8(binary.BigEndian.Uint16(b[flagsFO:]) >> 13) +} + +// More returns whether the more fragments flag is set. +func (b IPv4) More() bool { + return b.Flags()&IPv4FlagMoreFragments != 0 +} + +// TTL returns the "TTL" field of the IPv4 header. +func (b IPv4) TTL() uint8 { + return b[ttl] +} + +// FragmentOffset returns the "fragment offset" field of the IPv4 header. +func (b IPv4) FragmentOffset() uint16 { + return binary.BigEndian.Uint16(b[flagsFO:]) << 3 +} + +// TotalLength returns the "total length" field of the IPv4 header. +func (b IPv4) TotalLength() uint16 { + return binary.BigEndian.Uint16(b[IPv4TotalLenOffset:]) +} + +// Checksum returns the checksum field of the IPv4 header. +func (b IPv4) Checksum() uint16 { + return binary.BigEndian.Uint16(b[xsum:]) +} + +// SourceAddress returns the "source address" field of the IPv4 header. +func (b IPv4) SourceAddress() tcpip.Address { + return tcpip.AddrFrom4([4]byte(b[srcAddr : srcAddr+IPv4AddressSize])) +} + +// DestinationAddress returns the "destination address" field of the IPv4 +// header. +func (b IPv4) DestinationAddress() tcpip.Address { + return tcpip.AddrFrom4([4]byte(b[dstAddr : dstAddr+IPv4AddressSize])) +} + +// SourceAddressSlice returns the "source address" field of the IPv4 header as a +// byte slice. +func (b IPv4) SourceAddressSlice() []byte { + return []byte(b[srcAddr : srcAddr+IPv4AddressSize]) +} + +// DestinationAddressSlice returns the "destination address" field of the IPv4 +// header as a byte slice. +func (b IPv4) DestinationAddressSlice() []byte { + return []byte(b[dstAddr : dstAddr+IPv4AddressSize]) +} + +// SetSourceAddressWithChecksumUpdate implements ChecksummableNetwork. +func (b IPv4) SetSourceAddressWithChecksumUpdate(new tcpip.Address) { + b.SetChecksum(^checksumUpdate2ByteAlignedAddress(^b.Checksum(), b.SourceAddress(), new)) + b.SetSourceAddress(new) +} + +// SetDestinationAddressWithChecksumUpdate implements ChecksummableNetwork. +func (b IPv4) SetDestinationAddressWithChecksumUpdate(new tcpip.Address) { + b.SetChecksum(^checksumUpdate2ByteAlignedAddress(^b.Checksum(), b.DestinationAddress(), new)) + b.SetDestinationAddress(new) +} + +// padIPv4OptionsLength returns the total length for IPv4 options of length l +// after applying padding according to RFC 791: +// +// The internet header padding is used to ensure that the internet +// header ends on a 32 bit boundary. +func padIPv4OptionsLength(length uint8) uint8 { + return (length + IPv4IHLStride - 1) & ^uint8(IPv4IHLStride-1) +} + +// IPv4Options is a buffer that holds all the raw IP options. +type IPv4Options []byte + +// Options returns a buffer holding the options. +func (b IPv4) Options() IPv4Options { + hdrLen := b.HeaderLength() + return IPv4Options(b[options:hdrLen:hdrLen]) +} + +// TransportProtocol implements Network.TransportProtocol. +func (b IPv4) TransportProtocol() tcpip.TransportProtocolNumber { + return tcpip.TransportProtocolNumber(b.Protocol()) +} + +// Payload implements Network.Payload. +func (b IPv4) Payload() []byte { + return b[b.HeaderLength():][:b.PayloadLength()] +} + +// PayloadLength returns the length of the payload portion of the IPv4 packet. +func (b IPv4) PayloadLength() uint16 { + return b.TotalLength() - uint16(b.HeaderLength()) +} + +// TOS returns the "type of service" field of the IPv4 header. +func (b IPv4) TOS() (uint8, uint32) { + return b[tos], 0 +} + +// SetTOS sets the "type of service" field of the IPv4 header. +func (b IPv4) SetTOS(v uint8, _ uint32) { + b[tos] = v +} + +// SetTTL sets the "Time to Live" field of the IPv4 header. +func (b IPv4) SetTTL(v byte) { + b[ttl] = v +} + +// SetTotalLength sets the "total length" field of the IPv4 header. +func (b IPv4) SetTotalLength(totalLength uint16) { + binary.BigEndian.PutUint16(b[IPv4TotalLenOffset:], totalLength) +} + +// SetChecksum sets the checksum field of the IPv4 header. +func (b IPv4) SetChecksum(v uint16) { + checksum.Put(b[xsum:], v) +} + +// SetFlagsFragmentOffset sets the "flags" and "fragment offset" fields of the +// IPv4 header. +func (b IPv4) SetFlagsFragmentOffset(flags uint8, offset uint16) { + v := (uint16(flags) << 13) | (offset >> 3) + binary.BigEndian.PutUint16(b[flagsFO:], v) +} + +// SetID sets the identification field. +func (b IPv4) SetID(v uint16) { + binary.BigEndian.PutUint16(b[id:], v) +} + +// SetSourceAddress sets the "source address" field of the IPv4 header. +func (b IPv4) SetSourceAddress(addr tcpip.Address) { + copy(b[srcAddr:srcAddr+IPv4AddressSize], addr.AsSlice()) +} + +// SetDestinationAddress sets the "destination address" field of the IPv4 +// header. +func (b IPv4) SetDestinationAddress(addr tcpip.Address) { + copy(b[dstAddr:dstAddr+IPv4AddressSize], addr.AsSlice()) +} + +// CalculateChecksum calculates the checksum of the IPv4 header. +func (b IPv4) CalculateChecksum() uint16 { + // return checksum.Checksum(b[:b.HeaderLength()], 0) + xsum0 := checksum.Checksum(b[:xsum], 0) + xsum0 = checksum.Checksum(b[xsum+2:b.HeaderLength()], xsum0) + return xsum0 +} + +// Encode encodes all the fields of the IPv4 header. +func (b IPv4) Encode(i *IPv4Fields) { + // The size of the options defines the size of the whole header and thus the + // IHL field. Options are rare and this is a heavily used function so it is + // worth a bit of optimisation here to keep the serializer out of the fast + // path. + hdrLen := uint8(IPv4MinimumSize) + if len(i.Options) != 0 { + hdrLen += i.Options.Serialize(b[options:]) + } + if hdrLen > IPv4MaximumHeaderSize { + panic(fmt.Sprintf("%d is larger than maximum IPv4 header size of %d", hdrLen, IPv4MaximumHeaderSize)) + } + b.SetHeaderLength(hdrLen) + b[tos] = i.TOS + b.SetTotalLength(i.TotalLength) + binary.BigEndian.PutUint16(b[id:], i.ID) + b.SetFlagsFragmentOffset(i.Flags, i.FragmentOffset) + b[ttl] = i.TTL + b[protocol] = i.Protocol + b.SetChecksum(i.Checksum) + copy(b[srcAddr:srcAddr+IPv4AddressSize], i.SrcAddr.AsSlice()) + copy(b[dstAddr:dstAddr+IPv4AddressSize], i.DstAddr.AsSlice()) +} + +// EncodePartial updates the total length and checksum fields of IPv4 header, +// taking in the partial checksum, which is the checksum of the header without +// the total length and checksum fields. It is useful in cases when similar +// packets are produced. +func (b IPv4) EncodePartial(partialChecksum, totalLength uint16) { + b.SetTotalLength(totalLength) + xsum := checksum.Checksum(b[IPv4TotalLenOffset:IPv4TotalLenOffset+2], partialChecksum) + b.SetChecksum(^xsum) +} + +// IsValid performs basic validation on the packet. +func (b IPv4) IsValid(pktSize int) bool { + if len(b) < IPv4MinimumSize { + return false + } + + hlen := int(b.HeaderLength()) + tlen := int(b.TotalLength()) + if hlen < IPv4MinimumSize || hlen > tlen || tlen > pktSize { + return false + } + + if IPVersion(b) != IPv4Version { + return false + } + + return true +} + +// IsV4LinkLocalUnicastAddress determines if the provided address is an IPv4 +// link-local unicast address. +func IsV4LinkLocalUnicastAddress(addr tcpip.Address) bool { + return ipv4LinkLocalUnicastSubnet.Contains(addr) +} + +// IsV4LinkLocalMulticastAddress determines if the provided address is an IPv4 +// link-local multicast address. +func IsV4LinkLocalMulticastAddress(addr tcpip.Address) bool { + return ipv4LinkLocalMulticastSubnet.Contains(addr) +} + +// IsChecksumValid returns true iff the IPv4 header's checksum is valid. +func (b IPv4) IsChecksumValid() bool { + // There has been some confusion regarding verifying checksums. We need + // just look for negative 0 (0xffff) as the checksum, as it's not possible to + // get positive 0 (0) for the checksum. Some bad implementations could get it + // when doing entry replacement in the early days of the Internet, + // however the lore that one needs to check for both persists. + // + // RFC 1624 section 1 describes the source of this confusion as: + // [the partial recalculation method described in RFC 1071] computes a + // result for certain cases that differs from the one obtained from + // scratch (one's complement of one's complement sum of the original + // fields). + // + // However RFC 1624 section 5 clarifies that if using the verification method + // "recommended by RFC 1071, it does not matter if an intermediate system + // generated a -0 instead of +0". + // + // RFC1071 page 1 specifies the verification method as: + // (3) To check a checksum, the 1's complement sum is computed over the + // same set of octets, including the checksum field. If the result + // is all 1 bits (-0 in 1's complement arithmetic), the check + // succeeds. + //return b.CalculateChecksum() == 0xffff + return checksum.Checksum(b[:b.HeaderLength()], 0) == 0xffff +} + +// IsV4MulticastAddress determines if the provided address is an IPv4 multicast +// address (range 224.0.0.0 to 239.255.255.255). The four most significant bits +// will be 1110 = 0xe0. +func IsV4MulticastAddress(addr tcpip.Address) bool { + if addr.BitLen() != IPv4AddressSizeBits { + return false + } + addrBytes := addr.As4() + return (addrBytes[0] & 0xf0) == 0xe0 +} + +// IsV4LoopbackAddress determines if the provided address is an IPv4 loopback +// address (belongs to 127.0.0.0/8 subnet). See RFC 1122 section 3.2.1.3. +func IsV4LoopbackAddress(addr tcpip.Address) bool { + if addr.BitLen() != IPv4AddressSizeBits { + return false + } + addrBytes := addr.As4() + return addrBytes[0] == 0x7f +} + +// ========================= Options ========================== + +// An IPv4OptionType can hold the value for the Type in an IPv4 option. +type IPv4OptionType byte + +// These constants are needed to identify individual options in the option list. +// While RFC 791 (page 31) says "Every internet module must be able to act on +// every option." This has not generally been adhered to and some options have +// very low rates of support. We do not support options other than those shown +// below. + +const ( + // IPv4OptionListEndType is the option type for the End Of Option List + // option. Anything following is ignored. + IPv4OptionListEndType IPv4OptionType = 0 + + // IPv4OptionNOPType is the No-Operation option. May appear between other + // options and may appear multiple times. + IPv4OptionNOPType IPv4OptionType = 1 + + // IPv4OptionRouterAlertType is the option type for the Router Alert option, + // defined in RFC 2113 Section 2.1. + IPv4OptionRouterAlertType IPv4OptionType = 20 | 0x80 + + // IPv4OptionRecordRouteType is used by each router on the path of the packet + // to record its path. It is carried over to an Echo Reply. + IPv4OptionRecordRouteType IPv4OptionType = 7 + + // IPv4OptionTimestampType is the option type for the Timestamp option. + IPv4OptionTimestampType IPv4OptionType = 68 + + // IPv4OptionExperimentType is the option type for the Experiment option. + IPv4OptionExperimentType IPv4OptionType = 30 + + // ipv4OptionTypeOffset is the offset in an option of its type field. + ipv4OptionTypeOffset = 0 + + // IPv4OptionLengthOffset is the offset in an option of its length field. + IPv4OptionLengthOffset = 1 +) + +// IPv4OptParameterProblem indicates that a Parameter Problem message +// should be generated, and gives the offset in the current entity +// that should be used in that packet. +type IPv4OptParameterProblem struct { + Pointer uint8 + NeedICMP bool +} + +// IPv4Option is an interface representing various option types. +type IPv4Option interface { + // Type returns the type identifier of the option. + Type() IPv4OptionType + + // Size returns the size of the option in bytes. + Size() uint8 + + // Contents returns a slice holding the contents of the option. + Contents() []byte +} + +var _ IPv4Option = (*IPv4OptionGeneric)(nil) + +// IPv4OptionGeneric is an IPv4 Option of unknown type. +type IPv4OptionGeneric []byte + +// Type implements IPv4Option. +func (o *IPv4OptionGeneric) Type() IPv4OptionType { + return IPv4OptionType((*o)[ipv4OptionTypeOffset]) +} + +// Size implements IPv4Option. +func (o *IPv4OptionGeneric) Size() uint8 { return uint8(len(*o)) } + +// Contents implements IPv4Option. +func (o *IPv4OptionGeneric) Contents() []byte { return *o } + +// IPv4OptionIterator is an iterator pointing to a specific IP option +// at any point of time. It also holds information as to a new options buffer +// that we are building up to hand back to the caller. +// TODO(https://gvisor.dev/issues/5513): Add unit tests for IPv4OptionIterator. +type IPv4OptionIterator struct { + options IPv4Options + // ErrCursor is where we are while parsing options. It is exported as any + // resulting ICMP packet is supposed to have a pointer to the byte within + // the IP packet where the error was detected. + ErrCursor uint8 + nextErrCursor uint8 + newOptions [IPv4MaximumOptionsSize]byte + writePoint int +} + +// MakeIterator sets up and returns an iterator of options. It also sets up the +// building of a new option set. +func (o IPv4Options) MakeIterator() IPv4OptionIterator { + return IPv4OptionIterator{ + options: o, + nextErrCursor: IPv4MinimumSize, + } +} + +// InitReplacement copies the option into the new option buffer. +func (i *IPv4OptionIterator) InitReplacement(option IPv4Option) IPv4Options { + replacementOption := i.RemainingBuffer()[:option.Size()] + if copied := copy(replacementOption, option.Contents()); copied != len(replacementOption) { + panic(fmt.Sprintf("copied %d bytes in the replacement option buffer, expected %d bytes", copied, len(replacementOption))) + } + return replacementOption +} + +// RemainingBuffer returns the remaining (unused) part of the new option buffer, +// into which a new option may be written. +func (i *IPv4OptionIterator) RemainingBuffer() IPv4Options { + return i.newOptions[i.writePoint:] +} + +// ConsumeBuffer marks a portion of the new buffer as used. +func (i *IPv4OptionIterator) ConsumeBuffer(size int) { + i.writePoint += size +} + +// PushNOPOrEnd puts one of the single byte options onto the new options. +// Only values 0 or 1 (ListEnd or NOP) are valid input. +func (i *IPv4OptionIterator) PushNOPOrEnd(val IPv4OptionType) { + if val > IPv4OptionNOPType { + panic(fmt.Sprintf("invalid option type %d pushed onto option build buffer", val)) + } + i.newOptions[i.writePoint] = byte(val) + i.writePoint++ +} + +// Finalize returns the completed replacement options buffer padded +// as needed. +func (i *IPv4OptionIterator) Finalize() IPv4Options { + // RFC 791 page 31 says: + // The options might not end on a 32-bit boundary. The internet header + // must be filled out with octets of zeros. The first of these would + // be interpreted as the end-of-options option, and the remainder as + // internet header padding. + // Since the buffer is already zero filled we just need to step the write + // pointer up to the next multiple of 4. + options := IPv4Options(i.newOptions[:(i.writePoint+0x3) & ^0x3]) + // Poison the write pointer. + i.writePoint = len(i.newOptions) + return options +} + +// Next returns the next IP option in the buffer/list of IP options. +// It returns +// - A slice of bytes holding the next option or nil if there is error. +// - A boolean which is true if parsing of all the options is complete. +// Undefined in the case of error. +// - An error indication which is non-nil if an error condition was found. +func (i *IPv4OptionIterator) Next() (IPv4Option, bool, *IPv4OptParameterProblem) { + // The opts slice gets shorter as we process the options. When we have no + // bytes left we are done. + if len(i.options) == 0 { + return nil, true, nil + } + + i.ErrCursor = i.nextErrCursor + + optType := IPv4OptionType(i.options[ipv4OptionTypeOffset]) + + if optType == IPv4OptionNOPType || optType == IPv4OptionListEndType { + optionBody := i.options[:1] + i.options = i.options[1:] + i.nextErrCursor = i.ErrCursor + 1 + retval := IPv4OptionGeneric(optionBody) + return &retval, false, nil + } + + // There are no more single byte options defined. All the rest have a length + // field so we need to sanity check it. + if len(i.options) == 1 { + return nil, false, &IPv4OptParameterProblem{ + Pointer: i.ErrCursor, + NeedICMP: true, + } + } + + optLen := i.options[IPv4OptionLengthOffset] + + if optLen <= IPv4OptionLengthOffset || optLen > uint8(len(i.options)) { + // The actual error is in the length (2nd byte of the option) but we + // return the start of the option for compatibility with Linux. + + return nil, false, &IPv4OptParameterProblem{ + Pointer: i.ErrCursor, + NeedICMP: true, + } + } + + optionBody := i.options[:optLen] + i.nextErrCursor = i.ErrCursor + optLen + i.options = i.options[optLen:] + + // Check the length of some option types that we know. + switch optType { + case IPv4OptionTimestampType: + if optLen < IPv4OptionTimestampHdrLength { + i.ErrCursor++ + return nil, false, &IPv4OptParameterProblem{ + Pointer: i.ErrCursor, + NeedICMP: true, + } + } + retval := IPv4OptionTimestamp(optionBody) + return &retval, false, nil + + case IPv4OptionRecordRouteType: + if optLen < IPv4OptionRecordRouteHdrLength { + i.ErrCursor++ + return nil, false, &IPv4OptParameterProblem{ + Pointer: i.ErrCursor, + NeedICMP: true, + } + } + retval := IPv4OptionRecordRoute(optionBody) + return &retval, false, nil + + case IPv4OptionRouterAlertType: + if optLen != IPv4OptionRouterAlertLength { + i.ErrCursor++ + return nil, false, &IPv4OptParameterProblem{ + Pointer: i.ErrCursor, + NeedICMP: true, + } + } + retval := IPv4OptionRouterAlert(optionBody) + return &retval, false, nil + + case IPv4OptionExperimentType: + if optLen != IPv4OptionExperimentLength { + i.ErrCursor++ + return nil, false, &IPv4OptParameterProblem{ + Pointer: i.ErrCursor, + NeedICMP: true, + } + } + retval := IPv4OptionExperiment(optionBody) + return &retval, false, nil + } + retval := IPv4OptionGeneric(optionBody) + return &retval, false, nil +} + +// +// IP Timestamp option - RFC 791 page 22. +// +--------+--------+--------+--------+ +// |01000100| length | pointer|oflw|flg| +// +--------+--------+--------+--------+ +// | internet address | +// +--------+--------+--------+--------+ +// | timestamp | +// +--------+--------+--------+--------+ +// | ... | +// +// Type = 68 +// +// The Option Length is the number of octets in the option counting +// the type, length, pointer, and overflow/flag octets (maximum +// length 40). +// +// The Pointer is the number of octets from the beginning of this +// option to the end of timestamps plus one (i.e., it points to the +// octet beginning the space for next timestamp). The smallest +// legal value is 5. The timestamp area is full when the pointer +// is greater than the length. +// +// The Overflow (oflw) [4 bits] is the number of IP modules that +// cannot register timestamps due to lack of space. +// +// The Flag (flg) [4 bits] values are +// +// 0 -- time stamps only, stored in consecutive 32-bit words, +// +// 1 -- each timestamp is preceded with internet address of the +// registering entity, +// +// 3 -- the internet address fields are prespecified. An IP +// module only registers its timestamp if it matches its own +// address with the next specified internet address. +// +// Timestamps are defined in RFC 791 page 22 as milliseconds since midnight UTC. +// +// The Timestamp is a right-justified, 32-bit timestamp in +// milliseconds since midnight UT. If the time is not available in +// milliseconds or cannot be provided with respect to midnight UT +// then any time may be inserted as a timestamp provided the high +// order bit of the timestamp field is set to one to indicate the +// use of a non-standard value. + +// IPv4OptTSFlags sefines the values expected in the Timestamp +// option Flags field. +type IPv4OptTSFlags uint8 + +// Timestamp option specific related constants. +const ( + // IPv4OptionTimestampHdrLength is the length of the timestamp option header. + IPv4OptionTimestampHdrLength = 4 + + // IPv4OptionTimestampSize is the size of an IP timestamp. + IPv4OptionTimestampSize = 4 + + // IPv4OptionTimestampWithAddrSize is the size of an IP timestamp + Address. + IPv4OptionTimestampWithAddrSize = IPv4AddressSize + IPv4OptionTimestampSize + + // IPv4OptionTimestampMaxSize is limited by space for options + IPv4OptionTimestampMaxSize = IPv4MaximumOptionsSize + + // IPv4OptionTimestampOnlyFlag is a flag indicating that only timestamp + // is present. + IPv4OptionTimestampOnlyFlag IPv4OptTSFlags = 0 + + // IPv4OptionTimestampWithIPFlag is a flag indicating that both timestamps and + // IP are present. + IPv4OptionTimestampWithIPFlag IPv4OptTSFlags = 1 + + // IPv4OptionTimestampWithPredefinedIPFlag is a flag indicating that + // predefined IP is present. + IPv4OptionTimestampWithPredefinedIPFlag IPv4OptTSFlags = 3 +) + +// ipv4TimestampTime provides the current time as specified in RFC 791. +func ipv4TimestampTime(clock tcpip.Clock) uint32 { + // Per RFC 791 page 21: + // The Timestamp is a right-justified, 32-bit timestamp in + // milliseconds since midnight UT. + now := clock.Now().UTC() + midnight := now.Truncate(24 * time.Hour) + return uint32(now.Sub(midnight).Milliseconds()) +} + +// IP Timestamp option fields. +const ( + // IPv4OptTSPointerOffset is the offset of the Timestamp pointer field. + IPv4OptTSPointerOffset = 2 + + // IPv4OptTSPointerOffset is the offset of the combined Flag and Overflow + // fields, (each being 4 bits). + IPv4OptTSOFLWAndFLGOffset = 3 + // These constants define the sub byte fields of the Flag and OverFlow field. + ipv4OptionTimestampOverflowshift = 4 + ipv4OptionTimestampFlagsMask byte = 0x0f +) + +var _ IPv4Option = (*IPv4OptionTimestamp)(nil) + +// IPv4OptionTimestamp is a Timestamp option from RFC 791. +type IPv4OptionTimestamp []byte + +// Type implements IPv4Option.Type(). +func (ts *IPv4OptionTimestamp) Type() IPv4OptionType { return IPv4OptionTimestampType } + +// Size implements IPv4Option. +func (ts *IPv4OptionTimestamp) Size() uint8 { return uint8(len(*ts)) } + +// Contents implements IPv4Option. +func (ts *IPv4OptionTimestamp) Contents() []byte { return *ts } + +// Pointer returns the pointer field in the IP Timestamp option. +func (ts *IPv4OptionTimestamp) Pointer() uint8 { + return (*ts)[IPv4OptTSPointerOffset] +} + +// Flags returns the flags field in the IP Timestamp option. +func (ts *IPv4OptionTimestamp) Flags() IPv4OptTSFlags { + return IPv4OptTSFlags((*ts)[IPv4OptTSOFLWAndFLGOffset] & ipv4OptionTimestampFlagsMask) +} + +// Overflow returns the Overflow field in the IP Timestamp option. +func (ts *IPv4OptionTimestamp) Overflow() uint8 { + return (*ts)[IPv4OptTSOFLWAndFLGOffset] >> ipv4OptionTimestampOverflowshift +} + +// IncOverflow increments the Overflow field in the IP Timestamp option. It +// returns the incremented value. If the return value is 0 then the field +// overflowed. +func (ts *IPv4OptionTimestamp) IncOverflow() uint8 { + (*ts)[IPv4OptTSOFLWAndFLGOffset] += 1 << ipv4OptionTimestampOverflowshift + return ts.Overflow() +} + +// UpdateTimestamp updates the fields of the next free timestamp slot. +func (ts *IPv4OptionTimestamp) UpdateTimestamp(addr tcpip.Address, clock tcpip.Clock) { + slot := (*ts)[ts.Pointer()-1:] + + switch ts.Flags() { + case IPv4OptionTimestampOnlyFlag: + binary.BigEndian.PutUint32(slot, ipv4TimestampTime(clock)) + (*ts)[IPv4OptTSPointerOffset] += IPv4OptionTimestampSize + case IPv4OptionTimestampWithIPFlag: + if n := copy(slot, addr.AsSlice()); n != IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, IPv4AddressSize)) + } + binary.BigEndian.PutUint32(slot[IPv4AddressSize:], ipv4TimestampTime(clock)) + (*ts)[IPv4OptTSPointerOffset] += IPv4OptionTimestampWithAddrSize + case IPv4OptionTimestampWithPredefinedIPFlag: + if tcpip.AddrFrom4([4]byte(slot[:IPv4AddressSize])) == addr { + binary.BigEndian.PutUint32(slot[IPv4AddressSize:], ipv4TimestampTime(clock)) + (*ts)[IPv4OptTSPointerOffset] += IPv4OptionTimestampWithAddrSize + } + } +} + +// RecordRoute option specific related constants. +// +// from RFC 791 page 20: +// +// Record Route +// +// +--------+--------+--------+---------//--------+ +// |00000111| length | pointer| route data | +// +--------+--------+--------+---------//--------+ +// Type=7 +// +// The record route option provides a means to record the route of +// an internet datagram. +// +// The option begins with the option type code. The second octet +// is the option length which includes the option type code and the +// length octet, the pointer octet, and length-3 octets of route +// data. The third octet is the pointer into the route data +// indicating the octet which begins the next area to store a route +// address. The pointer is relative to this option, and the +// smallest legal value for the pointer is 4. +const ( + // IPv4OptionRecordRouteHdrLength is the length of the Record Route option + // header. + IPv4OptionRecordRouteHdrLength = 3 + + // IPv4OptRRPointerOffset is the offset to the pointer field in an RR + // option, which points to the next free slot in the list of addresses. + IPv4OptRRPointerOffset = 2 +) + +var _ IPv4Option = (*IPv4OptionRecordRoute)(nil) + +// IPv4OptionRecordRoute is an IPv4 RecordRoute option defined by RFC 791. +type IPv4OptionRecordRoute []byte + +// Pointer returns the pointer field in the IP RecordRoute option. +func (rr *IPv4OptionRecordRoute) Pointer() uint8 { + return (*rr)[IPv4OptRRPointerOffset] +} + +// StoreAddress stores the given IPv4 address into the next free slot. +func (rr *IPv4OptionRecordRoute) StoreAddress(addr tcpip.Address) { + start := rr.Pointer() - 1 // A one based number. + // start and room checked by caller. + if n := copy((*rr)[start:], addr.AsSlice()); n != IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, IPv4AddressSize)) + } + (*rr)[IPv4OptRRPointerOffset] += IPv4AddressSize +} + +// Type implements IPv4Option. +func (rr *IPv4OptionRecordRoute) Type() IPv4OptionType { return IPv4OptionRecordRouteType } + +// Size implements IPv4Option. +func (rr *IPv4OptionRecordRoute) Size() uint8 { return uint8(len(*rr)) } + +// Contents implements IPv4Option. +func (rr *IPv4OptionRecordRoute) Contents() []byte { return *rr } + +// Router Alert option specific related constants. +// +// from RFC 2113 section 2.1: +// +// +--------+--------+--------+--------+ +// |10010100|00000100| 2 octet value | +// +--------+--------+--------+--------+ +// +// Type: +// Copied flag: 1 (all fragments must carry the option) +// Option class: 0 (control) +// Option number: 20 (decimal) +// +// Length: 4 +// +// Value: A two octet code with the following values: +// 0 - Router shall examine packet +// 1-65535 - Reserved +const ( + // IPv4OptionRouterAlertLength is the length of a Router Alert option. + IPv4OptionRouterAlertLength = 4 + + // IPv4OptionRouterAlertValue is the only permissible value of the 16 bit + // payload of the router alert option. + IPv4OptionRouterAlertValue = 0 + + // IPv4OptionRouterAlertValueOffset is the offset for the value of a + // RouterAlert option. + IPv4OptionRouterAlertValueOffset = 2 +) + +var _ IPv4Option = (*IPv4OptionRouterAlert)(nil) + +// IPv4OptionRouterAlert is an IPv4 RouterAlert option defined by RFC 2113. +type IPv4OptionRouterAlert []byte + +// Type implements IPv4Option. +func (*IPv4OptionRouterAlert) Type() IPv4OptionType { return IPv4OptionRouterAlertType } + +// Size implements IPv4Option. +func (ra *IPv4OptionRouterAlert) Size() uint8 { return uint8(len(*ra)) } + +// Contents implements IPv4Option. +func (ra *IPv4OptionRouterAlert) Contents() []byte { return *ra } + +// Value returns the value of the IPv4OptionRouterAlert. +func (ra *IPv4OptionRouterAlert) Value() uint16 { + return binary.BigEndian.Uint16(ra.Contents()[IPv4OptionRouterAlertValueOffset:]) +} + +// Experiment option specific related constants. +const ( + // IPv4OptionExperimentLength is the length of an Experiment option. + IPv4OptionExperimentLength = 4 + + // IPv4OptionExperimentValueOffset is the offset for the value of an + // Experiment option. + IPv4OptionExperimentValueOffset = 2 +) + +var _ IPv4Option = (*IPv4OptionExperiment)(nil) + +// IPv4OptionExperiment is an IPv4 option defined by RFC 4727. +type IPv4OptionExperiment []byte + +// Type implements IPv4Option. +func (*IPv4OptionExperiment) Type() IPv4OptionType { return IPv4OptionExperimentType } + +// Size implements IPv4Option. +func (*IPv4OptionExperiment) Size() uint8 { return uint8(IPv4OptionExperimentLength) } + +// Contents implements IPv4Option. +func (ex *IPv4OptionExperiment) Contents() []byte { return *ex } + +// Value returns the value of the IPv4OptionRouterAlert. +func (ex *IPv4OptionExperiment) Value() uint16 { + return binary.BigEndian.Uint16(ex.Contents()[IPv4OptionExperimentValueOffset:]) +} + +// IPv4SerializableOption is an interface to represent serializable IPv4 option +// types. +type IPv4SerializableOption interface { + // optionType returns the type identifier of the option. + optionType() IPv4OptionType +} + +// IPv4SerializableOptionPayload is an interface providing serialization of the +// payload of an IPv4 option. +type IPv4SerializableOptionPayload interface { + // length returns the size of the payload. + length() uint8 + + // serializeInto serializes the payload into the provided byte buffer. + // + // Note, the caller MUST provide a byte buffer with size of at least + // Length. Implementers of this function may assume that the byte buffer + // is of sufficient size. serializeInto MUST panic if the provided byte + // buffer is not of sufficient size. + // + // serializeInto will return the number of bytes that was used to + // serialize the receiver. Implementers must only use the number of + // bytes required to serialize the receiver. Callers MAY provide a + // larger buffer than required to serialize into. + serializeInto(buffer []byte) uint8 +} + +// IPv4OptionsSerializer is a serializer for IPv4 options. +type IPv4OptionsSerializer []IPv4SerializableOption + +// Length returns the total number of bytes required to serialize the options. +func (s IPv4OptionsSerializer) Length() uint8 { + var total uint8 + for _, opt := range s { + total++ + if withPayload, ok := opt.(IPv4SerializableOptionPayload); ok { + // Add 1 to reported length to account for the length byte. + total += 1 + withPayload.length() + } + } + return padIPv4OptionsLength(total) +} + +// Serialize serializes the provided list of IPV4 options into b. +// +// Note, b must be of sufficient size to hold all the options in s. See +// IPv4OptionsSerializer.Length for details on the getting the total size +// of a serialized IPv4OptionsSerializer. +// +// Serialize panics if b is not of sufficient size to hold all the options in s. +func (s IPv4OptionsSerializer) Serialize(b []byte) uint8 { + var total uint8 + for _, opt := range s { + ty := opt.optionType() + if withPayload, ok := opt.(IPv4SerializableOptionPayload); ok { + // Serialize first to reduce bounds checks. + l := 2 + withPayload.serializeInto(b[2:]) + b[0] = byte(ty) + b[1] = l + b = b[l:] + total += l + continue + } + // Options without payload consist only of the type field. + // + // NB: Repeating code from the branch above is intentional to minimize + // bounds checks. + b[0] = byte(ty) + b = b[1:] + total++ + } + + // According to RFC 791: + // + // The internet header padding is used to ensure that the internet + // header ends on a 32 bit boundary. The padding is zero. + padded := padIPv4OptionsLength(total) + b = b[:padded-total] + clear(b) + return padded +} + +var ( + _ IPv4SerializableOptionPayload = (*IPv4SerializableRouterAlertOption)(nil) + _ IPv4SerializableOption = (*IPv4SerializableRouterAlertOption)(nil) +) + +// IPv4SerializableRouterAlertOption provides serialization of the Router Alert +// IPv4 option according to RFC 2113. +type IPv4SerializableRouterAlertOption struct{} + +// Type implements IPv4SerializableOption. +func (*IPv4SerializableRouterAlertOption) optionType() IPv4OptionType { + return IPv4OptionRouterAlertType +} + +// Length implements IPv4SerializableOption. +func (*IPv4SerializableRouterAlertOption) length() uint8 { + return IPv4OptionRouterAlertLength - IPv4OptionRouterAlertValueOffset +} + +// SerializeInto implements IPv4SerializableOption. +func (o *IPv4SerializableRouterAlertOption) serializeInto(buffer []byte) uint8 { + binary.BigEndian.PutUint16(buffer, IPv4OptionRouterAlertValue) + return o.length() +} + +var ( + _ IPv4SerializableOptionPayload = (*IPv4SerializableExperimentOption)(nil) + _ IPv4SerializableOption = (*IPv4SerializableExperimentOption)(nil) +) + +// IPv4SerializableExperimentOption provides serialization for the IPv4 +// Experiment option. +type IPv4SerializableExperimentOption struct { + Tag uint16 +} + +func (*IPv4SerializableExperimentOption) optionType() IPv4OptionType { + return IPv4OptionExperimentType +} + +func (*IPv4SerializableExperimentOption) length() uint8 { + return IPv4OptionExperimentLength - IPv4OptionExperimentValueOffset +} + +func (o *IPv4SerializableExperimentOption) serializeInto(buffer []byte) uint8 { + binary.BigEndian.PutUint16(buffer, o.Tag) + return o.length() +} + +var _ IPv4SerializableOption = (*IPv4SerializableNOPOption)(nil) + +// IPv4SerializableNOPOption provides serialization for the IPv4 no-op option. +type IPv4SerializableNOPOption struct{} + +// Type implements IPv4SerializableOption. +func (*IPv4SerializableNOPOption) optionType() IPv4OptionType { + return IPv4OptionNOPType +} + +var _ IPv4SerializableOption = (*IPv4SerializableListEndOption)(nil) + +// IPv4SerializableListEndOption provides serialization for the IPv4 List End +// option. +type IPv4SerializableListEndOption struct{} + +// Type implements IPv4SerializableOption. +func (*IPv4SerializableListEndOption) optionType() IPv4OptionType { + return IPv4OptionListEndType +} diff --git a/pkg/tcpip/header/ipv6.go b/pkg/tcpip/header/ipv6.go new file mode 100644 index 0000000..53584df --- /dev/null +++ b/pkg/tcpip/header/ipv6.go @@ -0,0 +1,597 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + versTCFL = 0 + // IPv6PayloadLenOffset is the offset of the PayloadLength field in + // IPv6 header. + IPv6PayloadLenOffset = 4 + // IPv6NextHeaderOffset is the offset of the NextHeader field in + // IPv6 header. + IPv6NextHeaderOffset = 6 + hopLimit = 7 + v6SrcAddr = 8 + v6DstAddr = v6SrcAddr + IPv6AddressSize + + // IPv6FixedHeaderSize is the size of the fixed header. + IPv6FixedHeaderSize = v6DstAddr + IPv6AddressSize +) + +// IPv6Fields contains the fields of an IPv6 packet. It is used to describe the +// fields of a packet that needs to be encoded. +type IPv6Fields struct { + // TrafficClass is the "traffic class" field of an IPv6 packet. + TrafficClass uint8 + + // FlowLabel is the "flow label" field of an IPv6 packet. + FlowLabel uint32 + + // PayloadLength is the "payload length" field of an IPv6 packet, including + // the length of all extension headers. + PayloadLength uint16 + + // TransportProtocol is the transport layer protocol number. Serialized in the + // last "next header" field of the IPv6 header + extension headers. + TransportProtocol tcpip.TransportProtocolNumber + + // HopLimit is the "Hop Limit" field of an IPv6 packet. + HopLimit uint8 + + // SrcAddr is the "source ip address" of an IPv6 packet. + SrcAddr tcpip.Address + + // DstAddr is the "destination ip address" of an IPv6 packet. + DstAddr tcpip.Address + + // ExtensionHeaders are the extension headers following the IPv6 header. + ExtensionHeaders IPv6ExtHdrSerializer +} + +// IPv6 represents an ipv6 header stored in a byte array. +// Most of the methods of IPv6 access to the underlying slice without +// checking the boundaries and could panic because of 'index out of range'. +// Always call IsValid() to validate an instance of IPv6 before using other methods. +type IPv6 []byte + +const ( + // IPv6MinimumSize is the minimum size of a valid IPv6 packet. + IPv6MinimumSize = IPv6FixedHeaderSize + + // IPv6AddressSize is the size, in bytes, of an IPv6 address. + IPv6AddressSize = 16 + + // IPv6AddressSizeBits is the size, in bits, of an IPv6 address. + IPv6AddressSizeBits = 128 + + // IPv6MaximumPayloadSize is the maximum size of a valid IPv6 payload per + // RFC 8200 Section 4.5. + IPv6MaximumPayloadSize = 65535 + + // IPv6ProtocolNumber is IPv6's network protocol number. + IPv6ProtocolNumber tcpip.NetworkProtocolNumber = 0x86dd + + // IPv6Version is the version of the ipv6 protocol. + IPv6Version = 6 + + // IIDSize is the size of an interface identifier (IID), in bytes, as + // defined by RFC 4291 section 2.5.1. + IIDSize = 8 + + // IPv6MinimumMTU is the minimum MTU required by IPv6, per RFC 8200, + // section 5: + // IPv6 requires that every link in the Internet have an MTU of 1280 octets + // or greater. This is known as the IPv6 minimum link MTU. + IPv6MinimumMTU = 1280 + + // IIDOffsetInIPv6Address is the offset, in bytes, from the start + // of an IPv6 address to the beginning of the interface identifier + // (IID) for auto-generated addresses. That is, all bytes before + // the IIDOffsetInIPv6Address-th byte are the prefix bytes, and all + // bytes including and after the IIDOffsetInIPv6Address-th byte are + // for the IID. + IIDOffsetInIPv6Address = 8 + + // OpaqueIIDSecretKeyMinBytes is the recommended minimum number of bytes + // for the secret key used to generate an opaque interface identifier as + // outlined by RFC 7217. + OpaqueIIDSecretKeyMinBytes = 16 + + // ipv6MulticastAddressScopeByteIdx is the byte where the scope (scop) field + // is located within a multicast IPv6 address, as per RFC 4291 section 2.7. + ipv6MulticastAddressScopeByteIdx = 1 + + // ipv6MulticastAddressScopeMask is the mask for the scope (scop) field, + // within the byte holding the field, as per RFC 4291 section 2.7. + ipv6MulticastAddressScopeMask = 0xF +) + +var ( + // IPv6AllNodesMulticastAddress is a link-local multicast group that + // all IPv6 nodes MUST join, as per RFC 4291, section 2.8. Packets + // destined to this address will reach all nodes on a link. + // + // The address is ff02::1. + IPv6AllNodesMulticastAddress = tcpip.AddrFrom16([16]byte{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}) + + // IPv6AllRoutersInterfaceLocalMulticastAddress is an interface-local + // multicast group that all IPv6 routers MUST join, as per RFC 4291, section + // 2.8. Packets destined to this address will reach the router on an + // interface. + // + // The address is ff01::2. + IPv6AllRoutersInterfaceLocalMulticastAddress = tcpip.AddrFrom16([16]byte{0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}) + + // IPv6AllRoutersLinkLocalMulticastAddress is a link-local multicast group + // that all IPv6 routers MUST join, as per RFC 4291, section 2.8. Packets + // destined to this address will reach all routers on a link. + // + // The address is ff02::2. + IPv6AllRoutersLinkLocalMulticastAddress = tcpip.AddrFrom16([16]byte{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}) + + // IPv6AllRoutersSiteLocalMulticastAddress is a site-local multicast group + // that all IPv6 routers MUST join, as per RFC 4291, section 2.8. Packets + // destined to this address will reach all routers in a site. + // + // The address is ff05::2. + IPv6AllRoutersSiteLocalMulticastAddress = tcpip.AddrFrom16([16]byte{0xff, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}) + + // IPv6Loopback is the IPv6 Loopback address. + IPv6Loopback = tcpip.AddrFrom16([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}) + + // IPv6Any is the non-routable IPv6 "any" meta address. It is also + // known as the unspecified address. + IPv6Any = tcpip.AddrFrom16([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}) +) + +// IPv6EmptySubnet is the empty IPv6 subnet. It may also be known as the +// catch-all or wildcard subnet. That is, all IPv6 addresses are considered to +// be contained within this subnet. +var IPv6EmptySubnet = tcpip.AddressWithPrefix{ + Address: IPv6Any, + PrefixLen: 0, +}.Subnet() + +// IPv4MappedIPv6Subnet is the prefix for an IPv4 mapped IPv6 address as defined +// by RFC 4291 section 2.5.5. +var IPv4MappedIPv6Subnet = tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00}), + PrefixLen: 96, +}.Subnet() + +// IPv6LinkLocalPrefix is the prefix for IPv6 link-local addresses, as defined +// by RFC 4291 section 2.5.6. +// +// The prefix is fe80::/64 +var IPv6LinkLocalPrefix = tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16([16]byte{0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), + PrefixLen: 64, +} + +// PayloadLength returns the value of the "payload length" field of the ipv6 +// header. +func (b IPv6) PayloadLength() uint16 { + return binary.BigEndian.Uint16(b[IPv6PayloadLenOffset:]) +} + +// HopLimit returns the value of the "Hop Limit" field of the ipv6 header. +func (b IPv6) HopLimit() uint8 { + return b[hopLimit] +} + +// NextHeader returns the value of the "next header" field of the ipv6 header. +func (b IPv6) NextHeader() uint8 { + return b[IPv6NextHeaderOffset] +} + +// TransportProtocol implements Network.TransportProtocol. +func (b IPv6) TransportProtocol() tcpip.TransportProtocolNumber { + return tcpip.TransportProtocolNumber(b.NextHeader()) +} + +// Payload implements Network.Payload. +func (b IPv6) Payload() []byte { + return b[IPv6MinimumSize:][:b.PayloadLength()] +} + +// SourceAddress returns the "source address" field of the ipv6 header. +func (b IPv6) SourceAddress() tcpip.Address { + return tcpip.AddrFrom16([16]byte(b[v6SrcAddr:][:IPv6AddressSize])) +} + +// DestinationAddress returns the "destination address" field of the ipv6 +// header. +func (b IPv6) DestinationAddress() tcpip.Address { + return tcpip.AddrFrom16([16]byte(b[v6DstAddr:][:IPv6AddressSize])) +} + +// SourceAddressSlice returns the "source address" field of the ipv6 header as a +// byte slice. +func (b IPv6) SourceAddressSlice() []byte { + return []byte(b[v6SrcAddr:][:IPv6AddressSize]) +} + +// DestinationAddressSlice returns the "destination address" field of the ipv6 +// header as a byte slice. +func (b IPv6) DestinationAddressSlice() []byte { + return []byte(b[v6DstAddr:][:IPv6AddressSize]) +} + +// Checksum implements Network.Checksum. Given that IPv6 doesn't have a +// checksum, it just returns 0. +func (IPv6) Checksum() uint16 { + return 0 +} + +// TOS returns the "traffic class" and "flow label" fields of the ipv6 header. +func (b IPv6) TOS() (uint8, uint32) { + v := binary.BigEndian.Uint32(b[versTCFL:]) + return uint8(v >> 20), v & 0xfffff +} + +// SetTOS sets the "traffic class" and "flow label" fields of the ipv6 header. +func (b IPv6) SetTOS(t uint8, l uint32) { + vtf := (6 << 28) | (uint32(t) << 20) | (l & 0xfffff) + binary.BigEndian.PutUint32(b[versTCFL:], vtf) +} + +// SetPayloadLength sets the "payload length" field of the ipv6 header. +func (b IPv6) SetPayloadLength(payloadLength uint16) { + binary.BigEndian.PutUint16(b[IPv6PayloadLenOffset:], payloadLength) +} + +// SetSourceAddress sets the "source address" field of the ipv6 header. +func (b IPv6) SetSourceAddress(addr tcpip.Address) { + copy(b[v6SrcAddr:][:IPv6AddressSize], addr.AsSlice()) +} + +// SetDestinationAddress sets the "destination address" field of the ipv6 +// header. +func (b IPv6) SetDestinationAddress(addr tcpip.Address) { + copy(b[v6DstAddr:][:IPv6AddressSize], addr.AsSlice()) +} + +// SetHopLimit sets the value of the "Hop Limit" field. +func (b IPv6) SetHopLimit(v uint8) { + b[hopLimit] = v +} + +// SetNextHeader sets the value of the "next header" field of the ipv6 header. +func (b IPv6) SetNextHeader(v uint8) { + b[IPv6NextHeaderOffset] = v +} + +// SetChecksum implements Network.SetChecksum. Given that IPv6 doesn't have a +// checksum, it is empty. +func (IPv6) SetChecksum(uint16) { +} + +// Encode encodes all the fields of the ipv6 header. +func (b IPv6) Encode(i *IPv6Fields) { + extHdr := b[IPv6MinimumSize:] + b.SetTOS(i.TrafficClass, i.FlowLabel) + b.SetPayloadLength(i.PayloadLength) + b[hopLimit] = i.HopLimit + b.SetSourceAddress(i.SrcAddr) + b.SetDestinationAddress(i.DstAddr) + nextHeader, _ := i.ExtensionHeaders.Serialize(i.TransportProtocol, extHdr) + b[IPv6NextHeaderOffset] = nextHeader +} + +// IsValid performs basic validation on the packet. +func (b IPv6) IsValid(pktSize int) bool { + if len(b) < IPv6MinimumSize { + return false + } + + dlen := int(b.PayloadLength()) + if dlen > pktSize-IPv6MinimumSize { + return false + } + + if IPVersion(b) != IPv6Version { + return false + } + + return true +} + +// IsV4MappedAddress determines if the provided address is an IPv4 mapped +// address by checking if its prefix is 0:0:0:0:0:ffff::/96. +func IsV4MappedAddress(addr tcpip.Address) bool { + if addr.BitLen() != IPv6AddressSizeBits { + return false + } + + return IPv4MappedIPv6Subnet.Contains(addr) +} + +// IsV6MulticastAddress determines if the provided address is an IPv6 +// multicast address (anything starting with FF). +func IsV6MulticastAddress(addr tcpip.Address) bool { + if addr.BitLen() != IPv6AddressSizeBits { + return false + } + return addr.As16()[0] == 0xff +} + +// IsV6UnicastAddress determines if the provided address is a valid IPv6 +// unicast (and specified) address. That is, IsV6UnicastAddress returns +// true if addr contains IPv6AddressSize bytes, is not the unspecified +// address and is not a multicast address. +func IsV6UnicastAddress(addr tcpip.Address) bool { + if addr.BitLen() != IPv6AddressSizeBits { + return false + } + + // Must not be unspecified + if addr == IPv6Any { + return false + } + + // Return if not a multicast. + return addr.As16()[0] != 0xff +} + +var solicitedNodeMulticastPrefix = [13]byte{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff} + +// SolicitedNodeAddr computes the solicited-node multicast address. This is +// used for NDP. Described in RFC 4291. The argument must be a full-length IPv6 +// address. +func SolicitedNodeAddr(addr tcpip.Address) tcpip.Address { + addrBytes := addr.As16() + return tcpip.AddrFrom16([16]byte(append(solicitedNodeMulticastPrefix[:], addrBytes[len(addrBytes)-3:]...))) +} + +// IsSolicitedNodeAddr determines whether the address is a solicited-node +// multicast address. +func IsSolicitedNodeAddr(addr tcpip.Address) bool { + addrBytes := addr.As16() + return solicitedNodeMulticastPrefix == [13]byte(addrBytes[:len(addrBytes)-3]) +} + +// EthernetAdddressToModifiedEUI64IntoBuf populates buf with a modified EUI-64 +// from a 48-bit Ethernet/MAC address, as per RFC 4291 section 2.5.1. +// +// buf MUST be at least 8 bytes. +func EthernetAdddressToModifiedEUI64IntoBuf(linkAddr tcpip.LinkAddress, buf []byte) { + buf[0] = linkAddr[0] ^ 2 + buf[1] = linkAddr[1] + buf[2] = linkAddr[2] + buf[3] = 0xFF + buf[4] = 0xFE + buf[5] = linkAddr[3] + buf[6] = linkAddr[4] + buf[7] = linkAddr[5] +} + +// EthernetAddressToModifiedEUI64 computes a modified EUI-64 from a 48-bit +// Ethernet/MAC address, as per RFC 4291 section 2.5.1. +func EthernetAddressToModifiedEUI64(linkAddr tcpip.LinkAddress) [IIDSize]byte { + var buf [IIDSize]byte + EthernetAdddressToModifiedEUI64IntoBuf(linkAddr, buf[:]) + return buf +} + +// LinkLocalAddr computes the default IPv6 link-local address from a link-layer +// (MAC) address. +func LinkLocalAddr(linkAddr tcpip.LinkAddress) tcpip.Address { + // Convert a 48-bit MAC to a modified EUI-64 and then prepend the + // link-local header, FE80::. + // + // The conversion is very nearly: + // aa:bb:cc:dd:ee:ff => FE80::Aabb:ccFF:FEdd:eeff + // Note the capital A. The conversion aa->Aa involves a bit flip. + lladdrb := [IPv6AddressSize]byte{ + 0: 0xFE, + 1: 0x80, + } + EthernetAdddressToModifiedEUI64IntoBuf(linkAddr, lladdrb[IIDOffsetInIPv6Address:]) + return tcpip.AddrFrom16(lladdrb) +} + +// IsV6LinkLocalUnicastAddress returns true iff the provided address is an IPv6 +// link-local unicast address, as defined by RFC 4291 section 2.5.6. +func IsV6LinkLocalUnicastAddress(addr tcpip.Address) bool { + if addr.BitLen() != IPv6AddressSizeBits { + return false + } + addrBytes := addr.As16() + return addrBytes[0] == 0xfe && (addrBytes[1]&0xc0) == 0x80 +} + +// IsV6LoopbackAddress returns true iff the provided address is an IPv6 loopback +// address, as defined by RFC 4291 section 2.5.3. +func IsV6LoopbackAddress(addr tcpip.Address) bool { + return addr == IPv6Loopback +} + +// IsV6LinkLocalMulticastAddress returns true iff the provided address is an +// IPv6 link-local multicast address, as defined by RFC 4291 section 2.7. +func IsV6LinkLocalMulticastAddress(addr tcpip.Address) bool { + return IsV6MulticastAddress(addr) && V6MulticastScope(addr) == IPv6LinkLocalMulticastScope +} + +// AppendOpaqueInterfaceIdentifier appends a 64 bit opaque interface identifier +// (IID) to buf as outlined by RFC 7217 and returns the extended buffer. +// +// The opaque IID is generated from the cryptographic hash of the concatenation +// of the prefix, NIC's name, DAD counter (DAD retry counter) and the secret +// key. The secret key SHOULD be at least OpaqueIIDSecretKeyMinBytes bytes and +// MUST be generated to a pseudo-random number. See RFC 4086 for randomness +// requirements for security. +// +// If buf has enough capacity for the IID (IIDSize bytes), a new underlying +// array for the buffer will not be allocated. +func AppendOpaqueInterfaceIdentifier(buf []byte, prefix tcpip.Subnet, nicName string, dadCounter uint8, secretKey []byte) []byte { + // As per RFC 7217 section 5, the opaque identifier can be generated as a + // cryptographic hash of the concatenation of each of the function parameters. + // Note, we omit the optional Network_ID field. + h := sha256.New() + // h.Write never returns an error. + prefixID := prefix.ID() + h.Write([]byte(prefixID.AsSlice()[:IIDOffsetInIPv6Address])) + h.Write([]byte(nicName)) + h.Write([]byte{dadCounter}) + h.Write(secretKey) + + var sumBuf [sha256.Size]byte + sum := h.Sum(sumBuf[:0]) + + return append(buf, sum[:IIDSize]...) +} + +// LinkLocalAddrWithOpaqueIID computes the default IPv6 link-local address with +// an opaque IID. +func LinkLocalAddrWithOpaqueIID(nicName string, dadCounter uint8, secretKey []byte) tcpip.Address { + lladdrb := [IPv6AddressSize]byte{ + 0: 0xFE, + 1: 0x80, + } + + return tcpip.AddrFrom16([16]byte(AppendOpaqueInterfaceIdentifier(lladdrb[:IIDOffsetInIPv6Address], IPv6LinkLocalPrefix.Subnet(), nicName, dadCounter, secretKey))) +} + +// IPv6AddressScope is the scope of an IPv6 address. +type IPv6AddressScope int + +const ( + // LinkLocalScope indicates a link-local address. + LinkLocalScope IPv6AddressScope = iota + + // GlobalScope indicates a global address. + GlobalScope +) + +// ScopeForIPv6Address returns the scope for an IPv6 address. +func ScopeForIPv6Address(addr tcpip.Address) (IPv6AddressScope, tcpip.Error) { + if addr.BitLen() != IPv6AddressSizeBits { + return GlobalScope, &tcpip.ErrBadAddress{} + } + + switch { + case IsV6LinkLocalMulticastAddress(addr): + return LinkLocalScope, nil + + case IsV6LinkLocalUnicastAddress(addr): + return LinkLocalScope, nil + + default: + return GlobalScope, nil + } +} + +// InitialTempIID generates the initial temporary IID history value to generate +// temporary SLAAC addresses with. +// +// Panics if initialTempIIDHistory is not at least IIDSize bytes. +func InitialTempIID(initialTempIIDHistory []byte, seed []byte, nicID tcpip.NICID) { + h := sha256.New() + // h.Write never returns an error. + h.Write(seed) + var nicIDBuf [4]byte + binary.BigEndian.PutUint32(nicIDBuf[:], uint32(nicID)) + h.Write(nicIDBuf[:]) + + var sumBuf [sha256.Size]byte + sum := h.Sum(sumBuf[:0]) + + if n := copy(initialTempIIDHistory, sum[sha256.Size-IIDSize:]); n != IIDSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, IIDSize)) + } +} + +// GenerateTempIPv6SLAACAddr generates a temporary SLAAC IPv6 address for an +// associated stable/permanent SLAAC address. +// +// GenerateTempIPv6SLAACAddr will update the temporary IID history value to be +// used when generating a new temporary IID. +// +// Panics if tempIIDHistory is not at least IIDSize bytes. +func GenerateTempIPv6SLAACAddr(tempIIDHistory []byte, stableAddr tcpip.Address) tcpip.AddressWithPrefix { + addrBytes := stableAddr.As16() + h := sha256.New() + h.Write(tempIIDHistory) + h.Write(addrBytes[IIDOffsetInIPv6Address:]) + var sumBuf [sha256.Size]byte + sum := h.Sum(sumBuf[:0]) + + // The rightmost 64 bits of sum are saved for the next iteration. + if n := copy(tempIIDHistory, sum[sha256.Size-IIDSize:]); n != IIDSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, IIDSize)) + } + + // The leftmost 64 bits of sum is used as the IID. + if n := copy(addrBytes[IIDOffsetInIPv6Address:], sum); n != IIDSize { + panic(fmt.Sprintf("copied %d IID bytes, expected %d bytes", n, IIDSize)) + } + + return tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16(addrBytes), + PrefixLen: IIDOffsetInIPv6Address * 8, + } +} + +// IPv6MulticastScope is the scope of a multicast IPv6 address, as defined by +// RFC 7346 section 2. +type IPv6MulticastScope uint8 + +// The various values for IPv6 multicast scopes, as per RFC 7346 section 2: +// +// +------+--------------------------+-------------------------+ +// | scop | NAME | REFERENCE | +// +------+--------------------------+-------------------------+ +// | 0 | Reserved | [RFC4291], RFC 7346 | +// | 1 | Interface-Local scope | [RFC4291], RFC 7346 | +// | 2 | Link-Local scope | [RFC4291], RFC 7346 | +// | 3 | Realm-Local scope | [RFC4291], RFC 7346 | +// | 4 | Admin-Local scope | [RFC4291], RFC 7346 | +// | 5 | Site-Local scope | [RFC4291], RFC 7346 | +// | 6 | Unassigned | | +// | 7 | Unassigned | | +// | 8 | Organization-Local scope | [RFC4291], RFC 7346 | +// | 9 | Unassigned | | +// | A | Unassigned | | +// | B | Unassigned | | +// | C | Unassigned | | +// | D | Unassigned | | +// | E | Global scope | [RFC4291], RFC 7346 | +// | F | Reserved | [RFC4291], RFC 7346 | +// +------+--------------------------+-------------------------+ +const ( + IPv6Reserved0MulticastScope = IPv6MulticastScope(0x0) + IPv6InterfaceLocalMulticastScope = IPv6MulticastScope(0x1) + IPv6LinkLocalMulticastScope = IPv6MulticastScope(0x2) + IPv6RealmLocalMulticastScope = IPv6MulticastScope(0x3) + IPv6AdminLocalMulticastScope = IPv6MulticastScope(0x4) + IPv6SiteLocalMulticastScope = IPv6MulticastScope(0x5) + IPv6OrganizationLocalMulticastScope = IPv6MulticastScope(0x8) + IPv6GlobalMulticastScope = IPv6MulticastScope(0xE) + IPv6ReservedFMulticastScope = IPv6MulticastScope(0xF) +) + +// V6MulticastScope returns the scope of a multicast address. +func V6MulticastScope(addr tcpip.Address) IPv6MulticastScope { + addrBytes := addr.As16() + return IPv6MulticastScope(addrBytes[ipv6MulticastAddressScopeByteIdx] & ipv6MulticastAddressScopeMask) +} diff --git a/pkg/tcpip/header/ipv6_extension_headers.go b/pkg/tcpip/header/ipv6_extension_headers.go new file mode 100644 index 0000000..2d64c24 --- /dev/null +++ b/pkg/tcpip/header/ipv6_extension_headers.go @@ -0,0 +1,1010 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "math" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// IPv6ExtensionHeaderIdentifier is an IPv6 extension header identifier. +type IPv6ExtensionHeaderIdentifier uint8 + +const ( + // IPv6HopByHopOptionsExtHdrIdentifier is the header identifier of a Hop by + // Hop Options extension header, as per RFC 8200 section 4.3. + IPv6HopByHopOptionsExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 0 + + // IPv6RoutingExtHdrIdentifier is the header identifier of a Routing extension + // header, as per RFC 8200 section 4.4. + IPv6RoutingExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 43 + + // IPv6FragmentExtHdrIdentifier is the header identifier of a Fragment + // extension header, as per RFC 8200 section 4.5. + IPv6FragmentExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 44 + + // IPv6DestinationOptionsExtHdrIdentifier is the header identifier of a + // Destination Options extension header, as per RFC 8200 section 4.6. + IPv6DestinationOptionsExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 60 + + // IPv6NoNextHeaderIdentifier is the header identifier used to signify the end + // of an IPv6 payload, as per RFC 8200 section 4.7. + IPv6NoNextHeaderIdentifier IPv6ExtensionHeaderIdentifier = 59 + + // IPv6ExperimentExtHdrIdentifier is the header identifier of an Experiment + // extension header, as per RFC 4727 section 3.3. + IPv6ExperimentExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 253 + + // IPv6UnknownExtHdrIdentifier is reserved by IANA. + // https://www.iana.org/assignments/ipv6-parameters/ipv6-parameters.xhtml#extension-header + // "254 Use for experimentation and testing [RFC3692][RFC4727]" + IPv6UnknownExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 254 +) + +const ( + // ipv6UnknownExtHdrOptionActionMask is the mask of the action to take when + // a node encounters an unrecognized option. + ipv6UnknownExtHdrOptionActionMask = 192 + + // ipv6UnknownExtHdrOptionActionShift is the least significant bits to discard + // from the action value for an unrecognized option identifier. + ipv6UnknownExtHdrOptionActionShift = 6 + + // ipv6RoutingExtHdrSegmentsLeftIdx is the index to the Segments Left field + // within an IPv6RoutingExtHdr. + ipv6RoutingExtHdrSegmentsLeftIdx = 1 + + // IPv6FragmentExtHdrLength is the length of an IPv6 extension header, in + // bytes. + IPv6FragmentExtHdrLength = 8 + + // ipv6FragmentExtHdrFragmentOffsetOffset is the offset to the start of the + // Fragment Offset field within an IPv6FragmentExtHdr. + ipv6FragmentExtHdrFragmentOffsetOffset = 0 + + // ipv6FragmentExtHdrFragmentOffsetShift is the bit offset of the Fragment + // Offset field within an IPv6FragmentExtHdr. + ipv6FragmentExtHdrFragmentOffsetShift = 3 + + // ipv6FragmentExtHdrFlagsIdx is the index to the flags field within an + // IPv6FragmentExtHdr. + ipv6FragmentExtHdrFlagsIdx = 1 + + // ipv6FragmentExtHdrMFlagMask is the mask of the More (M) flag within the + // flags field of an IPv6FragmentExtHdr. + ipv6FragmentExtHdrMFlagMask = 1 + + // ipv6FragmentExtHdrIdentificationOffset is the offset to the Identification + // field within an IPv6FragmentExtHdr. + ipv6FragmentExtHdrIdentificationOffset = 2 + + // ipv6ExtHdrLenBytesPerUnit is the unit size of an extension header's length + // field. That is, given a Length field of 2, the extension header expects + // 16 bytes following the first 8 bytes (see ipv6ExtHdrLenBytesExcluded for + // details about the first 8 bytes' exclusion from the Length field). + ipv6ExtHdrLenBytesPerUnit = 8 + + // ipv6ExtHdrLenBytesExcluded is the number of bytes excluded from an + // extension header's Length field following the Length field. + // + // The Length field excludes the first 8 bytes, but the Next Header and Length + // field take up the first 2 of the 8 bytes so we expect (at minimum) 6 bytes + // after the Length field. + // + // This ensures that every extension header is at least 8 bytes. + ipv6ExtHdrLenBytesExcluded = 6 + + // IPv6FragmentExtHdrFragmentOffsetBytesPerUnit is the unit size of a Fragment + // extension header's Fragment Offset field. That is, given a Fragment Offset + // of 2, the extension header is indicating that the fragment's payload + // starts at the 16th byte in the reassembled packet. + IPv6FragmentExtHdrFragmentOffsetBytesPerUnit = 8 +) + +// padIPv6OptionsLength returns the total length for IPv6 options of length l +// considering the 8-octet alignment as stated in RFC 8200 Section 4.2. +func padIPv6OptionsLength(length int) int { + return (length + ipv6ExtHdrLenBytesPerUnit - 1) & ^(ipv6ExtHdrLenBytesPerUnit - 1) +} + +// padIPv6Option fills b with the appropriate padding options depending on its +// length. +func padIPv6Option(b []byte) { + switch len(b) { + case 0: // No padding needed. + case 1: // Pad with Pad1. + b[ipv6ExtHdrOptionTypeOffset] = uint8(ipv6Pad1ExtHdrOptionIdentifier) + default: // Pad with PadN. + s := b[ipv6ExtHdrOptionPayloadOffset:] + clear(s) + b[ipv6ExtHdrOptionTypeOffset] = uint8(ipv6PadNExtHdrOptionIdentifier) + b[ipv6ExtHdrOptionLengthOffset] = uint8(len(s)) + } +} + +// ipv6OptionsAlignmentPadding returns the number of padding bytes needed to +// serialize an option at headerOffset with alignment requirements +// [align]n + alignOffset. +func ipv6OptionsAlignmentPadding(headerOffset int, align int, alignOffset int) int { + padLen := headerOffset - alignOffset + return ((padLen + align - 1) & ^(align - 1)) - padLen +} + +// IPv6PayloadHeader is implemented by the various headers that can be found +// in an IPv6 payload. +// +// These headers include IPv6 extension headers or upper layer data. +type IPv6PayloadHeader interface { + isIPv6PayloadHeader() + + // Release frees all resources held by the header. + Release() +} + +// IPv6RawPayloadHeader the remainder of an IPv6 payload after an iterator +// encounters a Next Header field it does not recognize as an IPv6 extension +// header. The caller is responsible for releasing the underlying buffer after +// it's no longer needed. +type IPv6RawPayloadHeader struct { + Identifier IPv6ExtensionHeaderIdentifier + Buf buffer.Buffer +} + +// isIPv6PayloadHeader implements IPv6PayloadHeader.isIPv6PayloadHeader. +func (IPv6RawPayloadHeader) isIPv6PayloadHeader() {} + +// Release implements IPv6PayloadHeader.Release. +func (i IPv6RawPayloadHeader) Release() { + i.Buf.Release() +} + +// ipv6OptionsExtHdr is an IPv6 extension header that holds options. +type ipv6OptionsExtHdr struct { + buf *buffer.View +} + +// Release implements IPv6PayloadHeader.Release. +func (i ipv6OptionsExtHdr) Release() { + if i.buf != nil { + i.buf.Release() + } +} + +// Iter returns an iterator over the IPv6 extension header options held in b. +func (i ipv6OptionsExtHdr) Iter() IPv6OptionsExtHdrOptionsIterator { + it := IPv6OptionsExtHdrOptionsIterator{} + it.reader = i.buf + return it +} + +// IPv6OptionsExtHdrOptionsIterator is an iterator over IPv6 extension header +// options. +// +// Note, between when an IPv6OptionsExtHdrOptionsIterator is obtained and last +// used, no changes to the underlying buffer may happen. Doing so may cause +// undefined and unexpected behaviour. It is fine to obtain an +// IPv6OptionsExtHdrOptionsIterator, iterate over the first few options then +// modify the backing payload so long as the IPv6OptionsExtHdrOptionsIterator +// obtained before modification is no longer used. +type IPv6OptionsExtHdrOptionsIterator struct { + reader *buffer.View + + // optionOffset is the number of bytes from the first byte of the + // options field to the beginning of the current option. + optionOffset uint32 + + // nextOptionOffset is the offset of the next option. + nextOptionOffset uint32 +} + +// OptionOffset returns the number of bytes parsed while processing the +// option field of the current Extension Header. +func (i *IPv6OptionsExtHdrOptionsIterator) OptionOffset() uint32 { + return i.optionOffset +} + +// IPv6OptionUnknownAction is the action that must be taken if the processing +// IPv6 node does not recognize the option, as outlined in RFC 8200 section 4.2. +type IPv6OptionUnknownAction int + +const ( + // IPv6OptionUnknownActionSkip indicates that the unrecognized option must + // be skipped and the node should continue processing the header. + IPv6OptionUnknownActionSkip IPv6OptionUnknownAction = 0 + + // IPv6OptionUnknownActionDiscard indicates that the packet must be silently + // discarded. + IPv6OptionUnknownActionDiscard IPv6OptionUnknownAction = 1 + + // IPv6OptionUnknownActionDiscardSendICMP indicates that the packet must be + // discarded and the node must send an ICMP Parameter Problem, Code 2, message + // to the packet's source, regardless of whether or not the packet's + // Destination was a multicast address. + IPv6OptionUnknownActionDiscardSendICMP IPv6OptionUnknownAction = 2 + + // IPv6OptionUnknownActionDiscardSendICMPNoMulticastDest indicates that the + // packet must be discarded and the node must send an ICMP Parameter Problem, + // Code 2, message to the packet's source only if the packet's Destination was + // not a multicast address. + IPv6OptionUnknownActionDiscardSendICMPNoMulticastDest IPv6OptionUnknownAction = 3 +) + +// IPv6ExtHdrOption is implemented by the various IPv6 extension header options. +type IPv6ExtHdrOption interface { + // UnknownAction returns the action to take in response to an unrecognized + // option. + UnknownAction() IPv6OptionUnknownAction + + // isIPv6ExtHdrOption is used to "lock" this interface so it is not + // implemented by other packages. + isIPv6ExtHdrOption() +} + +// IPv6ExtHdrOptionIdentifier is an IPv6 extension header option identifier. +type IPv6ExtHdrOptionIdentifier uint8 + +const ( + // ipv6Pad1ExtHdrOptionIdentifier is the identifier for a padding option that + // provides 1 byte padding, as outlined in RFC 8200 section 4.2. + ipv6Pad1ExtHdrOptionIdentifier IPv6ExtHdrOptionIdentifier = 0 + + // ipv6PadNExtHdrOptionIdentifier is the identifier for a padding option that + // provides variable length byte padding, as outlined in RFC 8200 section 4.2. + ipv6PadNExtHdrOptionIdentifier IPv6ExtHdrOptionIdentifier = 1 + + // ipv6RouterAlertHopByHopOptionIdentifier is the identifier for the Router + // Alert Hop by Hop option as defined in RFC 2711 section 2.1. + ipv6RouterAlertHopByHopOptionIdentifier IPv6ExtHdrOptionIdentifier = 5 + + // ipv6ExtHdrOptionTypeOffset is the option type offset in an extension header + // option as defined in RFC 8200 section 4.2. + ipv6ExtHdrOptionTypeOffset = 0 + + // ipv6ExtHdrOptionLengthOffset is the option length offset in an extension + // header option as defined in RFC 8200 section 4.2. + ipv6ExtHdrOptionLengthOffset = 1 + + // ipv6ExtHdrOptionPayloadOffset is the option payload offset in an extension + // header option as defined in RFC 8200 section 4.2. + ipv6ExtHdrOptionPayloadOffset = 2 +) + +// ipv6UnknownActionFromIdentifier maps an extension header option's +// identifier's high bits to the action to take when the identifier is unknown. +func ipv6UnknownActionFromIdentifier(id IPv6ExtHdrOptionIdentifier) IPv6OptionUnknownAction { + return IPv6OptionUnknownAction((id & ipv6UnknownExtHdrOptionActionMask) >> ipv6UnknownExtHdrOptionActionShift) +} + +// ErrMalformedIPv6ExtHdrOption indicates that an IPv6 extension header option +// is malformed. +var ErrMalformedIPv6ExtHdrOption = errors.New("malformed IPv6 extension header option") + +// IPv6UnknownExtHdrOption holds the identifier and data for an IPv6 extension +// header option that is unknown by the parsing utilities. +type IPv6UnknownExtHdrOption struct { + Identifier IPv6ExtHdrOptionIdentifier + Data *buffer.View +} + +// UnknownAction implements IPv6OptionUnknownAction.UnknownAction. +func (o *IPv6UnknownExtHdrOption) UnknownAction() IPv6OptionUnknownAction { + return ipv6UnknownActionFromIdentifier(o.Identifier) +} + +// isIPv6ExtHdrOption implements IPv6ExtHdrOption.isIPv6ExtHdrOption. +func (*IPv6UnknownExtHdrOption) isIPv6ExtHdrOption() {} + +// Next returns the next option in the options data. +// +// If the next item is not a known extension header option, +// IPv6UnknownExtHdrOption will be returned with the option identifier and data. +// +// The return is of the format (option, done, error). done will be true when +// Next is unable to return anything because the iterator has reached the end of +// the options data, or an error occurred. +func (i *IPv6OptionsExtHdrOptionsIterator) Next() (IPv6ExtHdrOption, bool, error) { + for { + i.optionOffset = i.nextOptionOffset + temp, err := i.reader.ReadByte() + if err != nil { + // If we can't read the first byte of a new option, then we know the + // options buffer has been exhausted and we are done iterating. + return nil, true, nil + } + id := IPv6ExtHdrOptionIdentifier(temp) + + // If the option identifier indicates the option is a Pad1 option, then we + // know the option does not have Length and Data fields. End processing of + // the Pad1 option and continue processing the buffer as a new option. + if id == ipv6Pad1ExtHdrOptionIdentifier { + i.nextOptionOffset = i.optionOffset + 1 + continue + } + + length, err := i.reader.ReadByte() + if err != nil { + if err != io.EOF { + // ReadByte should only ever return nil or io.EOF. + panic(fmt.Sprintf("unexpected error when reading the option's Length field for option with id = %d: %s", id, err)) + } + + // We use io.ErrUnexpectedEOF as exhausting the buffer is unexpected once + // we start parsing an option; we expect the reader to contain enough + // bytes for the whole option. + return nil, true, fmt.Errorf("error when reading the option's Length field for option with id = %d: %w", id, io.ErrUnexpectedEOF) + } + + // Do we have enough bytes in the reader for the next option? + if n := i.reader.Size(); n < int(length) { + // Consume the remaining buffer. + i.reader.TrimFront(i.reader.Size()) + + // We return the same error as if we failed to read a non-padding option + // so consumers of this iterator don't need to differentiate between + // padding and non-padding options. + return nil, true, fmt.Errorf("read %d out of %d option data bytes for option with id = %d: %w", n, length, id, io.ErrUnexpectedEOF) + } + + i.nextOptionOffset = i.optionOffset + uint32(length) + 1 /* option ID */ + 1 /* length byte */ + + switch id { + case ipv6PadNExtHdrOptionIdentifier: + // Special-case the variable length padding option to avoid a copy. + i.reader.TrimFront(int(length)) + continue + case ipv6RouterAlertHopByHopOptionIdentifier: + var routerAlertValue [ipv6RouterAlertPayloadLength]byte + if n, err := io.ReadFull(i.reader, routerAlertValue[:]); err != nil { + switch err { + case io.EOF, io.ErrUnexpectedEOF: + return nil, true, fmt.Errorf("got invalid length (%d) for router alert option (want = %d): %w", length, ipv6RouterAlertPayloadLength, ErrMalformedIPv6ExtHdrOption) + default: + return nil, true, fmt.Errorf("read %d out of %d option data bytes for router alert option: %w", n, ipv6RouterAlertPayloadLength, err) + } + } else if n != int(length) { + return nil, true, fmt.Errorf("got invalid length (%d) for router alert option (want = %d): %w", length, ipv6RouterAlertPayloadLength, ErrMalformedIPv6ExtHdrOption) + } + return &IPv6RouterAlertOption{Value: IPv6RouterAlertValue(binary.BigEndian.Uint16(routerAlertValue[:]))}, false, nil + default: + bytes := buffer.NewView(int(length)) + if n, err := io.CopyN(bytes, i.reader, int64(length)); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + + return nil, true, fmt.Errorf("read %d out of %d option data bytes for option with id = %d: %w", n, length, id, err) + } + return &IPv6UnknownExtHdrOption{Identifier: id, Data: bytes}, false, nil + } + } +} + +// IPv6HopByHopOptionsExtHdr is a buffer holding the Hop By Hop Options +// extension header. +type IPv6HopByHopOptionsExtHdr struct { + ipv6OptionsExtHdr +} + +// isIPv6PayloadHeader implements IPv6PayloadHeader.isIPv6PayloadHeader. +func (IPv6HopByHopOptionsExtHdr) isIPv6PayloadHeader() {} + +// IPv6DestinationOptionsExtHdr is a buffer holding the Destination Options +// extension header. +type IPv6DestinationOptionsExtHdr struct { + ipv6OptionsExtHdr +} + +// isIPv6PayloadHeader implements IPv6PayloadHeader.isIPv6PayloadHeader. +func (IPv6DestinationOptionsExtHdr) isIPv6PayloadHeader() {} + +// IPv6ExperimentExtHdr is a buffer holding the Experiment extension header. +type IPv6ExperimentExtHdr struct { + Value uint16 +} + +// Release implements IPv6PayloadHeader.Release. +func (IPv6ExperimentExtHdr) Release() {} + +// isIPv6PayloadHeader implements IPv6PayloadHeader.isIPv6PayloadHeader. +func (IPv6ExperimentExtHdr) isIPv6PayloadHeader() {} + +// IPv6RoutingExtHdr is a buffer holding the Routing extension header specific +// data as outlined in RFC 8200 section 4.4. +type IPv6RoutingExtHdr struct { + Buf *buffer.View +} + +// isIPv6PayloadHeader implements IPv6PayloadHeader.isIPv6PayloadHeader. +func (IPv6RoutingExtHdr) isIPv6PayloadHeader() {} + +// Release implements IPv6PayloadHeader.Release. +func (b IPv6RoutingExtHdr) Release() { + b.Buf.Release() +} + +// SegmentsLeft returns the Segments Left field. +func (b IPv6RoutingExtHdr) SegmentsLeft() uint8 { + return b.Buf.AsSlice()[ipv6RoutingExtHdrSegmentsLeftIdx] +} + +// IPv6FragmentExtHdr is a buffer holding the Fragment extension header specific +// data as outlined in RFC 8200 section 4.5. +// +// Note, the buffer does not include the Next Header and Reserved fields. +type IPv6FragmentExtHdr [6]byte + +// isIPv6PayloadHeader implements IPv6PayloadHeader.isIPv6PayloadHeader. +func (IPv6FragmentExtHdr) isIPv6PayloadHeader() {} + +// Release implements IPv6PayloadHeader.Release. +func (IPv6FragmentExtHdr) Release() {} + +// FragmentOffset returns the Fragment Offset field. +// +// This value indicates where the buffer following the Fragment extension header +// starts in the target (reassembled) packet. +func (b IPv6FragmentExtHdr) FragmentOffset() uint16 { + return binary.BigEndian.Uint16(b[ipv6FragmentExtHdrFragmentOffsetOffset:]) >> ipv6FragmentExtHdrFragmentOffsetShift +} + +// More returns the More (M) flag. +// +// This indicates whether any fragments are expected to succeed b. +func (b IPv6FragmentExtHdr) More() bool { + return b[ipv6FragmentExtHdrFlagsIdx]&ipv6FragmentExtHdrMFlagMask != 0 +} + +// ID returns the Identification field. +// +// This value is used to uniquely identify the packet, between a +// source and destination. +func (b IPv6FragmentExtHdr) ID() uint32 { + return binary.BigEndian.Uint32(b[ipv6FragmentExtHdrIdentificationOffset:]) +} + +// IsAtomic returns whether the fragment header indicates an atomic fragment. An +// atomic fragment is a fragment that contains all the data required to +// reassemble a full packet. +func (b IPv6FragmentExtHdr) IsAtomic() bool { + return !b.More() && b.FragmentOffset() == 0 +} + +// IPv6PayloadIterator is an iterator over the contents of an IPv6 payload. +// +// The IPv6 payload may contain IPv6 extension headers before any upper layer +// data. +// +// Note, between when an IPv6PayloadIterator is obtained and last used, no +// changes to the payload may happen. Doing so may cause undefined and +// unexpected behaviour. It is fine to obtain an IPv6PayloadIterator, iterate +// over the first few headers then modify the backing payload so long as the +// IPv6PayloadIterator obtained before modification is no longer used. +type IPv6PayloadIterator struct { + // The identifier of the next header to parse. + nextHdrIdentifier IPv6ExtensionHeaderIdentifier + + payload buffer.Buffer + + // Indicates to the iterator that it should return the remaining payload as a + // raw payload on the next call to Next. + forceRaw bool + + // headerOffset is the offset of the beginning of the current extension + // header starting from the beginning of the fixed header. + headerOffset uint32 + + // parseOffset is the byte offset into the current extension header of the + // field we are currently examining. It can be added to the header offset + // if the absolute offset within the packet is required. + parseOffset uint32 + + // nextOffset is the offset of the next header. + nextOffset uint32 +} + +// HeaderOffset returns the offset to the start of the extension +// header most recently processed. +func (i IPv6PayloadIterator) HeaderOffset() uint32 { + return i.headerOffset +} + +// ParseOffset returns the number of bytes successfully parsed. +func (i IPv6PayloadIterator) ParseOffset() uint32 { + return i.headerOffset + i.parseOffset +} + +// MakeIPv6PayloadIterator returns an iterator over the IPv6 payload containing +// extension headers, or a raw payload if the payload cannot be parsed. The +// iterator takes ownership of the payload. +func MakeIPv6PayloadIterator(nextHdrIdentifier IPv6ExtensionHeaderIdentifier, payload buffer.Buffer) IPv6PayloadIterator { + return IPv6PayloadIterator{ + nextHdrIdentifier: nextHdrIdentifier, + payload: payload, + nextOffset: IPv6FixedHeaderSize, + } +} + +// Release frees the resources owned by the iterator. +func (i *IPv6PayloadIterator) Release() { + i.payload.Release() +} + +// AsRawHeader returns the remaining payload of i as a raw header and +// optionally consumes the iterator. +// +// If consume is true, calls to Next after calling AsRawHeader on i will +// indicate that the iterator is done. The returned header takes ownership of +// its payload. +func (i *IPv6PayloadIterator) AsRawHeader(consume bool) IPv6RawPayloadHeader { + identifier := i.nextHdrIdentifier + + var buf buffer.Buffer + if consume { + // Since we consume the iterator, we return the payload as is. + buf = i.payload + + // Mark i as done, but keep track of where we were for error reporting. + *i = IPv6PayloadIterator{ + nextHdrIdentifier: IPv6NoNextHeaderIdentifier, + headerOffset: i.headerOffset, + nextOffset: i.nextOffset, + } + } else { + buf = i.payload.Clone() + } + + return IPv6RawPayloadHeader{Identifier: identifier, Buf: buf} +} + +// Next returns the next item in the payload. +// +// If the next item is not a known IPv6 extension header, IPv6RawPayloadHeader +// will be returned with the remaining bytes and next header identifier. +// +// The return is of the format (header, done, error). done will be true when +// Next is unable to return anything because the iterator has reached the end of +// the payload, or an error occurred. +func (i *IPv6PayloadIterator) Next() (IPv6PayloadHeader, bool, error) { + i.headerOffset = i.nextOffset + i.parseOffset = 0 + // We could be forced to return i as a raw header when the previous header was + // a fragment extension header as the data following the fragment extension + // header may not be complete. + if i.forceRaw { + return i.AsRawHeader(true /* consume */), false, nil + } + + // Is the header we are parsing a known extension header? + switch i.nextHdrIdentifier { + case IPv6HopByHopOptionsExtHdrIdentifier: + nextHdrIdentifier, view, err := i.nextHeaderData(false /* ignoreLength */, nil) + if err != nil { + return nil, true, err + } + + i.nextHdrIdentifier = nextHdrIdentifier + return IPv6HopByHopOptionsExtHdr{ipv6OptionsExtHdr{view}}, false, nil + case IPv6RoutingExtHdrIdentifier: + nextHdrIdentifier, view, err := i.nextHeaderData(false /* ignoreLength */, nil) + if err != nil { + return nil, true, err + } + + i.nextHdrIdentifier = nextHdrIdentifier + return IPv6RoutingExtHdr{view}, false, nil + case IPv6FragmentExtHdrIdentifier: + var data [6]byte + // We ignore the returned bytes because we know the fragment extension + // header specific data will fit in data. + nextHdrIdentifier, _, err := i.nextHeaderData(true /* ignoreLength */, data[:]) + if err != nil { + return nil, true, err + } + + fragmentExtHdr := IPv6FragmentExtHdr(data) + + // If the packet is not the first fragment, do not attempt to parse anything + // after the fragment extension header as the payload following the fragment + // extension header should not contain any headers; the first fragment must + // hold all the headers up to and including any upper layer headers, as per + // RFC 8200 section 4.5. + if fragmentExtHdr.FragmentOffset() != 0 { + i.forceRaw = true + } + + i.nextHdrIdentifier = nextHdrIdentifier + return fragmentExtHdr, false, nil + case IPv6DestinationOptionsExtHdrIdentifier: + nextHdrIdentifier, view, err := i.nextHeaderData(false /* ignoreLength */, nil) + if err != nil { + return nil, true, err + } + + i.nextHdrIdentifier = nextHdrIdentifier + return IPv6DestinationOptionsExtHdr{ipv6OptionsExtHdr{view}}, false, nil + case IPv6ExperimentExtHdrIdentifier: + var data [IPv6ExperimentHdrLength - ipv6ExperimentHdrValueOffset]byte + nextHdrIdentifier, _, err := i.nextHeaderData(true /* ignoreLength */, data[:]) + if err != nil { + return nil, true, err + } + i.nextHdrIdentifier = nextHdrIdentifier + hdr := IPv6ExperimentExtHdr{ + Value: binary.BigEndian.Uint16(data[:ipv6ExperimentHdrTagLength]), + } + return hdr, false, nil + case IPv6NoNextHeaderIdentifier: + // This indicates the end of the IPv6 payload. + return nil, true, nil + + default: + // The header we are parsing is not a known extension header. Return the + // raw payload. + return i.AsRawHeader(true /* consume */), false, nil + } +} + +// NextHeaderIdentifier returns the identifier of the header next returned by +// it.Next(). +func (i *IPv6PayloadIterator) NextHeaderIdentifier() IPv6ExtensionHeaderIdentifier { + return i.nextHdrIdentifier +} + +// nextHeaderData returns the extension header's Next Header field and raw data. +// +// ignoreLength indicates that the extension header being parsed should ignore +// the Length field as it is reserved. This is for the Fragment and Experiment +// extension headers. +// +// If bytes is not nil, extension header specific data will be read into bytes +// if it has enough capacity. If bytes is provided but does not have enough +// capacity for the data, nextHeaderData will panic. +func (i *IPv6PayloadIterator) nextHeaderData(ignoreLength bool, bytes []byte) (IPv6ExtensionHeaderIdentifier, *buffer.View, error) { + // We ignore the number of bytes read because we know we will only ever read + // at max 1 bytes since rune has a length of 1. If we read 0 bytes, the Read + // would return io.EOF to indicate that io.Reader has reached the end of the + // payload. + rdr := i.payload.AsBufferReader() + nextHdrIdentifier, err := rdr.ReadByte() + if err != nil { + return 0, nil, fmt.Errorf("error when reading the Next Header field for extension header with id = %d: %w", i.nextHdrIdentifier, err) + } + i.parseOffset++ + + var length uint8 + length, err = rdr.ReadByte() + if err != nil { + if ignoreLength { + return 0, nil, fmt.Errorf("error when reading the Length field for extension header with id = %d: %w", i.nextHdrIdentifier, err) + } + + return 0, nil, fmt.Errorf("error when reading the Reserved field for extension header with id = %d: %w", i.nextHdrIdentifier, err) + } + if ignoreLength { + length = 0 + } + + // Make parseOffset point to the first byte of the Extension Header + // specific data. + i.parseOffset++ + + // length is in 8 byte chunks but doesn't include the first one. + // See RFC 8200 for each header type, sections 4.3-4.6 and the requirement + // in section 4.8 for new extension headers at the top of page 24. + // [ Hdr Ext Len ] ... Length of the Destination Options header in 8-octet + // units, not including the first 8 octets. + i.nextOffset += uint32((length + 1) * ipv6ExtHdrLenBytesPerUnit) + + bytesLen := int(length)*ipv6ExtHdrLenBytesPerUnit + ipv6ExtHdrLenBytesExcluded + if ignoreLength { + if n := len(bytes); n < bytesLen { + panic(fmt.Sprintf("bytes only has space for %d bytes but need space for %d bytes (length = %d) for extension header with id = %d", n, bytesLen, length, i.nextHdrIdentifier)) + } + if n, err := io.ReadFull(&rdr, bytes); err != nil { + return 0, nil, fmt.Errorf("read %d out of %d extension header data bytes (length = %d) for header with id = %d: %w", n, bytesLen, length, i.nextHdrIdentifier, err) + } + return IPv6ExtensionHeaderIdentifier(nextHdrIdentifier), nil, nil + } + v := buffer.NewView(bytesLen) + if n, err := io.CopyN(v, &rdr, int64(bytesLen)); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + v.Release() + return 0, nil, fmt.Errorf("read %d out of %d extension header data bytes (length = %d) for header with id = %d: %w", n, bytesLen, length, i.nextHdrIdentifier, err) + } + return IPv6ExtensionHeaderIdentifier(nextHdrIdentifier), v, nil +} + +// IPv6SerializableExtHdr provides serialization for IPv6 extension +// headers. +type IPv6SerializableExtHdr interface { + // identifier returns the assigned IPv6 header identifier for this extension + // header. + identifier() IPv6ExtensionHeaderIdentifier + + // length returns the total serialized length in bytes of this extension + // header, including the common next header and length fields. + length() int + + // serializeInto serializes the receiver into the provided byte + // buffer and with the provided nextHeader value. + // + // Note, the caller MUST provide a byte buffer with size of at least + // length. Implementers of this function may assume that the byte buffer + // is of sufficient size. serializeInto MAY panic if the provided byte + // buffer is not of sufficient size. + // + // serializeInto returns the number of bytes that was used to serialize the + // receiver. Implementers must only use the number of bytes required to + // serialize the receiver. Callers MAY provide a larger buffer than required + // to serialize into. + serializeInto(nextHeader uint8, b []byte) int +} + +// ipv6RouterAlertPayloadLength is the length of the Router Alert payload +// as defined in RFC 4727 section 3.3. +const ( + IPv6ExperimentHdrLength = 8 + ipv6ExperimentNextHeaderOffset = 0 + ipv6ExperimentLengthOffset = 1 + ipv6ExperimentHdrValueOffset = 2 + ipv6ExperimentHdrTagLength = 2 +) + +var _ IPv6SerializableExtHdr = (*IPv6ExperimentExtHdr)(nil) + +// identifier implements IPv6SerializableExtHdr. +func (h IPv6ExperimentExtHdr) identifier() IPv6ExtensionHeaderIdentifier { + return IPv6ExperimentExtHdrIdentifier +} + +// length implements IPv6SerializableExtHdr. +func (h IPv6ExperimentExtHdr) length() int { + return IPv6ExperimentHdrLength +} + +// serializeInto implements IPv6SerializableExtHdr. +func (h IPv6ExperimentExtHdr) serializeInto(nextHeader uint8, b []byte) int { + b[ipv6ExperimentNextHeaderOffset] = nextHeader + b[ipv6ExperimentLengthOffset] = (IPv6ExperimentHdrLength / ipv6ExtHdrLenBytesPerUnit) - 1 + binary.BigEndian.PutUint16(b[ipv6ExperimentHdrValueOffset:][:ipv6ExperimentHdrTagLength], uint16(h.Value)) + return IPv6ExperimentHdrLength +} + +var _ IPv6SerializableExtHdr = (*IPv6SerializableHopByHopExtHdr)(nil) + +// IPv6SerializableHopByHopExtHdr implements serialization of the Hop by Hop +// options extension header. +type IPv6SerializableHopByHopExtHdr []IPv6SerializableHopByHopOption + +const ( + // ipv6HopByHopExtHdrNextHeaderOffset is the offset of the next header field + // in a hop by hop extension header as defined in RFC 8200 section 4.3. + ipv6HopByHopExtHdrNextHeaderOffset = 0 + + // ipv6HopByHopExtHdrLengthOffset is the offset of the length field in a hop + // by hop extension header as defined in RFC 8200 section 4.3. + ipv6HopByHopExtHdrLengthOffset = 1 + + // ipv6HopByHopExtHdrPayloadOffset is the offset of the options in a hop by + // hop extension header as defined in RFC 8200 section 4.3. + ipv6HopByHopExtHdrOptionsOffset = 2 + + // ipv6HopByHopExtHdrUnaccountedLenWords is the implicit number of 8-octet + // words in a hop by hop extension header's length field, as stated in RFC + // 8200 section 4.3: + // Length of the Hop-by-Hop Options header in 8-octet units, + // not including the first 8 octets. + ipv6HopByHopExtHdrUnaccountedLenWords = 1 +) + +// identifier implements IPv6SerializableExtHdr. +func (IPv6SerializableHopByHopExtHdr) identifier() IPv6ExtensionHeaderIdentifier { + return IPv6HopByHopOptionsExtHdrIdentifier +} + +// length implements IPv6SerializableExtHdr. +func (h IPv6SerializableHopByHopExtHdr) length() int { + var total int + for _, opt := range h { + align, alignOffset := opt.alignment() + total += ipv6OptionsAlignmentPadding(total, align, alignOffset) + total += ipv6ExtHdrOptionPayloadOffset + int(opt.length()) + } + // Account for next header and total length fields and add padding. + return padIPv6OptionsLength(ipv6HopByHopExtHdrOptionsOffset + total) +} + +// serializeInto implements IPv6SerializableExtHdr. +func (h IPv6SerializableHopByHopExtHdr) serializeInto(nextHeader uint8, b []byte) int { + optBuffer := b[ipv6HopByHopExtHdrOptionsOffset:] + totalLength := ipv6HopByHopExtHdrOptionsOffset + for _, opt := range h { + // Calculate alignment requirements and pad buffer if necessary. + align, alignOffset := opt.alignment() + padLen := ipv6OptionsAlignmentPadding(totalLength, align, alignOffset) + if padLen != 0 { + padIPv6Option(optBuffer[:padLen]) + totalLength += padLen + optBuffer = optBuffer[padLen:] + } + + l := opt.serializeInto(optBuffer[ipv6ExtHdrOptionPayloadOffset:]) + optBuffer[ipv6ExtHdrOptionTypeOffset] = uint8(opt.identifier()) + optBuffer[ipv6ExtHdrOptionLengthOffset] = l + l += ipv6ExtHdrOptionPayloadOffset + totalLength += int(l) + optBuffer = optBuffer[l:] + } + padded := padIPv6OptionsLength(totalLength) + if padded != totalLength { + padIPv6Option(optBuffer[:padded-totalLength]) + totalLength = padded + } + wordsLen := totalLength/ipv6ExtHdrLenBytesPerUnit - ipv6HopByHopExtHdrUnaccountedLenWords + if wordsLen > math.MaxUint8 { + panic(fmt.Sprintf("IPv6 hop by hop options too large: %d+1 64-bit words", wordsLen)) + } + b[ipv6HopByHopExtHdrNextHeaderOffset] = nextHeader + b[ipv6HopByHopExtHdrLengthOffset] = uint8(wordsLen) + return totalLength +} + +// IPv6SerializableHopByHopOption provides serialization for hop by hop options. +type IPv6SerializableHopByHopOption interface { + // identifier returns the option identifier of this Hop by Hop option. + identifier() IPv6ExtHdrOptionIdentifier + + // length returns the *payload* size of the option (not considering the type + // and length fields). + length() uint8 + + // alignment returns the alignment requirements from this option. + // + // Alignment requirements take the form [align]n + offset as specified in + // RFC 8200 section 4.2. The alignment requirement is on the offset between + // the option type byte and the start of the hop by hop header. + // + // align must be a power of 2. + alignment() (align int, offset int) + + // serializeInto serializes the receiver into the provided byte + // buffer. + // + // Note, the caller MUST provide a byte buffer with size of at least + // length. Implementers of this function may assume that the byte buffer + // is of sufficient size. serializeInto MAY panic if the provided byte + // buffer is not of sufficient size. + // + // serializeInto will return the number of bytes that was used to + // serialize the receiver. Implementers must only use the number of + // bytes required to serialize the receiver. Callers MAY provide a + // larger buffer than required to serialize into. + serializeInto([]byte) uint8 +} + +var _ IPv6SerializableHopByHopOption = (*IPv6RouterAlertOption)(nil) + +// IPv6RouterAlertOption is the IPv6 Router alert Hop by Hop option defined in +// RFC 2711 section 2.1. +type IPv6RouterAlertOption struct { + Value IPv6RouterAlertValue +} + +// IPv6RouterAlertValue is the payload of an IPv6 Router Alert option. +type IPv6RouterAlertValue uint16 + +const ( + // IPv6RouterAlertMLD indicates a datagram containing a Multicast Listener + // Discovery message as defined in RFC 2711 section 2.1. + IPv6RouterAlertMLD IPv6RouterAlertValue = 0 + // IPv6RouterAlertRSVP indicates a datagram containing an RSVP message as + // defined in RFC 2711 section 2.1. + IPv6RouterAlertRSVP IPv6RouterAlertValue = 1 + // IPv6RouterAlertActiveNetworks indicates a datagram containing an Active + // Networks message as defined in RFC 2711 section 2.1. + IPv6RouterAlertActiveNetworks IPv6RouterAlertValue = 2 + + // ipv6RouterAlertPayloadLength is the length of the Router Alert payload + // as defined in RFC 2711. + ipv6RouterAlertPayloadLength = 2 + + // ipv6RouterAlertAlignmentRequirement is the alignment requirement for the + // Router Alert option defined as 2n+0 in RFC 2711. + ipv6RouterAlertAlignmentRequirement = 2 + + // ipv6RouterAlertAlignmentOffsetRequirement is the alignment offset + // requirement for the Router Alert option defined as 2n+0 in RFC 2711 section + // 2.1. + ipv6RouterAlertAlignmentOffsetRequirement = 0 +) + +// UnknownAction implements IPv6ExtHdrOption. +func (*IPv6RouterAlertOption) UnknownAction() IPv6OptionUnknownAction { + return ipv6UnknownActionFromIdentifier(ipv6RouterAlertHopByHopOptionIdentifier) +} + +// isIPv6ExtHdrOption implements IPv6ExtHdrOption. +func (*IPv6RouterAlertOption) isIPv6ExtHdrOption() {} + +// identifier implements IPv6SerializableHopByHopOption. +func (*IPv6RouterAlertOption) identifier() IPv6ExtHdrOptionIdentifier { + return ipv6RouterAlertHopByHopOptionIdentifier +} + +// length implements IPv6SerializableHopByHopOption. +func (*IPv6RouterAlertOption) length() uint8 { + return ipv6RouterAlertPayloadLength +} + +// alignment implements IPv6SerializableHopByHopOption. +func (*IPv6RouterAlertOption) alignment() (int, int) { + // From RFC 2711 section 2.1: + // Alignment requirement: 2n+0. + return ipv6RouterAlertAlignmentRequirement, ipv6RouterAlertAlignmentOffsetRequirement +} + +// serializeInto implements IPv6SerializableHopByHopOption. +func (o *IPv6RouterAlertOption) serializeInto(b []byte) uint8 { + binary.BigEndian.PutUint16(b, uint16(o.Value)) + return ipv6RouterAlertPayloadLength +} + +// IPv6ExtHdrSerializer provides serialization of IPv6 extension headers. +type IPv6ExtHdrSerializer []IPv6SerializableExtHdr + +// Serialize serializes the provided list of IPv6 extension headers into b. +// +// Note, b must be of sufficient size to hold all the headers in s. See +// IPv6ExtHdrSerializer.Length for details on the getting the total size of a +// serialized IPv6ExtHdrSerializer. +// +// Serialize may panic if b is not of sufficient size to hold all the options +// in s. +// +// Serialize takes the transportProtocol value to be used as the last extension +// header's Next Header value and returns the header identifier of the first +// serialized extension header and the total serialized length. +func (s IPv6ExtHdrSerializer) Serialize(transportProtocol tcpip.TransportProtocolNumber, b []byte) (uint8, int) { + nextHeader := uint8(transportProtocol) + if len(s) == 0 { + return nextHeader, 0 + } + var totalLength int + for i, h := range s[:len(s)-1] { + length := h.serializeInto(uint8(s[i+1].identifier()), b) + b = b[length:] + totalLength += length + } + totalLength += s[len(s)-1].serializeInto(nextHeader, b) + return uint8(s[0].identifier()), totalLength +} + +// Length returns the total number of bytes required to serialize the extension +// headers. +func (s IPv6ExtHdrSerializer) Length() int { + var totalLength int + for _, h := range s { + totalLength += h.length() + } + return totalLength +} diff --git a/pkg/tcpip/header/ipv6_fragment.go b/pkg/tcpip/header/ipv6_fragment.go new file mode 100644 index 0000000..5b71ae8 --- /dev/null +++ b/pkg/tcpip/header/ipv6_fragment.go @@ -0,0 +1,158 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + nextHdrFrag = 0 + fragOff = 2 + more = 3 + idV6 = 4 +) + +var _ IPv6SerializableExtHdr = (*IPv6SerializableFragmentExtHdr)(nil) + +// IPv6SerializableFragmentExtHdr is used to serialize an IPv6 fragment +// extension header as defined in RFC 8200 section 4.5. +type IPv6SerializableFragmentExtHdr struct { + // FragmentOffset is the "fragment offset" field of an IPv6 fragment. + FragmentOffset uint16 + + // M is the "more" field of an IPv6 fragment. + M bool + + // Identification is the "identification" field of an IPv6 fragment. + Identification uint32 +} + +// identifier implements IPv6SerializableFragmentExtHdr. +func (h *IPv6SerializableFragmentExtHdr) identifier() IPv6ExtensionHeaderIdentifier { + return IPv6FragmentHeader +} + +// length implements IPv6SerializableFragmentExtHdr. +func (h *IPv6SerializableFragmentExtHdr) length() int { + return IPv6FragmentHeaderSize +} + +// serializeInto implements IPv6SerializableFragmentExtHdr. +func (h *IPv6SerializableFragmentExtHdr) serializeInto(nextHeader uint8, b []byte) int { + // Prevent too many bounds checks. + _ = b[IPv6FragmentHeaderSize:] + binary.BigEndian.PutUint32(b[idV6:], h.Identification) + binary.BigEndian.PutUint16(b[fragOff:], h.FragmentOffset<= IPv6FragmentHeaderSize +} + +// NextHeader returns the value of the "next header" field of the ipv6 fragment. +func (b IPv6Fragment) NextHeader() uint8 { + return b[nextHdrFrag] +} + +// FragmentOffset returns the "fragment offset" field of the ipv6 fragment. +func (b IPv6Fragment) FragmentOffset() uint16 { + return binary.BigEndian.Uint16(b[fragOff:]) >> 3 +} + +// More returns the "more" field of the ipv6 fragment. +func (b IPv6Fragment) More() bool { + return b[more]&1 > 0 +} + +// Payload implements Network.Payload. +func (b IPv6Fragment) Payload() []byte { + return b[IPv6FragmentHeaderSize:] +} + +// ID returns the value of the identifier field of the ipv6 fragment. +func (b IPv6Fragment) ID() uint32 { + return binary.BigEndian.Uint32(b[idV6:]) +} + +// TransportProtocol implements Network.TransportProtocol. +func (b IPv6Fragment) TransportProtocol() tcpip.TransportProtocolNumber { + return tcpip.TransportProtocolNumber(b.NextHeader()) +} + +// The functions below have been added only to satisfy the Network interface. + +// Checksum is not supported by IPv6Fragment. +func (b IPv6Fragment) Checksum() uint16 { + panic("not supported") +} + +// SourceAddress is not supported by IPv6Fragment. +func (b IPv6Fragment) SourceAddress() tcpip.Address { + panic("not supported") +} + +// DestinationAddress is not supported by IPv6Fragment. +func (b IPv6Fragment) DestinationAddress() tcpip.Address { + panic("not supported") +} + +// SetSourceAddress is not supported by IPv6Fragment. +func (b IPv6Fragment) SetSourceAddress(tcpip.Address) { + panic("not supported") +} + +// SetDestinationAddress is not supported by IPv6Fragment. +func (b IPv6Fragment) SetDestinationAddress(tcpip.Address) { + panic("not supported") +} + +// SetChecksum is not supported by IPv6Fragment. +func (b IPv6Fragment) SetChecksum(uint16) { + panic("not supported") +} + +// TOS is not supported by IPv6Fragment. +func (b IPv6Fragment) TOS() (uint8, uint32) { + panic("not supported") +} + +// SetTOS is not supported by IPv6Fragment. +func (b IPv6Fragment) SetTOS(t uint8, l uint32) { + panic("not supported") +} diff --git a/pkg/tcpip/header/mld.go b/pkg/tcpip/header/mld.go new file mode 100644 index 0000000..b789dad --- /dev/null +++ b/pkg/tcpip/header/mld.go @@ -0,0 +1,103 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // MLDMinimumSize is the minimum size for an MLD message. + MLDMinimumSize = 20 + + // MLDHopLimit is the Hop Limit for all IPv6 packets with an MLD message, as + // per RFC 2710 section 3. + MLDHopLimit = 1 + + // mldMaximumResponseDelayOffset is the offset to the Maximum Response Delay + // field within MLD. + mldMaximumResponseDelayOffset = 0 + + // mldMulticastAddressOffset is the offset to the Multicast Address field + // within MLD. + mldMulticastAddressOffset = 4 +) + +// MLD is a Multicast Listener Discovery message in an ICMPv6 packet. +// +// MLD will only contain the body of an ICMPv6 packet. +// +// As per RFC 2710 section 3, MLD messages have the following format (MLD only +// holds the bytes after the first four bytes in the diagram below): +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type | Code | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Maximum Response Delay | Reserved | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + + +// | | +// + Multicast Address + +// | | +// + + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type MLD []byte + +// MaximumResponseDelay returns the Maximum Response Delay. +func (m MLD) MaximumResponseDelay() time.Duration { + // As per RFC 2710 section 3.4: + // + // The Maximum Response Delay field is meaningful only in Query + // messages, and specifies the maximum allowed delay before sending a + // responding Report, in units of milliseconds. In all other messages, + // it is set to zero by the sender and ignored by receivers. + return time.Duration(binary.BigEndian.Uint16(m[mldMaximumResponseDelayOffset:])) * time.Millisecond +} + +// SetMaximumResponseDelay sets the Maximum Response Delay field. +// +// maxRespDelayMS is the value in milliseconds. +func (m MLD) SetMaximumResponseDelay(maxRespDelayMS uint16) { + binary.BigEndian.PutUint16(m[mldMaximumResponseDelayOffset:], maxRespDelayMS) +} + +// MulticastAddress returns the Multicast Address. +func (m MLD) MulticastAddress() tcpip.Address { + // As per RFC 2710 section 3.5: + // + // In a Query message, the Multicast Address field is set to zero when + // sending a General Query, and set to a specific IPv6 multicast address + // when sending a Multicast-Address-Specific Query. + // + // In a Report or Done message, the Multicast Address field holds a + // specific IPv6 multicast address to which the message sender is + // listening or is ceasing to listen, respectively. + return tcpip.AddrFrom16([16]byte(m[mldMulticastAddressOffset:][:IPv6AddressSize])) +} + +// SetMulticastAddress sets the Multicast Address field. +func (m MLD) SetMulticastAddress(multicastAddress tcpip.Address) { + if n := copy(m[mldMulticastAddressOffset:], multicastAddress.AsSlice()); n != IPv6AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected to copy %d bytes", n, IPv6AddressSize)) + } +} diff --git a/pkg/tcpip/header/mldv2.go b/pkg/tcpip/header/mldv2.go new file mode 100644 index 0000000..b1a2544 --- /dev/null +++ b/pkg/tcpip/header/mldv2.go @@ -0,0 +1,539 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "bytes" + "encoding/binary" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // MLDv2QueryMinimumSize is the minimum size for an MLDv2 message. + MLDv2QueryMinimumSize = 24 + + mldv2QueryMaximumResponseCodeOffset = 0 + mldv2QueryResvSQRVOffset = 20 + mldv2QueryQRVMask = 0b111 + mldv2QueryQQICOffset = 21 + // mldv2QueryNumberOfSourcesOffset is the offset to the Number of Sources + // field within MLDv2Query. + mldv2QueryNumberOfSourcesOffset = 22 + + // MLDv2ReportMinimumSize is the minimum size of an MLDv2 report. + MLDv2ReportMinimumSize = 24 + + // mldv2QuerySourcesOffset is the offset to the Sources field within + // MLDv2Query. + mldv2QuerySourcesOffset = 24 +) + +// MLDv2RoutersAddress is the address to send MLDv2 reports to. +// +// As per RFC 3810 section 5.2.14, +// +// Version 2 Multicast Listener Reports are sent with an IP destination +// address of FF02:0:0:0:0:0:0:16, to which all MLDv2-capable multicast +// routers listen (see section 11 for IANA considerations related to +// this special destination address). +var MLDv2RoutersAddress = tcpip.AddrFrom16([16]byte{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16}) + +// MLDv2Query is a Multicast Listener Discovery Version 2 Query message in an +// ICMPv6 packet. +// +// MLDv2Query will only contain the body of an ICMPv6 packet. +// +// As per RFC 3810 section 5.1, MLDv2 Query messages have the following format +// (MLDv2Query only holds the bytes after the first four bytes in the diagram +// below): +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type = 130 | Code | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Maximum Response Code | Reserved | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Multicast Address * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Resv |S| QRV | QQIC | Number of Sources (N) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Source Address [1] * +// | | +// * * +// | | +// +- -+ +// | | +// * * +// | | +// * Source Address [2] * +// | | +// * * +// | | +// +- . -+ +// . . . +// . . . +// +- -+ +// | | +// * * +// | | +// * Source Address [N] * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type MLDv2Query MLD + +// MaximumResponseCode returns the Maximum Response Code +func (m MLDv2Query) MaximumResponseCode() uint16 { + return binary.BigEndian.Uint16(m[mldv2QueryMaximumResponseCodeOffset:]) +} + +// MLDv2MaximumResponseDelay returns the Maximum Response Delay in an MLDv2 +// Maximum Response Code. +// +// As per RFC 3810 section 5.1.3, +// +// The Maximum Response Code field specifies the maximum time allowed +// before sending a responding Report. The actual time allowed, called +// the Maximum Response Delay, is represented in units of milliseconds, +// and is derived from the Maximum Response Code as follows: +// +// If Maximum Response Code < 32768, +// Maximum Response Delay = Maximum Response Code +// +// If Maximum Response Code >=32768, Maximum Response Code represents a +// floating-point value as follows: +// +// 0 1 2 3 4 5 6 7 8 9 A B C D E F +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |1| exp | mant | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Maximum Response Delay = (mant | 0x1000) << (exp+3) +// +// Small values of Maximum Response Delay allow MLDv2 routers to tune +// the "leave latency" (the time between the moment the last node on a +// link ceases to listen to a specific multicast address and the moment +// the routing protocol is notified that there are no more listeners for +// that address). Larger values, especially in the exponential range, +// allow the tuning of the burstiness of MLD traffic on a link. +func MLDv2MaximumResponseDelay(codeRaw uint16) time.Duration { + code := time.Duration(codeRaw) + if code < 32768 { + return code * time.Millisecond + } + + const mantBits = 12 + const expMask = 0b111 + exp := (code >> mantBits) & expMask + mant := code & ((1 << mantBits) - 1) + return (mant | 0x1000) << (exp + 3) * time.Millisecond +} + +// MulticastAddress returns the Multicast Address. +func (m MLDv2Query) MulticastAddress() tcpip.Address { + // As per RFC 2710 section 3.5: + // + // In a Query message, the Multicast Address field is set to zero when + // sending a General Query, and set to a specific IPv6 multicast address + // when sending a Multicast-Address-Specific Query. + // + // In a Report or Done message, the Multicast Address field holds a + // specific IPv6 multicast address to which the message sender is + // listening or is ceasing to listen, respectively. + return tcpip.AddrFrom16([16]byte(m[mldMulticastAddressOffset:][:IPv6AddressSize])) +} + +// QuerierRobustnessVariable returns the querier's robustness variable. +func (m MLDv2Query) QuerierRobustnessVariable() uint8 { + return m[mldv2QueryResvSQRVOffset] & mldv2QueryQRVMask +} + +// QuerierQueryInterval returns the querier's query interval. +func (m MLDv2Query) QuerierQueryInterval() time.Duration { + return mldv2AndIGMPv3QuerierQueryCodeToInterval(m[mldv2QueryQQICOffset]) +} + +// Sources returns an iterator over source addresses in the query. +// +// Returns false if the message cannot hold the expected number of sources. +func (m MLDv2Query) Sources() (AddressIterator, bool) { + return makeAddressIterator( + m[mldv2QuerySourcesOffset:], + binary.BigEndian.Uint16(m[mldv2QueryNumberOfSourcesOffset:]), + IPv6AddressSize, + ) +} + +// MLDv2ReportRecordType is the type of an MLDv2 multicast address record +// found in an MLDv2 report, as per RFC 3810 section 5.2.12. +type MLDv2ReportRecordType int + +// MLDv2 multicast address record types, as per RFC 3810 section 5.2.12. +const ( + MLDv2ReportRecordModeIsInclude MLDv2ReportRecordType = 1 + MLDv2ReportRecordModeIsExclude MLDv2ReportRecordType = 2 + MLDv2ReportRecordChangeToIncludeMode MLDv2ReportRecordType = 3 + MLDv2ReportRecordChangeToExcludeMode MLDv2ReportRecordType = 4 + MLDv2ReportRecordAllowNewSources MLDv2ReportRecordType = 5 + MLDv2ReportRecordBlockOldSources MLDv2ReportRecordType = 6 +) + +const ( + mldv2ReportMulticastAddressRecordMinimumSize = 20 + mldv2ReportMulticastAddressRecordTypeOffset = 0 + mldv2ReportMulticastAddressRecordAuxDataLenOffset = 1 + mldv2ReportMulticastAddressRecordAuxDataLenUnits = 4 + mldv2ReportMulticastAddressRecordNumberOfSourcesOffset = 2 + mldv2ReportMulticastAddressRecordMulticastAddressOffset = 4 + mldv2ReportMulticastAddressRecordSourcesOffset = 20 +) + +// MLDv2ReportMulticastAddressRecordSerializer is an MLDv2 Multicast Address +// Record serializer. +// +// As per RFC 3810 section 5.2, a Multicast Address Record has the following +// internal format: +// +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Record Type | Aux Data Len | Number of Sources (N) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Multicast Address * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Source Address [1] * +// | | +// * * +// | | +// +- -+ +// | | +// * * +// | | +// * Source Address [2] * +// | | +// * * +// | | +// +- -+ +// . . . +// . . . +// . . . +// +- -+ +// | | +// * * +// | | +// * Source Address [N] * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Auxiliary Data . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type MLDv2ReportMulticastAddressRecordSerializer struct { + RecordType MLDv2ReportRecordType + MulticastAddress tcpip.Address + Sources []tcpip.Address +} + +// Length returns the number of bytes this serializer would occupy. +func (s *MLDv2ReportMulticastAddressRecordSerializer) Length() int { + return mldv2ReportMulticastAddressRecordSourcesOffset + len(s.Sources)*IPv6AddressSize +} + +func copyIPv6Address(dst []byte, src tcpip.Address) { + if n := copy(dst, src.AsSlice()); n != IPv6AddressSize { + panic(fmt.Sprintf("got copy(...) = %d, want = %d", n, IPv6AddressSize)) + } +} + +// SerializeInto serializes the record into the buffer. +// +// Panics if the buffer does not have enough space to fit the record. +func (s *MLDv2ReportMulticastAddressRecordSerializer) SerializeInto(b []byte) { + b[mldv2ReportMulticastAddressRecordTypeOffset] = byte(s.RecordType) + b[mldv2ReportMulticastAddressRecordAuxDataLenOffset] = 0 + binary.BigEndian.PutUint16(b[mldv2ReportMulticastAddressRecordNumberOfSourcesOffset:], uint16(len(s.Sources))) + copyIPv6Address(b[mldv2ReportMulticastAddressRecordMulticastAddressOffset:], s.MulticastAddress) + b = b[mldv2ReportMulticastAddressRecordSourcesOffset:] + for _, source := range s.Sources { + copyIPv6Address(b, source) + b = b[IPv6AddressSize:] + } +} + +const ( + mldv2ReportReservedOffset = 0 + mldv2ReportNumberOfMulticastAddressRecordsOffset = 2 + mldv2ReportMulticastAddressRecordsOffset = 4 +) + +// MLDv2ReportSerializer is an MLD Version 2 Report serializer. +// +// As per RFC 3810 section 5.2, +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type = 143 | Reserved | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reserved |Nr of Mcast Address Records (M)| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [1] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [2] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | . | +// . . . +// | . | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [M] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type MLDv2ReportSerializer struct { + Records []MLDv2ReportMulticastAddressRecordSerializer +} + +// Length returns the number of bytes this serializer would occupy. +func (s *MLDv2ReportSerializer) Length() int { + ret := mldv2ReportMulticastAddressRecordsOffset + for _, record := range s.Records { + ret += record.Length() + } + return ret +} + +// SerializeInto serializes the report into the buffer. +// +// Panics if the buffer does not have enough space to fit the report. +func (s *MLDv2ReportSerializer) SerializeInto(b []byte) { + binary.BigEndian.PutUint16(b[mldv2ReportReservedOffset:], 0) + binary.BigEndian.PutUint16(b[mldv2ReportNumberOfMulticastAddressRecordsOffset:], uint16(len(s.Records))) + b = b[mldv2ReportMulticastAddressRecordsOffset:] + for _, record := range s.Records { + len := record.Length() + record.SerializeInto(b[:len]) + b = b[len:] + } +} + +// MLDv2ReportMulticastAddressRecord is an MLDv2 record. +// +// As per RFC 3810 section 5.2, a Multicast Address Record has the following +// internal format: +// +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Record Type | Aux Data Len | Number of Sources (N) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Multicast Address * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// * * +// | | +// * Source Address [1] * +// | | +// * * +// | | +// +- -+ +// | | +// * * +// | | +// * Source Address [2] * +// | | +// * * +// | | +// +- -+ +// . . . +// . . . +// . . . +// +- -+ +// | | +// * * +// | | +// * Source Address [N] * +// | | +// * * +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Auxiliary Data . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type MLDv2ReportMulticastAddressRecord []byte + +// RecordType returns the type of this record. +func (r MLDv2ReportMulticastAddressRecord) RecordType() MLDv2ReportRecordType { + return MLDv2ReportRecordType(r[mldv2ReportMulticastAddressRecordTypeOffset]) +} + +// AuxDataLen returns the length of the auxiliary data in this record. +func (r MLDv2ReportMulticastAddressRecord) AuxDataLen() int { + return int(r[mldv2ReportMulticastAddressRecordAuxDataLenOffset]) * mldv2ReportMulticastAddressRecordAuxDataLenUnits +} + +// numberOfSources returns the number of sources in this record. +func (r MLDv2ReportMulticastAddressRecord) numberOfSources() uint16 { + return binary.BigEndian.Uint16(r[mldv2ReportMulticastAddressRecordNumberOfSourcesOffset:]) +} + +// MulticastAddress returns the multicast address this record targets. +func (r MLDv2ReportMulticastAddressRecord) MulticastAddress() tcpip.Address { + return tcpip.AddrFrom16([16]byte(r[mldv2ReportMulticastAddressRecordMulticastAddressOffset:][:IPv6AddressSize])) +} + +// Sources returns an iterator over source addresses in the query. +// +// Returns false if the message cannot hold the expected number of sources. +func (r MLDv2ReportMulticastAddressRecord) Sources() (AddressIterator, bool) { + expectedLen := int(r.numberOfSources()) * IPv6AddressSize + b := r[mldv2ReportMulticastAddressRecordSourcesOffset:] + if len(b) < expectedLen { + return AddressIterator{}, false + } + return AddressIterator{addressSize: IPv6AddressSize, buf: bytes.NewBuffer(b[:expectedLen])}, true +} + +// MLDv2Report is an MLDv2 Report. +// +// As per RFC 3810 section 5.2, +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type = 143 | Reserved | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reserved |Nr of Mcast Address Records (M)| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [1] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [2] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | . | +// . . . +// | . | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Multicast Address Record [M] . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +type MLDv2Report []byte + +// MLDv2ReportMulticastAddressRecordIterator is an iterator over MLDv2 Multicast +// Address Records. +type MLDv2ReportMulticastAddressRecordIterator struct { + recordsLeft uint16 + buf *bytes.Buffer +} + +// MLDv2ReportMulticastAddressRecordIteratorNextDisposition is the possible +// return values from MLDv2ReportMulticastAddressRecordIterator.Next. +type MLDv2ReportMulticastAddressRecordIteratorNextDisposition int + +const ( + // MLDv2ReportMulticastAddressRecordIteratorNextOk indicates that a multicast + // address record was yielded. + MLDv2ReportMulticastAddressRecordIteratorNextOk MLDv2ReportMulticastAddressRecordIteratorNextDisposition = iota + + // MLDv2ReportMulticastAddressRecordIteratorNextDone indicates that the iterator + // has been exhausted. + MLDv2ReportMulticastAddressRecordIteratorNextDone + + // MLDv2ReportMulticastAddressRecordIteratorNextErrBufferTooShort indicates + // that the iterator expected another record, but the buffer ended + // prematurely. + MLDv2ReportMulticastAddressRecordIteratorNextErrBufferTooShort +) + +// Next returns the next MLDv2 Multicast Address Record. +func (it *MLDv2ReportMulticastAddressRecordIterator) Next() (MLDv2ReportMulticastAddressRecord, MLDv2ReportMulticastAddressRecordIteratorNextDisposition) { + if it.recordsLeft == 0 { + return MLDv2ReportMulticastAddressRecord{}, MLDv2ReportMulticastAddressRecordIteratorNextDone + } + if it.buf.Len() < mldv2ReportMulticastAddressRecordMinimumSize { + return MLDv2ReportMulticastAddressRecord{}, MLDv2ReportMulticastAddressRecordIteratorNextErrBufferTooShort + } + + hdr := MLDv2ReportMulticastAddressRecord(it.buf.Bytes()) + expectedLen := mldv2ReportMulticastAddressRecordMinimumSize + + int(hdr.AuxDataLen()) + int(hdr.numberOfSources())*IPv6AddressSize + + bytes := it.buf.Next(expectedLen) + if len(bytes) < expectedLen { + return MLDv2ReportMulticastAddressRecord{}, MLDv2ReportMulticastAddressRecordIteratorNextErrBufferTooShort + } + it.recordsLeft-- + return MLDv2ReportMulticastAddressRecord(bytes), MLDv2ReportMulticastAddressRecordIteratorNextOk +} + +// MulticastAddressRecords returns an iterator of MLDv2 Multicast Address +// Records. +func (m MLDv2Report) MulticastAddressRecords() MLDv2ReportMulticastAddressRecordIterator { + return MLDv2ReportMulticastAddressRecordIterator{ + recordsLeft: binary.BigEndian.Uint16(m[mldv2ReportNumberOfMulticastAddressRecordsOffset:]), + buf: bytes.NewBuffer(m[mldv2ReportMulticastAddressRecordsOffset:]), + } +} diff --git a/pkg/tcpip/header/mldv2_igmpv3_common.go b/pkg/tcpip/header/mldv2_igmpv3_common.go new file mode 100644 index 0000000..a2729c5 --- /dev/null +++ b/pkg/tcpip/header/mldv2_igmpv3_common.go @@ -0,0 +1,124 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "bytes" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +func mldv2AndIGMPv3QuerierQueryCodeToInterval(code uint8) time.Duration { + // MLDv2: As per RFC 3810 section 5.1.19, + // + // The Querier's Query Interval Code field specifies the [Query + // Interval] used by the Querier. The actual interval, called the + // Querier's Query Interval (QQI), is represented in units of seconds, + // and is derived from the Querier's Query Interval Code as follows: + // + // If QQIC < 128, QQI = QQIC + // + // If QQIC >= 128, QQIC represents a floating-point value as follows: + // + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |1| exp | mant | + // +-+-+-+-+-+-+-+-+ + // + // QQI = (mant | 0x10) << (exp + 3) + // + // Multicast routers that are not the current Querier adopt the QQI + // value from the most recently received Query as their own [Query + // Interval] value, unless that most recently received QQI was zero, in + // which case the receiving routers use the default [Query Interval] + // value specified in section 9.2. + // + // IGMPv3: As per RFC 3376 section 4.1.7, + // + // The Querier's Query Interval Code field specifies the [Query + // Interval] used by the querier. The actual interval, called the + // Querier's Query Interval (QQI), is represented in units of seconds + // and is derived from the Querier's Query Interval Code as follows: + // + // If QQIC < 128, QQI = QQIC + // + // If QQIC >= 128, QQIC represents a floating-point value as follows: + // + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |1| exp | mant | + // +-+-+-+-+-+-+-+-+ + // + // QQI = (mant | 0x10) << (exp + 3) + // + // Multicast routers that are not the current querier adopt the QQI + // value from the most recently received Query as their own [Query + // Interval] value, unless that most recently received QQI was zero, in + // which case the receiving routers use the default [Query Interval] + // value specified in section 8.2. + interval := time.Duration(code) + if interval < 128 { + return interval * time.Second + } + + const expMask = 0b111 + const mantBits = 4 + mant := interval & ((1 << mantBits) - 1) + exp := (interval >> mantBits) & expMask + return (mant | 0x10) << (exp + 3) * time.Second +} + +// MakeAddressIterator returns an AddressIterator. +func MakeAddressIterator(addressSize int, buf *bytes.Buffer) AddressIterator { + return AddressIterator{addressSize: addressSize, buf: buf} +} + +// AddressIterator is an iterator over IPv6 addresses. +type AddressIterator struct { + addressSize int + buf *bytes.Buffer +} + +// Done indicates that the iterator has been exhausted/has no more elements. +func (it *AddressIterator) Done() bool { + return it.buf.Len() == 0 +} + +// Next returns the next address in the iterator. +// +// Returns false if the iterator has been exhausted. +func (it *AddressIterator) Next() (tcpip.Address, bool) { + if it.Done() { + var emptyAddress tcpip.Address + return emptyAddress, false + } + + b := it.buf.Next(it.addressSize) + if len(b) != it.addressSize { + panic(fmt.Sprintf("got len(buf.Next(%d)) = %d, want = %d", it.addressSize, len(b), it.addressSize)) + } + + return tcpip.AddrFromSlice(b), true +} + +func makeAddressIterator(b []byte, expectedAddresses uint16, addressSize int) (AddressIterator, bool) { + expectedLen := int(expectedAddresses) * addressSize + if len(b) < expectedLen { + return AddressIterator{}, false + } + return MakeAddressIterator(addressSize, bytes.NewBuffer(b[:expectedLen])), true +} diff --git a/pkg/tcpip/header/ndp_neighbor_advert.go b/pkg/tcpip/header/ndp_neighbor_advert.go new file mode 100644 index 0000000..57bef3f --- /dev/null +++ b/pkg/tcpip/header/ndp_neighbor_advert.go @@ -0,0 +1,110 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import "github.com/sagernet/gvisor/pkg/tcpip" + +// NDPNeighborAdvert is an NDP Neighbor Advertisement message. It will +// only contain the body of an ICMPv6 packet. +// +// See RFC 4861 section 4.4 for more details. +type NDPNeighborAdvert []byte + +const ( + // NDPNAMinimumSize is the minimum size of a valid NDP Neighbor + // Advertisement message (body of an ICMPv6 packet). + NDPNAMinimumSize = 20 + + // ndpNATargetAddressOffset is the start of the Target Address + // field within an NDPNeighborAdvert. + ndpNATargetAddressOffset = 4 + + // ndpNAOptionsOffset is the start of the NDP options in an + // NDPNeighborAdvert. + ndpNAOptionsOffset = ndpNATargetAddressOffset + IPv6AddressSize + + // ndpNAFlagsOffset is the offset of the flags within an + // NDPNeighborAdvert + ndpNAFlagsOffset = 0 + + // ndpNARouterFlagMask is the mask of the Router Flag field in + // the flags byte within in an NDPNeighborAdvert. + ndpNARouterFlagMask = (1 << 7) + + // ndpNASolicitedFlagMask is the mask of the Solicited Flag field in + // the flags byte within in an NDPNeighborAdvert. + ndpNASolicitedFlagMask = (1 << 6) + + // ndpNAOverrideFlagMask is the mask of the Override Flag field in + // the flags byte within in an NDPNeighborAdvert. + ndpNAOverrideFlagMask = (1 << 5) +) + +// TargetAddress returns the value within the Target Address field. +func (b NDPNeighborAdvert) TargetAddress() tcpip.Address { + return tcpip.AddrFrom16Slice(b[ndpNATargetAddressOffset:][:IPv6AddressSize]) +} + +// SetTargetAddress sets the value within the Target Address field. +func (b NDPNeighborAdvert) SetTargetAddress(addr tcpip.Address) { + copy(b[ndpNATargetAddressOffset:][:IPv6AddressSize], addr.AsSlice()) +} + +// RouterFlag returns the value of the Router Flag field. +func (b NDPNeighborAdvert) RouterFlag() bool { + return b[ndpNAFlagsOffset]&ndpNARouterFlagMask != 0 +} + +// SetRouterFlag sets the value in the Router Flag field. +func (b NDPNeighborAdvert) SetRouterFlag(f bool) { + if f { + b[ndpNAFlagsOffset] |= ndpNARouterFlagMask + } else { + b[ndpNAFlagsOffset] &^= ndpNARouterFlagMask + } +} + +// SolicitedFlag returns the value of the Solicited Flag field. +func (b NDPNeighborAdvert) SolicitedFlag() bool { + return b[ndpNAFlagsOffset]&ndpNASolicitedFlagMask != 0 +} + +// SetSolicitedFlag sets the value in the Solicited Flag field. +func (b NDPNeighborAdvert) SetSolicitedFlag(f bool) { + if f { + b[ndpNAFlagsOffset] |= ndpNASolicitedFlagMask + } else { + b[ndpNAFlagsOffset] &^= ndpNASolicitedFlagMask + } +} + +// OverrideFlag returns the value of the Override Flag field. +func (b NDPNeighborAdvert) OverrideFlag() bool { + return b[ndpNAFlagsOffset]&ndpNAOverrideFlagMask != 0 +} + +// SetOverrideFlag sets the value in the Override Flag field. +func (b NDPNeighborAdvert) SetOverrideFlag(f bool) { + if f { + b[ndpNAFlagsOffset] |= ndpNAOverrideFlagMask + } else { + b[ndpNAFlagsOffset] &^= ndpNAOverrideFlagMask + } +} + +// Options returns an NDPOptions of the options body. +func (b NDPNeighborAdvert) Options() NDPOptions { + return NDPOptions(b[ndpNAOptionsOffset:]) +} diff --git a/pkg/tcpip/header/ndp_neighbor_solicit.go b/pkg/tcpip/header/ndp_neighbor_solicit.go new file mode 100644 index 0000000..c8475dd --- /dev/null +++ b/pkg/tcpip/header/ndp_neighbor_solicit.go @@ -0,0 +1,52 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import "github.com/sagernet/gvisor/pkg/tcpip" + +// NDPNeighborSolicit is an NDP Neighbor Solicitation message. It will only +// contain the body of an ICMPv6 packet. +// +// See RFC 4861 section 4.3 for more details. +type NDPNeighborSolicit []byte + +const ( + // NDPNSMinimumSize is the minimum size of a valid NDP Neighbor + // Solicitation message (body of an ICMPv6 packet). + NDPNSMinimumSize = 20 + + // ndpNSTargetAddessOffset is the start of the Target Address + // field within an NDPNeighborSolicit. + ndpNSTargetAddessOffset = 4 + + // ndpNSOptionsOffset is the start of the NDP options in an + // NDPNeighborSolicit. + ndpNSOptionsOffset = ndpNSTargetAddessOffset + IPv6AddressSize +) + +// TargetAddress returns the value within the Target Address field. +func (b NDPNeighborSolicit) TargetAddress() tcpip.Address { + return tcpip.AddrFrom16Slice(b[ndpNSTargetAddessOffset:][:IPv6AddressSize]) +} + +// SetTargetAddress sets the value within the Target Address field. +func (b NDPNeighborSolicit) SetTargetAddress(addr tcpip.Address) { + copy(b[ndpNSTargetAddessOffset:][:IPv6AddressSize], addr.AsSlice()) +} + +// Options returns an NDPOptions of the options body. +func (b NDPNeighborSolicit) Options() NDPOptions { + return NDPOptions(b[ndpNSOptionsOffset:]) +} diff --git a/pkg/tcpip/header/ndp_options.go b/pkg/tcpip/header/ndp_options.go new file mode 100644 index 0000000..e58816f --- /dev/null +++ b/pkg/tcpip/header/ndp_options.go @@ -0,0 +1,1072 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// ndpOptionIdentifier is an NDP option type identifier. +type ndpOptionIdentifier uint8 + +const ( + // ndpSourceLinkLayerAddressOptionType is the type of the Source Link Layer + // Address option, as per RFC 4861 section 4.6.1. + ndpSourceLinkLayerAddressOptionType ndpOptionIdentifier = 1 + + // ndpTargetLinkLayerAddressOptionType is the type of the Target Link Layer + // Address option, as per RFC 4861 section 4.6.1. + ndpTargetLinkLayerAddressOptionType ndpOptionIdentifier = 2 + + // ndpPrefixInformationType is the type of the Prefix Information + // option, as per RFC 4861 section 4.6.2. + ndpPrefixInformationType ndpOptionIdentifier = 3 + + // ndpNonceOptionType is the type of the Nonce option, as per + // RFC 3971 section 5.3.2. + ndpNonceOptionType ndpOptionIdentifier = 14 + + // ndpRecursiveDNSServerOptionType is the type of the Recursive DNS + // Server option, as per RFC 8106 section 5.1. + ndpRecursiveDNSServerOptionType ndpOptionIdentifier = 25 + + // ndpDNSSearchListOptionType is the type of the DNS Search List option, + // as per RFC 8106 section 5.2. + ndpDNSSearchListOptionType ndpOptionIdentifier = 31 +) + +const ( + // NDPLinkLayerAddressSize is the size of a Source or Target Link Layer + // Address option for an Ethernet address. + NDPLinkLayerAddressSize = 8 + + // ndpPrefixInformationLength is the expected length, in bytes, of the + // body of an NDP Prefix Information option, as per RFC 4861 section + // 4.6.2 which specifies that the Length field is 4. Given this, the + // expected length, in bytes, is 30 because 4 * lengthByteUnits (8) - 2 + // (Type & Length) = 30. + ndpPrefixInformationLength = 30 + + // ndpPrefixInformationPrefixLengthOffset is the offset of the Prefix + // Length field within an NDPPrefixInformation. + ndpPrefixInformationPrefixLengthOffset = 0 + + // ndpPrefixInformationFlagsOffset is the offset of the flags byte + // within an NDPPrefixInformation. + ndpPrefixInformationFlagsOffset = 1 + + // ndpPrefixInformationOnLinkFlagMask is the mask of the On-Link Flag + // field in the flags byte within an NDPPrefixInformation. + ndpPrefixInformationOnLinkFlagMask = 1 << 7 + + // ndpPrefixInformationAutoAddrConfFlagMask is the mask of the + // Autonomous Address-Configuration flag field in the flags byte within + // an NDPPrefixInformation. + ndpPrefixInformationAutoAddrConfFlagMask = 1 << 6 + + // ndpPrefixInformationReserved1FlagsMask is the mask of the Reserved1 + // field in the flags byte within an NDPPrefixInformation. + ndpPrefixInformationReserved1FlagsMask = 63 + + // ndpPrefixInformationValidLifetimeOffset is the start of the 4-byte + // Valid Lifetime field within an NDPPrefixInformation. + ndpPrefixInformationValidLifetimeOffset = 2 + + // ndpPrefixInformationPreferredLifetimeOffset is the start of the + // 4-byte Preferred Lifetime field within an NDPPrefixInformation. + ndpPrefixInformationPreferredLifetimeOffset = 6 + + // ndpPrefixInformationReserved2Offset is the start of the 4-byte + // Reserved2 field within an NDPPrefixInformation. + ndpPrefixInformationReserved2Offset = 10 + + // ndpPrefixInformationReserved2Length is the length of the Reserved2 + // field. + // + // It is 4 bytes. + ndpPrefixInformationReserved2Length = 4 + + // ndpPrefixInformationPrefixOffset is the start of the Prefix field + // within an NDPPrefixInformation. + ndpPrefixInformationPrefixOffset = 14 + + // ndpRecursiveDNSServerLifetimeOffset is the start of the 4-byte + // Lifetime field within an NDPRecursiveDNSServer. + ndpRecursiveDNSServerLifetimeOffset = 2 + + // ndpRecursiveDNSServerAddressesOffset is the start of the addresses + // for IPv6 Recursive DNS Servers within an NDPRecursiveDNSServer. + ndpRecursiveDNSServerAddressesOffset = 6 + + // minNDPRecursiveDNSServerLength is the minimum NDP Recursive DNS Server + // option's body size when it contains at least one IPv6 address, as per + // RFC 8106 section 5.3.1. + minNDPRecursiveDNSServerBodySize = 22 + + // ndpDNSSearchListLifetimeOffset is the start of the 4-byte + // Lifetime field within an NDPDNSSearchList. + ndpDNSSearchListLifetimeOffset = 2 + + // ndpDNSSearchListDomainNamesOffset is the start of the DNS search list + // domain names within an NDPDNSSearchList. + ndpDNSSearchListDomainNamesOffset = 6 + + // minNDPDNSSearchListBodySize is the minimum NDP DNS Search List option's + // body size when it contains at least one domain name, as per RFC 8106 + // section 5.3.1. + minNDPDNSSearchListBodySize = 14 + + // maxDomainNameLabelLength is the maximum length of a domain name + // label, as per RFC 1035 section 3.1. + maxDomainNameLabelLength = 63 + + // maxDomainNameLength is the maximum length of a domain name, including + // label AND label length octet, as per RFC 1035 section 3.1. + maxDomainNameLength = 255 + + // lengthByteUnits is the multiplier factor for the Length field of an + // NDP option. That is, the length field for NDP options is in units of + // 8 octets, as per RFC 4861 section 4.6. + lengthByteUnits = 8 + + // NDPInfiniteLifetime is a value that represents infinity for the + // 4-byte lifetime fields found in various NDP options. Its value is + // (2^32 - 1)s = 4294967295s. + NDPInfiniteLifetime = time.Second * math.MaxUint32 +) + +// NDPOptionIterator is an iterator of NDPOption. +// +// Note, between when an NDPOptionIterator is obtained and last used, no changes +// to the NDPOptions may happen. Doing so may cause undefined and unexpected +// behaviour. It is fine to obtain an NDPOptionIterator, iterate over the first +// few NDPOption then modify the backing NDPOptions so long as the +// NDPOptionIterator obtained before modification is no longer used. +type NDPOptionIterator struct { + opts *bytes.Buffer +} + +// Potential errors when iterating over an NDPOptions. +var ( + ErrNDPOptMalformedBody = errors.New("NDP option has a malformed body") + ErrNDPOptMalformedHeader = errors.New("NDP option has a malformed header") +) + +// Next returns the next element in the backing NDPOptions, or true if we are +// done, or false if an error occurred. +// +// The return can be read as option, done, error. Note, option should only be +// used if done is false and error is nil. +func (i *NDPOptionIterator) Next() (NDPOption, bool, error) { + for { + // Do we still have elements to look at? + if i.opts.Len() == 0 { + return nil, true, nil + } + + // Get the Type field. + temp, err := i.opts.ReadByte() + if err != nil { + if err != io.EOF { + // ReadByte should only ever return nil or io.EOF. + panic(fmt.Sprintf("unexpected error when reading the option's Type field: %s", err)) + } + + // We use io.ErrUnexpectedEOF as exhausting the buffer is unexpected once + // we start parsing an option; we expect the buffer to contain enough + // bytes for the whole option. + return nil, true, fmt.Errorf("unexpectedly exhausted buffer when reading the option's Type field: %w", io.ErrUnexpectedEOF) + } + kind := ndpOptionIdentifier(temp) + + // Get the Length field. + length, err := i.opts.ReadByte() + if err != nil { + if err != io.EOF { + panic(fmt.Sprintf("unexpected error when reading the option's Length field for %s: %s", kind, err)) + } + + return nil, true, fmt.Errorf("unexpectedly exhausted buffer when reading the option's Length field for %s: %w", kind, io.ErrUnexpectedEOF) + } + + // This would indicate an erroneous NDP option as the Length field should + // never be 0. + if length == 0 { + return nil, true, fmt.Errorf("zero valued Length field for %s: %w", kind, ErrNDPOptMalformedHeader) + } + + // Get the body. + numBytes := int(length) * lengthByteUnits + numBodyBytes := numBytes - 2 + body := i.opts.Next(numBodyBytes) + if len(body) < numBodyBytes { + return nil, true, fmt.Errorf("unexpectedly exhausted buffer when reading the option's Body for %s: %w", kind, io.ErrUnexpectedEOF) + } + + switch kind { + case ndpSourceLinkLayerAddressOptionType: + return NDPSourceLinkLayerAddressOption(body), false, nil + + case ndpTargetLinkLayerAddressOptionType: + return NDPTargetLinkLayerAddressOption(body), false, nil + + case ndpNonceOptionType: + return NDPNonceOption(body), false, nil + + case ndpRouteInformationType: + if numBodyBytes > ndpRouteInformationMaxLength { + return nil, true, fmt.Errorf("got %d bytes for NDP Route Information option's body, expected at max %d bytes: %w", numBodyBytes, ndpRouteInformationMaxLength, ErrNDPOptMalformedBody) + } + opt := NDPRouteInformation(body) + if err := opt.hasError(); err != nil { + return nil, true, err + } + + return opt, false, nil + + case ndpPrefixInformationType: + // Make sure the length of a Prefix Information option + // body is ndpPrefixInformationLength, as per RFC 4861 + // section 4.6.2. + if numBodyBytes != ndpPrefixInformationLength { + return nil, true, fmt.Errorf("got %d bytes for NDP Prefix Information option's body, expected %d bytes: %w", numBodyBytes, ndpPrefixInformationLength, ErrNDPOptMalformedBody) + } + + return NDPPrefixInformation(body), false, nil + + case ndpRecursiveDNSServerOptionType: + opt := NDPRecursiveDNSServer(body) + if err := opt.checkAddresses(); err != nil { + return nil, true, err + } + + return opt, false, nil + + case ndpDNSSearchListOptionType: + opt := NDPDNSSearchList(body) + if err := opt.checkDomainNames(); err != nil { + return nil, true, err + } + + return opt, false, nil + + default: + // We do not yet recognize the option, just skip for + // now. This is okay because RFC 4861 allows us to + // skip/ignore any unrecognized options. However, + // we MUST recognized all the options in RFC 4861. + // + // TODO(b/141487990): Handle all NDP options as defined + // by RFC 4861. + } + } +} + +// NDPOptions is a buffer of NDP options as defined by RFC 4861 section 4.6. +type NDPOptions []byte + +// Iter returns an iterator of NDPOption. +// +// If check is true, Iter will do an integrity check on the options by iterating +// over it and returning an error if detected. +// +// See NDPOptionIterator for more information. +func (b NDPOptions) Iter(check bool) (NDPOptionIterator, error) { + it := NDPOptionIterator{ + opts: bytes.NewBuffer(b), + } + + if check { + it2 := NDPOptionIterator{ + opts: bytes.NewBuffer(b), + } + + for { + if _, done, err := it2.Next(); err != nil || done { + return it, err + } + } + } + + return it, nil +} + +// Serialize serializes the provided list of NDP options into b. +// +// Note, b must be of sufficient size to hold all the options in s. See +// NDPOptionsSerializer.Length for details on the getting the total size +// of a serialized NDPOptionsSerializer. +// +// Serialize may panic if b is not of sufficient size to hold all the options +// in s. +func (b NDPOptions) Serialize(s NDPOptionsSerializer) int { + done := 0 + + for _, o := range s { + l := paddedLength(o) + + if l == 0 { + continue + } + + b[0] = byte(o.kind()) + + // We know this safe because paddedLength would have returned + // 0 if o had an invalid length (> 255 * lengthByteUnits). + b[1] = uint8(l / lengthByteUnits) + + // Serialize NDP option body. + used := o.serializeInto(b[2:]) + + // Zero out remaining (padding) bytes, if any exists. + if used+2 < l { + clear(b[used+2 : l]) + } + + b = b[l:] + done += l + } + + return done +} + +// NDPOption is the set of functions to be implemented by all NDP option types. +type NDPOption interface { + fmt.Stringer + + // kind returns the type of the receiver. + kind() ndpOptionIdentifier + + // length returns the length of the body of the receiver, in bytes. + length() int + + // serializeInto serializes the receiver into the provided byte + // buffer. + // + // Note, the caller MUST provide a byte buffer with size of at least + // Length. Implementers of this function may assume that the byte buffer + // is of sufficient size. serializeInto MAY panic if the provided byte + // buffer is not of sufficient size. + // + // serializeInto will return the number of bytes that was used to + // serialize the receiver. Implementers must only use the number of + // bytes required to serialize the receiver. Callers MAY provide a + // larger buffer than required to serialize into. + serializeInto([]byte) int +} + +// paddedLength returns the length of o, in bytes, with any padding bytes, if +// required. +func paddedLength(o NDPOption) int { + l := o.length() + + if l == 0 { + return 0 + } + + // Length excludes the 2 Type and Length bytes. + l += 2 + + // Add extra bytes if needed to make sure the option is + // lengthByteUnits-byte aligned. We do this by adding lengthByteUnits-1 + // to l and then stripping off the last few LSBits from l. This will + // make sure that l is rounded up to the nearest unit of + // lengthByteUnits. This works since lengthByteUnits is a power of 2 + // (= 8). + mask := lengthByteUnits - 1 + l += mask + l &^= mask + + if l/lengthByteUnits > 255 { + // Should never happen because an option can only have a max + // value of 255 for its Length field, so just return 0 so this + // option does not get serialized. + // + // Returning 0 here will make sure that this option does not get + // serialized when NDPOptions.Serialize is called with the + // NDPOptionsSerializer that holds this option, effectively + // skipping this option during serialization. Also note that + // a value of zero for the Length field in an NDP option is + // invalid so this is another sign to the caller that this NDP + // option is malformed, as per RFC 4861 section 4.6. + return 0 + } + + return l +} + +// NDPOptionsSerializer is a serializer for NDP options. +type NDPOptionsSerializer []NDPOption + +// Length returns the total number of bytes required to serialize. +func (b NDPOptionsSerializer) Length() int { + l := 0 + + for _, o := range b { + l += paddedLength(o) + } + + return l +} + +// NDPNonceOption is the NDP Nonce Option as defined by RFC 3971 section 5.3.2. +// +// It is the first X bytes following the NDP option's Type and Length field +// where X is the value in Length multiplied by lengthByteUnits - 2 bytes. +type NDPNonceOption []byte + +// kind implements NDPOption. +func (o NDPNonceOption) kind() ndpOptionIdentifier { + return ndpNonceOptionType +} + +// length implements NDPOption. +func (o NDPNonceOption) length() int { + return len(o) +} + +// serializeInto implements NDPOption. +func (o NDPNonceOption) serializeInto(b []byte) int { + return copy(b, o) +} + +// String implements fmt.Stringer. +func (o NDPNonceOption) String() string { + return fmt.Sprintf("%T(%x)", o, []byte(o)) +} + +// Nonce returns the nonce value this option holds. +func (o NDPNonceOption) Nonce() []byte { + return o +} + +// NDPSourceLinkLayerAddressOption is the NDP Source Link Layer Option +// as defined by RFC 4861 section 4.6.1. +// +// It is the first X bytes following the NDP option's Type and Length field +// where X is the value in Length multiplied by lengthByteUnits - 2 bytes. +type NDPSourceLinkLayerAddressOption tcpip.LinkAddress + +// kind implements NDPOption. +func (o NDPSourceLinkLayerAddressOption) kind() ndpOptionIdentifier { + return ndpSourceLinkLayerAddressOptionType +} + +// length implements NDPOption. +func (o NDPSourceLinkLayerAddressOption) length() int { + return len(o) +} + +// serializeInto implements NDPOption. +func (o NDPSourceLinkLayerAddressOption) serializeInto(b []byte) int { + return copy(b, o) +} + +// String implements fmt.Stringer. +func (o NDPSourceLinkLayerAddressOption) String() string { + return fmt.Sprintf("%T(%s)", o, tcpip.LinkAddress(o)) +} + +// EthernetAddress will return an ethernet (MAC) address if the +// NDPSourceLinkLayerAddressOption's body has at minimum EthernetAddressSize +// bytes. If the body has more than EthernetAddressSize bytes, only the first +// EthernetAddressSize bytes are returned as that is all that is needed for an +// Ethernet address. +func (o NDPSourceLinkLayerAddressOption) EthernetAddress() tcpip.LinkAddress { + if len(o) >= EthernetAddressSize { + return tcpip.LinkAddress(o[:EthernetAddressSize]) + } + + return tcpip.LinkAddress([]byte(nil)) +} + +// NDPTargetLinkLayerAddressOption is the NDP Target Link Layer Option +// as defined by RFC 4861 section 4.6.1. +// +// It is the first X bytes following the NDP option's Type and Length field +// where X is the value in Length multiplied by lengthByteUnits - 2 bytes. +type NDPTargetLinkLayerAddressOption tcpip.LinkAddress + +// kind implements NDPOption. +func (o NDPTargetLinkLayerAddressOption) kind() ndpOptionIdentifier { + return ndpTargetLinkLayerAddressOptionType +} + +// length implements NDPOption. +func (o NDPTargetLinkLayerAddressOption) length() int { + return len(o) +} + +// serializeInto implements NDPOption. +func (o NDPTargetLinkLayerAddressOption) serializeInto(b []byte) int { + return copy(b, o) +} + +// String implements fmt.Stringer. +func (o NDPTargetLinkLayerAddressOption) String() string { + return fmt.Sprintf("%T(%s)", o, tcpip.LinkAddress(o)) +} + +// EthernetAddress will return an ethernet (MAC) address if the +// NDPTargetLinkLayerAddressOption's body has at minimum EthernetAddressSize +// bytes. If the body has more than EthernetAddressSize bytes, only the first +// EthernetAddressSize bytes are returned as that is all that is needed for an +// Ethernet address. +func (o NDPTargetLinkLayerAddressOption) EthernetAddress() tcpip.LinkAddress { + if len(o) >= EthernetAddressSize { + return tcpip.LinkAddress(o[:EthernetAddressSize]) + } + + return tcpip.LinkAddress([]byte(nil)) +} + +// NDPPrefixInformation is the NDP Prefix Information option as defined by +// RFC 4861 section 4.6.2. +// +// The length, in bytes, of a valid NDP Prefix Information option body MUST be +// ndpPrefixInformationLength bytes. +type NDPPrefixInformation []byte + +// kind implements NDPOption. +func (o NDPPrefixInformation) kind() ndpOptionIdentifier { + return ndpPrefixInformationType +} + +// length implements NDPOption. +func (o NDPPrefixInformation) length() int { + return ndpPrefixInformationLength +} + +// serializeInto implements NDPOption. +func (o NDPPrefixInformation) serializeInto(b []byte) int { + used := copy(b, o) + + // Zero out the Reserved1 field. + b[ndpPrefixInformationFlagsOffset] &^= ndpPrefixInformationReserved1FlagsMask + + // Zero out the Reserved2 field. + reserved2 := b[ndpPrefixInformationReserved2Offset:][:ndpPrefixInformationReserved2Length] + clear(reserved2) + + return used +} + +// String implements fmt.Stringer. +func (o NDPPrefixInformation) String() string { + return fmt.Sprintf("%T(O=%t, A=%t, PL=%s, VL=%s, Prefix=%s)", + o, + o.OnLinkFlag(), + o.AutonomousAddressConfigurationFlag(), + o.PreferredLifetime(), + o.ValidLifetime(), + o.Subnet()) +} + +// PrefixLength returns the value in the number of leading bits in the Prefix +// that are valid. +// +// Valid values are in the range [0, 128], but o may not always contain valid +// values. It is up to the caller to valdiate the Prefix Information option. +func (o NDPPrefixInformation) PrefixLength() uint8 { + return o[ndpPrefixInformationPrefixLengthOffset] +} + +// OnLinkFlag returns true of the prefix is considered on-link. On-link means +// that a forwarding node is not needed to send packets to other nodes on the +// same prefix. +// +// Note, when this function returns false, no statement is made about the +// on-link property of a prefix. That is, if OnLinkFlag returns false, the +// caller MUST NOT conclude that the prefix is off-link and MUST NOT update any +// previously stored state for this prefix about its on-link status. +func (o NDPPrefixInformation) OnLinkFlag() bool { + return o[ndpPrefixInformationFlagsOffset]&ndpPrefixInformationOnLinkFlagMask != 0 +} + +// AutonomousAddressConfigurationFlag returns true if the prefix can be used for +// Stateless Address Auto-Configuration (as specified in RFC 4862). +func (o NDPPrefixInformation) AutonomousAddressConfigurationFlag() bool { + return o[ndpPrefixInformationFlagsOffset]&ndpPrefixInformationAutoAddrConfFlagMask != 0 +} + +// ValidLifetime returns the length of time that the prefix is valid for the +// purpose of on-link determination. This value is relative to the send time of +// the packet that the Prefix Information option was present in. +// +// Note, a value of 0 implies the prefix should not be considered as on-link, +// and a value of infinity/forever is represented by +// NDPInfiniteLifetime. +func (o NDPPrefixInformation) ValidLifetime() time.Duration { + // The field is the time in seconds, as per RFC 4861 section 4.6.2. + return time.Second * time.Duration(binary.BigEndian.Uint32(o[ndpPrefixInformationValidLifetimeOffset:])) +} + +// PreferredLifetime returns the length of time that an address generated from +// the prefix via Stateless Address Auto-Configuration remains preferred. This +// value is relative to the send time of the packet that the Prefix Information +// option was present in. +// +// Note, a value of 0 implies that addresses generated from the prefix should +// no longer remain preferred, and a value of infinity is represented by +// NDPInfiniteLifetime. +// +// Also note that the value of this field MUST NOT exceed the Valid Lifetime +// field to avoid preferring addresses that are no longer valid, for the +// purpose of Stateless Address Auto-Configuration. +func (o NDPPrefixInformation) PreferredLifetime() time.Duration { + // The field is the time in seconds, as per RFC 4861 section 4.6.2. + return time.Second * time.Duration(binary.BigEndian.Uint32(o[ndpPrefixInformationPreferredLifetimeOffset:])) +} + +// Prefix returns an IPv6 address or a prefix of an IPv6 address. The Prefix +// Length field (see NDPPrefixInformation.PrefixLength) contains the number +// of valid leading bits in the prefix. +// +// Hosts SHOULD ignore an NDP Prefix Information option where the Prefix field +// holds the link-local prefix (fe80::). +func (o NDPPrefixInformation) Prefix() tcpip.Address { + return tcpip.AddrFrom16Slice(o[ndpPrefixInformationPrefixOffset:][:IPv6AddressSize]) +} + +// Subnet returns the Prefix field and Prefix Length field represented in a +// tcpip.Subnet. +func (o NDPPrefixInformation) Subnet() tcpip.Subnet { + addrWithPrefix := tcpip.AddressWithPrefix{ + Address: o.Prefix(), + PrefixLen: int(o.PrefixLength()), + } + return addrWithPrefix.Subnet() +} + +// NDPRecursiveDNSServer is the NDP Recursive DNS Server option, as defined by +// RFC 8106 section 5.1. +// +// To make sure that the option meets its minimum length and does not end in the +// middle of a DNS server's IPv6 address, the length of a valid +// NDPRecursiveDNSServer must meet the following constraint: +// +// (Length - ndpRecursiveDNSServerAddressesOffset) % IPv6AddressSize == 0 +type NDPRecursiveDNSServer []byte + +// Type returns the type of an NDP Recursive DNS Server option. +// +// kind implements NDPOption. +func (NDPRecursiveDNSServer) kind() ndpOptionIdentifier { + return ndpRecursiveDNSServerOptionType +} + +// length implements NDPOption. +func (o NDPRecursiveDNSServer) length() int { + return len(o) +} + +// serializeInto implements NDPOption. +func (o NDPRecursiveDNSServer) serializeInto(b []byte) int { + used := copy(b, o) + + // Zero out the reserved bytes that are before the Lifetime field. + clear(b[0:ndpRecursiveDNSServerLifetimeOffset]) + + return used +} + +// String implements fmt.Stringer. +func (o NDPRecursiveDNSServer) String() string { + lt := o.Lifetime() + addrs, err := o.Addresses() + if err != nil { + return fmt.Sprintf("%T([] valid for %s; err = %s)", o, lt, err) + } + return fmt.Sprintf("%T(%s valid for %s)", o, addrs, lt) +} + +// Lifetime returns the length of time that the DNS server addresses +// in this option may be used for name resolution. +// +// Note, a value of 0 implies the addresses should no longer be used, +// and a value of infinity/forever is represented by NDPInfiniteLifetime. +// +// Lifetime may panic if o does not have enough bytes to hold the Lifetime +// field. +func (o NDPRecursiveDNSServer) Lifetime() time.Duration { + // The field is the time in seconds, as per RFC 8106 section 5.1. + return time.Second * time.Duration(binary.BigEndian.Uint32(o[ndpRecursiveDNSServerLifetimeOffset:])) +} + +// Addresses returns the recursive DNS server IPv6 addresses that may be +// used for name resolution. +// +// Note, the addresses MAY be link-local addresses. +func (o NDPRecursiveDNSServer) Addresses() ([]tcpip.Address, error) { + var addrs []tcpip.Address + return addrs, o.iterAddresses(func(addr tcpip.Address) { addrs = append(addrs, addr) }) +} + +// checkAddresses iterates over the addresses in an NDP Recursive DNS Server +// option and returns any error it encounters. +func (o NDPRecursiveDNSServer) checkAddresses() error { + return o.iterAddresses(nil) +} + +// iterAddresses iterates over the addresses in an NDP Recursive DNS Server +// option and calls a function with each valid unicast IPv6 address. +// +// Note, the addresses MAY be link-local addresses. +func (o NDPRecursiveDNSServer) iterAddresses(fn func(tcpip.Address)) error { + if l := len(o); l < minNDPRecursiveDNSServerBodySize { + return fmt.Errorf("got %d bytes for NDP Recursive DNS Server option's body, expected at least %d bytes: %w", l, minNDPRecursiveDNSServerBodySize, io.ErrUnexpectedEOF) + } + + o = o[ndpRecursiveDNSServerAddressesOffset:] + l := len(o) + if l%IPv6AddressSize != 0 { + return fmt.Errorf("NDP Recursive DNS Server option's body ends in the middle of an IPv6 address (addresses body size = %d bytes): %w", l, ErrNDPOptMalformedBody) + } + + for i := 0; len(o) != 0; i++ { + addr := tcpip.AddrFrom16Slice(o[:IPv6AddressSize]) + if !IsV6UnicastAddress(addr) { + return fmt.Errorf("%d-th address (%s) in NDP Recursive DNS Server option is not a valid unicast IPv6 address: %w", i, addr, ErrNDPOptMalformedBody) + } + + if fn != nil { + fn(addr) + } + + o = o[IPv6AddressSize:] + } + + return nil +} + +// NDPDNSSearchList is the NDP DNS Search List option, as defined by +// RFC 8106 section 5.2. +type NDPDNSSearchList []byte + +// kind implements NDPOption. +func (o NDPDNSSearchList) kind() ndpOptionIdentifier { + return ndpDNSSearchListOptionType +} + +// length implements NDPOption. +func (o NDPDNSSearchList) length() int { + return len(o) +} + +// serializeInto implements NDPOption. +func (o NDPDNSSearchList) serializeInto(b []byte) int { + used := copy(b, o) + + // Zero out the reserved bytes that are before the Lifetime field. + clear(b[0:ndpDNSSearchListLifetimeOffset]) + + return used +} + +// String implements fmt.Stringer. +func (o NDPDNSSearchList) String() string { + lt := o.Lifetime() + domainNames, err := o.DomainNames() + if err != nil { + return fmt.Sprintf("%T([] valid for %s; err = %s)", o, lt, err) + } + return fmt.Sprintf("%T(%s valid for %s)", o, domainNames, lt) +} + +// Lifetime returns the length of time that the DNS search list of domain names +// in this option may be used for name resolution. +// +// Note, a value of 0 implies the domain names should no longer be used, +// and a value of infinity/forever is represented by NDPInfiniteLifetime. +func (o NDPDNSSearchList) Lifetime() time.Duration { + // The field is the time in seconds, as per RFC 8106 section 5.1. + return time.Second * time.Duration(binary.BigEndian.Uint32(o[ndpDNSSearchListLifetimeOffset:])) +} + +// DomainNames returns a DNS search list of domain names. +// +// DomainNames will parse the backing buffer as outlined by RFC 1035 section +// 3.1 and return a list of strings, with all domain names in lower case. +func (o NDPDNSSearchList) DomainNames() ([]string, error) { + var domainNames []string + return domainNames, o.iterDomainNames(func(domainName string) { domainNames = append(domainNames, domainName) }) +} + +// checkDomainNames iterates over the domain names in an NDP DNS Search List +// option and returns any error it encounters. +func (o NDPDNSSearchList) checkDomainNames() error { + return o.iterDomainNames(nil) +} + +// iterDomainNames iterates over the domain names in an NDP DNS Search List +// option and calls a function with each valid domain name. +func (o NDPDNSSearchList) iterDomainNames(fn func(string)) error { + if l := len(o); l < minNDPDNSSearchListBodySize { + return fmt.Errorf("got %d bytes for NDP DNS Search List option's body, expected at least %d bytes: %w", l, minNDPDNSSearchListBodySize, io.ErrUnexpectedEOF) + } + + var searchList bytes.Reader + searchList.Reset(o[ndpDNSSearchListDomainNamesOffset:]) + + var scratch [maxDomainNameLength]byte + domainName := bytes.NewBuffer(scratch[:]) + + // Parse the domain names, as per RFC 1035 section 3.1. + for searchList.Len() != 0 { + domainName.Reset() + + // Parse a label within a domain name, as per RFC 1035 section 3.1. + for { + // The first byte is the label length. + labelLenByte, err := searchList.ReadByte() + if err != nil { + if err != io.EOF { + // ReadByte should only ever return nil or io.EOF. + panic(fmt.Sprintf("unexpected error when reading a label's length: %s", err)) + } + + // We use io.ErrUnexpectedEOF as exhausting the buffer is unexpected + // once we start parsing a domain name; we expect the buffer to contain + // enough bytes for the whole domain name. + return fmt.Errorf("unexpected exhausted buffer while parsing a new label for a domain from NDP Search List option: %w", io.ErrUnexpectedEOF) + } + labelLen := int(labelLenByte) + + // A zero-length label implies the end of a domain name. + if labelLen == 0 { + // If the domain name is empty or we have no callback function, do + // nothing further with the current domain name. + if domainName.Len() == 0 || fn == nil { + break + } + + // Ignore the trailing period in the parsed domain name. + domainName.Truncate(domainName.Len() - 1) + fn(domainName.String()) + break + } + + // The label's length must not exceed the maximum length for a label. + if labelLen > maxDomainNameLabelLength { + return fmt.Errorf("label length of %d bytes is greater than the max label length of %d bytes for an NDP Search List option: %w", labelLen, maxDomainNameLabelLength, ErrNDPOptMalformedBody) + } + + // The label (and trailing period) must not make the domain name too long. + if labelLen+1 > domainName.Cap()-domainName.Len() { + return fmt.Errorf("label would make an NDP Search List option's domain name longer than the max domain name length of %d bytes: %w", maxDomainNameLength, ErrNDPOptMalformedBody) + } + + // Copy the label and add a trailing period. + for i := 0; i < labelLen; i++ { + b, err := searchList.ReadByte() + if err != nil { + if err != io.EOF { + panic(fmt.Sprintf("unexpected error when reading domain name's label: %s", err)) + } + + return fmt.Errorf("read %d out of %d bytes for a domain name's label from NDP Search List option: %w", i, labelLen, io.ErrUnexpectedEOF) + } + + // As per RFC 1035 section 2.3.1: + // 1) the label must only contain ASCII include letters, digits and + // hyphens + // 2) the first character in a label must be a letter + // 3) the last letter in a label must be a letter or digit + + if !isLetter(b) { + if i == 0 { + return fmt.Errorf("first character of a domain name's label in an NDP Search List option must be a letter, got character code = %d: %w", b, ErrNDPOptMalformedBody) + } + + if b == '-' { + if i == labelLen-1 { + return fmt.Errorf("last character of a domain name's label in an NDP Search List option must not be a hyphen (-): %w", ErrNDPOptMalformedBody) + } + } else if !isDigit(b) { + return fmt.Errorf("domain name's label in an NDP Search List option may only contain letters, digits and hyphens, got character code = %d: %w", b, ErrNDPOptMalformedBody) + } + } + + // If b is an upper case character, make it lower case. + if isUpperLetter(b) { + b = b - 'A' + 'a' + } + + if err := domainName.WriteByte(b); err != nil { + panic(fmt.Sprintf("unexpected error writing label to domain name buffer: %s", err)) + } + } + if err := domainName.WriteByte('.'); err != nil { + panic(fmt.Sprintf("unexpected error writing trailing period to domain name buffer: %s", err)) + } + } + } + + return nil +} + +func isLetter(b byte) bool { + return b >= 'a' && b <= 'z' || isUpperLetter(b) +} + +func isUpperLetter(b byte) bool { + return b >= 'A' && b <= 'Z' +} + +func isDigit(b byte) bool { + return b >= '0' && b <= '9' +} + +// As per RFC 4191 section 2.3, +// +// 2.3. Route Information Option +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type | Length | Prefix Length |Resvd|Prf|Resvd| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Route Lifetime | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Prefix (Variable Length) | +// . . +// . . +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Fields: +// +// Type 24 +// +// +// Length 8-bit unsigned integer. The length of the option +// (including the Type and Length fields) in units of 8 +// octets. The Length field is 1, 2, or 3 depending on the +// Prefix Length. If Prefix Length is greater than 64, then +// Length must be 3. If Prefix Length is greater than 0, +// then Length must be 2 or 3. If Prefix Length is zero, +// then Length must be 1, 2, or 3. +const ( + ndpRouteInformationType = ndpOptionIdentifier(24) + ndpRouteInformationMaxLength = 22 + + ndpRouteInformationPrefixLengthIdx = 0 + ndpRouteInformationFlagsIdx = 1 + ndpRouteInformationPrfShift = 3 + ndpRouteInformationPrfMask = 3 << ndpRouteInformationPrfShift + ndpRouteInformationRouteLifetimeIdx = 2 + ndpRouteInformationRoutePrefixIdx = 6 +) + +// NDPRouteInformation is the NDP Router Information option, as defined by +// RFC 4191 section 2.3. +type NDPRouteInformation []byte + +func (NDPRouteInformation) kind() ndpOptionIdentifier { + return ndpRouteInformationType +} + +func (o NDPRouteInformation) length() int { + return len(o) +} + +func (o NDPRouteInformation) serializeInto(b []byte) int { + return copy(b, o) +} + +// String implements fmt.Stringer. +func (o NDPRouteInformation) String() string { + return fmt.Sprintf("%T", o) +} + +// PrefixLength returns the length of the prefix. +func (o NDPRouteInformation) PrefixLength() uint8 { + return o[ndpRouteInformationPrefixLengthIdx] +} + +// RoutePreference returns the preference of the route over other routes to the +// same destination but through a different router. +func (o NDPRouteInformation) RoutePreference() NDPRoutePreference { + return NDPRoutePreference((o[ndpRouteInformationFlagsIdx] & ndpRouteInformationPrfMask) >> ndpRouteInformationPrfShift) +} + +// RouteLifetime returns the lifetime of the route. +// +// Note, a value of 0 implies the route is now invalid and a value of +// infinity/forever is represented by NDPInfiniteLifetime. +func (o NDPRouteInformation) RouteLifetime() time.Duration { + return time.Second * time.Duration(binary.BigEndian.Uint32(o[ndpRouteInformationRouteLifetimeIdx:])) +} + +// Prefix returns the prefix of the destination subnet this route is for. +func (o NDPRouteInformation) Prefix() (tcpip.Subnet, error) { + prefixLength := int(o.PrefixLength()) + if max := IPv6AddressSize * 8; prefixLength > max { + return tcpip.Subnet{}, fmt.Errorf("got prefix length = %d, want <= %d", prefixLength, max) + } + + prefix := o[ndpRouteInformationRoutePrefixIdx:] + var addrBytes [IPv6AddressSize]byte + if n := copy(addrBytes[:], prefix); n != len(prefix) { + panic(fmt.Sprintf("got copy(addrBytes, prefix) = %d, want = %d", n, len(prefix))) + } + + return tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16(addrBytes), + PrefixLen: prefixLength, + }.Subnet(), nil +} + +func (o NDPRouteInformation) hasError() error { + l := len(o) + if l < ndpRouteInformationRoutePrefixIdx { + return fmt.Errorf("%T too small, got = %d bytes: %w", o, l, ErrNDPOptMalformedBody) + } + + prefixLength := int(o.PrefixLength()) + if max := IPv6AddressSize * 8; prefixLength > max { + return fmt.Errorf("got prefix length = %d, want <= %d: %w", prefixLength, max, ErrNDPOptMalformedBody) + } + + // Length 8-bit unsigned integer. The length of the option + // (including the Type and Length fields) in units of 8 + // octets. The Length field is 1, 2, or 3 depending on the + // Prefix Length. If Prefix Length is greater than 64, then + // Length must be 3. If Prefix Length is greater than 0, + // then Length must be 2 or 3. If Prefix Length is zero, + // then Length must be 1, 2, or 3. + l += 2 // Add 2 bytes for the type and length bytes. + lengthField := l / lengthByteUnits + if prefixLength > 64 { + if lengthField != 3 { + return fmt.Errorf("Length field must be 3 when Prefix Length (%d) is > 64 (got = %d): %w", prefixLength, lengthField, ErrNDPOptMalformedBody) + } + } else if prefixLength > 0 { + if lengthField != 2 && lengthField != 3 { + return fmt.Errorf("Length field must be 2 or 3 when Prefix Length (%d) is between 0 and 64 (got = %d): %w", prefixLength, lengthField, ErrNDPOptMalformedBody) + } + } else if lengthField == 0 || lengthField > 3 { + return fmt.Errorf("Length field must be 1, 2, or 3 when Prefix Length is zero (got = %d): %w", lengthField, ErrNDPOptMalformedBody) + } + + return nil +} diff --git a/pkg/tcpip/header/ndp_router_advert.go b/pkg/tcpip/header/ndp_router_advert.go new file mode 100644 index 0000000..e2456c0 --- /dev/null +++ b/pkg/tcpip/header/ndp_router_advert.go @@ -0,0 +1,204 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + "fmt" + "time" +) + +var _ fmt.Stringer = NDPRoutePreference(0) + +// NDPRoutePreference is the preference values for default routers or +// more-specific routes. +// +// As per RFC 4191 section 2.1, +// +// Default router preferences and preferences for more-specific routes +// are encoded the same way. +// +// Preference values are encoded as a two-bit signed integer, as +// follows: +// +// 01 High +// 00 Medium (default) +// 11 Low +// 10 Reserved - MUST NOT be sent +// +// Note that implementations can treat the value as a two-bit signed +// integer. +// +// Having just three values reinforces that they are not metrics and +// more values do not appear to be necessary for reasonable scenarios. +type NDPRoutePreference uint8 + +const ( + // HighRoutePreference indicates a high preference, as per + // RFC 4191 section 2.1. + HighRoutePreference NDPRoutePreference = 0b01 + + // MediumRoutePreference indicates a medium preference, as per + // RFC 4191 section 2.1. + // + // This is the default preference value. + MediumRoutePreference = 0b00 + + // LowRoutePreference indicates a low preference, as per + // RFC 4191 section 2.1. + LowRoutePreference = 0b11 + + // ReservedRoutePreference is a reserved preference value, as per + // RFC 4191 section 2.1. + // + // It MUST NOT be sent. + ReservedRoutePreference = 0b10 +) + +// String implements fmt.Stringer. +func (p NDPRoutePreference) String() string { + switch p { + case HighRoutePreference: + return "HighRoutePreference" + case MediumRoutePreference: + return "MediumRoutePreference" + case LowRoutePreference: + return "LowRoutePreference" + case ReservedRoutePreference: + return "ReservedRoutePreference" + default: + return fmt.Sprintf("NDPRoutePreference(%d)", p) + } +} + +// NDPRouterAdvert is an NDP Router Advertisement message. It will only contain +// the body of an ICMPv6 packet. +// +// See RFC 4861 section 4.2 and RFC 4191 section 2.2 for more details. +type NDPRouterAdvert []byte + +// As per RFC 4191 section 2.2, +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Type | Code | Checksum | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Cur Hop Limit |M|O|H|Prf|Resvd| Router Lifetime | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reachable Time | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Retrans Timer | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Options ... +// +-+-+-+-+-+-+-+-+-+-+-+- +const ( + // NDPRAMinimumSize is the minimum size of a valid NDP Router + // Advertisement message (body of an ICMPv6 packet). + NDPRAMinimumSize = 12 + + // ndpRACurrHopLimitOffset is the byte of the Curr Hop Limit field + // within an NDPRouterAdvert. + ndpRACurrHopLimitOffset = 0 + + // ndpRAFlagsOffset is the byte with the NDP RA bit-fields/flags + // within an NDPRouterAdvert. + ndpRAFlagsOffset = 1 + + // ndpRAManagedAddrConfFlagMask is the mask of the Managed Address + // Configuration flag within the bit-field/flags byte of an + // NDPRouterAdvert. + ndpRAManagedAddrConfFlagMask = (1 << 7) + + // ndpRAOtherConfFlagMask is the mask of the Other Configuration flag + // within the bit-field/flags byte of an NDPRouterAdvert. + ndpRAOtherConfFlagMask = (1 << 6) + + // ndpDefaultRouterPreferenceShift is the shift of the Prf (Default Router + // Preference) field within the flags byte of an NDPRouterAdvert. + ndpDefaultRouterPreferenceShift = 3 + + // ndpDefaultRouterPreferenceMask is the mask of the Prf (Default Router + // Preference) field within the flags byte of an NDPRouterAdvert. + ndpDefaultRouterPreferenceMask = (0b11 << ndpDefaultRouterPreferenceShift) + + // ndpRARouterLifetimeOffset is the start of the 2-byte Router Lifetime + // field within an NDPRouterAdvert. + ndpRARouterLifetimeOffset = 2 + + // ndpRAReachableTimeOffset is the start of the 4-byte Reachable Time + // field within an NDPRouterAdvert. + ndpRAReachableTimeOffset = 4 + + // ndpRARetransTimerOffset is the start of the 4-byte Retrans Timer + // field within an NDPRouterAdvert. + ndpRARetransTimerOffset = 8 + + // ndpRAOptionsOffset is the start of the NDP options in an + // NDPRouterAdvert. + ndpRAOptionsOffset = 12 +) + +// CurrHopLimit returns the value of the Curr Hop Limit field. +func (b NDPRouterAdvert) CurrHopLimit() uint8 { + return b[ndpRACurrHopLimitOffset] +} + +// ManagedAddrConfFlag returns the value of the Managed Address Configuration +// flag. +func (b NDPRouterAdvert) ManagedAddrConfFlag() bool { + return b[ndpRAFlagsOffset]&ndpRAManagedAddrConfFlagMask != 0 +} + +// OtherConfFlag returns the value of the Other Configuration flag. +func (b NDPRouterAdvert) OtherConfFlag() bool { + return b[ndpRAFlagsOffset]&ndpRAOtherConfFlagMask != 0 +} + +// DefaultRouterPreference returns the Default Router Preference field. +func (b NDPRouterAdvert) DefaultRouterPreference() NDPRoutePreference { + return NDPRoutePreference((b[ndpRAFlagsOffset] & ndpDefaultRouterPreferenceMask) >> ndpDefaultRouterPreferenceShift) +} + +// RouterLifetime returns the lifetime associated with the default router. A +// value of 0 means the source of the Router Advertisement is not a default +// router and SHOULD NOT appear on the default router list. Note, a value of 0 +// only means that the router should not be used as a default router, it does +// not apply to other information contained in the Router Advertisement. +func (b NDPRouterAdvert) RouterLifetime() time.Duration { + // The field is the time in seconds, as per RFC 4861 section 4.2. + return time.Second * time.Duration(binary.BigEndian.Uint16(b[ndpRARouterLifetimeOffset:])) +} + +// ReachableTime returns the time that a node assumes a neighbor is reachable +// after having received a reachability confirmation. A value of 0 means +// that it is unspecified by the source of the Router Advertisement message. +func (b NDPRouterAdvert) ReachableTime() time.Duration { + // The field is the time in milliseconds, as per RFC 4861 section 4.2. + return time.Millisecond * time.Duration(binary.BigEndian.Uint32(b[ndpRAReachableTimeOffset:])) +} + +// RetransTimer returns the time between retransmitted Neighbor Solicitation +// messages. A value of 0 means that it is unspecified by the source of the +// Router Advertisement message. +func (b NDPRouterAdvert) RetransTimer() time.Duration { + // The field is the time in milliseconds, as per RFC 4861 section 4.2. + return time.Millisecond * time.Duration(binary.BigEndian.Uint32(b[ndpRARetransTimerOffset:])) +} + +// Options returns an NDPOptions of the options body. +func (b NDPRouterAdvert) Options() NDPOptions { + return NDPOptions(b[ndpRAOptionsOffset:]) +} diff --git a/pkg/tcpip/header/ndp_router_solicit.go b/pkg/tcpip/header/ndp_router_solicit.go new file mode 100644 index 0000000..5ca2e5c --- /dev/null +++ b/pkg/tcpip/header/ndp_router_solicit.go @@ -0,0 +1,36 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +// NDPRouterSolicit is an NDP Router Solicitation message. It will only contain +// the body of an ICMPv6 packet. +// +// See RFC 4861 section 4.1 for more details. +type NDPRouterSolicit []byte + +const ( + // NDPRSMinimumSize is the minimum size of a valid NDP Router + // Solicitation message (body of an ICMPv6 packet). + NDPRSMinimumSize = 4 + + // ndpRSOptionsOffset is the start of the NDP options in an + // NDPRouterSolicit. + ndpRSOptionsOffset = 4 +) + +// Options returns an NDPOptions of the options body. +func (b NDPRouterSolicit) Options() NDPOptions { + return NDPOptions(b[ndpRSOptionsOffset:]) +} diff --git a/pkg/tcpip/header/ndpoptionidentifier_string.go b/pkg/tcpip/header/ndpoptionidentifier_string.go new file mode 100644 index 0000000..952c974 --- /dev/null +++ b/pkg/tcpip/header/ndpoptionidentifier_string.go @@ -0,0 +1,56 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by "stringer -type ndpOptionIdentifier"; DO NOT EDIT. + +package header + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ndpSourceLinkLayerAddressOptionType-1] + _ = x[ndpTargetLinkLayerAddressOptionType-2] + _ = x[ndpPrefixInformationType-3] + _ = x[ndpNonceOptionType-14] + _ = x[ndpRecursiveDNSServerOptionType-25] + _ = x[ndpDNSSearchListOptionType-31] +} + +const ( + _ndpOptionIdentifier_name_0 = "ndpSourceLinkLayerAddressOptionTypendpTargetLinkLayerAddressOptionTypendpPrefixInformationType" + _ndpOptionIdentifier_name_1 = "ndpNonceOptionType" + _ndpOptionIdentifier_name_2 = "ndpRecursiveDNSServerOptionType" + _ndpOptionIdentifier_name_3 = "ndpDNSSearchListOptionType" +) + +var _ndpOptionIdentifier_index_0 = [...]uint8{0, 35, 70, 94} + +func (i ndpOptionIdentifier) String() string { + switch { + case 1 <= i && i <= 3: + i -= 1 + return _ndpOptionIdentifier_name_0[_ndpOptionIdentifier_index_0[i]:_ndpOptionIdentifier_index_0[i+1]] + case i == 14: + return _ndpOptionIdentifier_name_1 + case i == 25: + return _ndpOptionIdentifier_name_2 + case i == 31: + return _ndpOptionIdentifier_name_3 + default: + return "ndpOptionIdentifier(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/pkg/tcpip/header/parse/parse.go b/pkg/tcpip/header/parse/parse.go new file mode 100644 index 0000000..f31a62c --- /dev/null +++ b/pkg/tcpip/header/parse/parse.go @@ -0,0 +1,243 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package parse provides utilities to parse packets. +package parse + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// ARP populates pkt's network header with an ARP header found in +// pkt.Data. +// +// Returns true if the header was successfully parsed. +func ARP(pkt *stack.PacketBuffer) bool { + _, ok := pkt.NetworkHeader().Consume(header.ARPSize) + if ok { + pkt.NetworkProtocolNumber = header.ARPProtocolNumber + } + return ok +} + +// IPv4 parses an IPv4 packet found in pkt.Data and populates pkt's network +// header with the IPv4 header. +// +// Returns true if the header was successfully parsed. +func IPv4(pkt *stack.PacketBuffer) bool { + hdr, ok := pkt.Data().PullUp(header.IPv4MinimumSize) + if !ok { + return false + } + ipHdr := header.IPv4(hdr) + + // Header may have options, determine the true header length. + headerLen := int(ipHdr.HeaderLength()) + if headerLen < header.IPv4MinimumSize { + // TODO(gvisor.dev/issue/2404): Per RFC 791, IHL needs to be at least 5 in + // order for the packet to be valid. Figure out if we want to reject this + // case. + headerLen = header.IPv4MinimumSize + } + hdr, ok = pkt.NetworkHeader().Consume(headerLen) + if !ok { + return false + } + ipHdr = header.IPv4(hdr) + length := int(ipHdr.TotalLength()) - len(hdr) + if length < 0 { + return false + } + + pkt.NetworkProtocolNumber = header.IPv4ProtocolNumber + pkt.Data().CapLength(length) + return true +} + +// IPv6 parses an IPv6 packet found in pkt.Data and populates pkt's network +// header with the IPv6 header. +func IPv6(pkt *stack.PacketBuffer) (proto tcpip.TransportProtocolNumber, fragID uint32, fragOffset uint16, fragMore bool, ok bool) { + hdr, ok := pkt.Data().PullUp(header.IPv6MinimumSize) + if !ok { + return 0, 0, 0, false, false + } + ipHdr := header.IPv6(hdr) + + // Create a VV to parse the packet. We don't plan to modify anything here. + // dataVV consists of: + // - Any IPv6 header bytes after the first 40 (i.e. extensions). + // - The transport header, if present. + // - Any other payload data. + dataBuf := pkt.Data().ToBuffer() + dataBuf.TrimFront(header.IPv6MinimumSize) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(ipHdr.NextHeader()), dataBuf) + defer it.Release() + + // Iterate over the IPv6 extensions to find their length. + var nextHdr tcpip.TransportProtocolNumber + var extensionsSize int64 + +traverseExtensions: + for { + extHdr, done, err := it.Next() + if err != nil { + break + } + + // If we exhaust the extension list, the entire packet is the IPv6 header + // and (possibly) extensions. + if done { + extensionsSize = dataBuf.Size() + break + } + + switch extHdr := extHdr.(type) { + case header.IPv6FragmentExtHdr: + if extHdr.IsAtomic() { + // This fragment extension header indicates that this packet is an + // atomic fragment. An atomic fragment is a fragment that contains + // all the data required to reassemble a full packet. As per RFC 6946, + // atomic fragments must not interfere with "normal" fragmented traffic + // so we skip processing the fragment instead of feeding it through the + // reassembly process below. + continue + } + + if fragID == 0 && fragOffset == 0 && !fragMore { + fragID = extHdr.ID() + fragOffset = extHdr.FragmentOffset() + fragMore = extHdr.More() + } + rawPayload := it.AsRawHeader(true /* consume */) + extensionsSize = dataBuf.Size() - rawPayload.Buf.Size() + rawPayload.Release() + extHdr.Release() + break traverseExtensions + + case header.IPv6RawPayloadHeader: + // We've found the payload after any extensions. + extensionsSize = dataBuf.Size() - extHdr.Buf.Size() + nextHdr = tcpip.TransportProtocolNumber(extHdr.Identifier) + extHdr.Release() + break traverseExtensions + default: + extHdr.Release() + // Any other extension is a no-op, keep looping until we find the payload. + } + } + + // Put the IPv6 header with extensions in pkt.NetworkHeader(). + hdr, ok = pkt.NetworkHeader().Consume(header.IPv6MinimumSize + int(extensionsSize)) + if !ok { + panic(fmt.Sprintf("pkt.Data should have at least %d bytes, but only has %d.", header.IPv6MinimumSize+extensionsSize, pkt.Data().Size())) + } + ipHdr = header.IPv6(hdr) + pkt.Data().CapLength(int(ipHdr.PayloadLength())) + pkt.NetworkProtocolNumber = header.IPv6ProtocolNumber + + return nextHdr, fragID, fragOffset, fragMore, true +} + +// UDP parses a UDP packet found in pkt.Data and populates pkt's transport +// header with the UDP header. +// +// Returns true if the header was successfully parsed. +func UDP(pkt *stack.PacketBuffer) bool { + _, ok := pkt.TransportHeader().Consume(header.UDPMinimumSize) + pkt.TransportProtocolNumber = header.UDPProtocolNumber + return ok +} + +// TCP parses a TCP packet found in pkt.Data and populates pkt's transport +// header with the TCP header. +// +// Returns true if the header was successfully parsed. +func TCP(pkt *stack.PacketBuffer) bool { + // TCP header is variable length, peek at it first. + hdrLen := header.TCPMinimumSize + hdr, ok := pkt.Data().PullUp(hdrLen) + if !ok { + return false + } + + // If the header has options, pull those up as well. + if offset := int(header.TCP(hdr).DataOffset()); offset > header.TCPMinimumSize && offset <= pkt.Data().Size() { + // TODO(gvisor.dev/issue/2404): Figure out whether to reject this kind of + // packets. + hdrLen = offset + } + + _, ok = pkt.TransportHeader().Consume(hdrLen) + pkt.TransportProtocolNumber = header.TCPProtocolNumber + return ok +} + +// ICMPv4 populates the packet buffer's transport header with an ICMPv4 header, +// if present. +// +// Returns true if an ICMPv4 header was successfully parsed. +func ICMPv4(pkt *stack.PacketBuffer) bool { + if _, ok := pkt.TransportHeader().Consume(header.ICMPv4MinimumSize); ok { + pkt.TransportProtocolNumber = header.ICMPv4ProtocolNumber + return true + } + return false +} + +// ICMPv6 populates the packet buffer's transport header with an ICMPv4 header, +// if present. +// +// Returns true if an ICMPv6 header was successfully parsed. +func ICMPv6(pkt *stack.PacketBuffer) bool { + hdr, ok := pkt.Data().PullUp(header.ICMPv6MinimumSize) + if !ok { + return false + } + + h := header.ICMPv6(hdr) + switch h.Type() { + case header.ICMPv6RouterSolicit, + header.ICMPv6RouterAdvert, + header.ICMPv6NeighborSolicit, + header.ICMPv6NeighborAdvert, + header.ICMPv6RedirectMsg, + header.ICMPv6MulticastListenerQuery, + header.ICMPv6MulticastListenerReport, + header.ICMPv6MulticastListenerV2Report, + header.ICMPv6MulticastListenerDone: + size := pkt.Data().Size() + if _, ok := pkt.TransportHeader().Consume(size); !ok { + panic(fmt.Sprintf("expected to consume the full data of size = %d bytes into transport header", size)) + } + case header.ICMPv6DstUnreachable, + header.ICMPv6PacketTooBig, + header.ICMPv6TimeExceeded, + header.ICMPv6ParamProblem, + header.ICMPv6EchoRequest, + header.ICMPv6EchoReply: + fallthrough + default: + if _, ok := pkt.TransportHeader().Consume(header.ICMPv6MinimumSize); !ok { + // Checked above if the packet buffer holds at least the minimum size for + // an ICMPv6 packet. + panic(fmt.Sprintf("expected to consume %d bytes", header.ICMPv6MinimumSize)) + } + } + pkt.TransportProtocolNumber = header.ICMPv6ProtocolNumber + return true +} diff --git a/pkg/tcpip/header/parse/parse_state_autogen.go b/pkg/tcpip/header/parse/parse_state_autogen.go new file mode 100644 index 0000000..ad047be --- /dev/null +++ b/pkg/tcpip/header/parse/parse_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package parse diff --git a/pkg/tcpip/header/tcp.go b/pkg/tcpip/header/tcp.go new file mode 100644 index 0000000..a95d13e --- /dev/null +++ b/pkg/tcpip/header/tcp.go @@ -0,0 +1,730 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + + "github.com/google/btree" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +// These constants are the offsets of the respective fields in the TCP header. +const ( + TCPSrcPortOffset = 0 + TCPDstPortOffset = 2 + TCPSeqNumOffset = 4 + TCPAckNumOffset = 8 + TCPDataOffset = 12 + TCPFlagsOffset = 13 + TCPWinSizeOffset = 14 + TCPChecksumOffset = 16 + TCPUrgentPtrOffset = 18 +) + +const ( + // MaxWndScale is maximum allowed window scaling, as described in + // RFC 1323, section 2.3, page 11. + MaxWndScale = 14 + + // TCPMaxSACKBlocks is the maximum number of SACK blocks that can + // be encoded in a TCP option field. + TCPMaxSACKBlocks = 4 +) + +// TCPFlags is the dedicated type for TCP flags. +type TCPFlags uint8 + +// Intersects returns true iff there are flags common to both f and o. +func (f TCPFlags) Intersects(o TCPFlags) bool { + return f&o != 0 +} + +// Contains returns true iff all the flags in o are contained within f. +func (f TCPFlags) Contains(o TCPFlags) bool { + return f&o == o +} + +// String implements Stringer.String. +func (f TCPFlags) String() string { + flagsStr := []byte("FSRPAUEC") + for i := range flagsStr { + if f&(1<> 4) * 4 +} + +// Payload returns the data in the TCP packet. +func (b TCP) Payload() []byte { + return b[b.DataOffset():] +} + +// Flags returns the flags field of the TCP header. +func (b TCP) Flags() TCPFlags { + return TCPFlags(b[TCPFlagsOffset]) +} + +// WindowSize returns the "window size" field of the TCP header. +func (b TCP) WindowSize() uint16 { + return binary.BigEndian.Uint16(b[TCPWinSizeOffset:]) +} + +// Checksum returns the "checksum" field of the TCP header. +func (b TCP) Checksum() uint16 { + return binary.BigEndian.Uint16(b[TCPChecksumOffset:]) +} + +// UrgentPointer returns the "urgent pointer" field of the TCP header. +func (b TCP) UrgentPointer() uint16 { + return binary.BigEndian.Uint16(b[TCPUrgentPtrOffset:]) +} + +// SetSourcePort sets the "source port" field of the TCP header. +func (b TCP) SetSourcePort(port uint16) { + binary.BigEndian.PutUint16(b[TCPSrcPortOffset:], port) +} + +// SetDestinationPort sets the "destination port" field of the TCP header. +func (b TCP) SetDestinationPort(port uint16) { + binary.BigEndian.PutUint16(b[TCPDstPortOffset:], port) +} + +// SetChecksum sets the checksum field of the TCP header. +func (b TCP) SetChecksum(xsum uint16) { + checksum.Put(b[TCPChecksumOffset:], xsum) +} + +// SetDataOffset sets the data offset field of the TCP header. headerLen should +// be the length of the TCP header in bytes. +func (b TCP) SetDataOffset(headerLen uint8) { + b[TCPDataOffset] = (headerLen / 4) << 4 +} + +// SetSequenceNumber sets the sequence number field of the TCP header. +func (b TCP) SetSequenceNumber(seqNum uint32) { + binary.BigEndian.PutUint32(b[TCPSeqNumOffset:], seqNum) +} + +// SetAckNumber sets the ack number field of the TCP header. +func (b TCP) SetAckNumber(ackNum uint32) { + binary.BigEndian.PutUint32(b[TCPAckNumOffset:], ackNum) +} + +// SetFlags sets the flags field of the TCP header. +func (b TCP) SetFlags(flags uint8) { + b[TCPFlagsOffset] = flags +} + +// SetWindowSize sets the window size field of the TCP header. +func (b TCP) SetWindowSize(rcvwnd uint16) { + binary.BigEndian.PutUint16(b[TCPWinSizeOffset:], rcvwnd) +} + +// SetUrgentPointer sets the window size field of the TCP header. +func (b TCP) SetUrgentPointer(urgentPointer uint16) { + binary.BigEndian.PutUint16(b[TCPUrgentPtrOffset:], urgentPointer) +} + +// CalculateChecksum calculates the checksum of the TCP segment. +// partialChecksum is the checksum of the network-layer pseudo-header +// and the checksum of the segment data. +func (b TCP) CalculateChecksum(partialChecksum uint16) uint16 { + // Calculate the rest of the checksum. + // return checksum.Checksum(b[:b.DataOffset()], partialChecksum) + xsum := checksum.Checksum(b[:TCPChecksumOffset], partialChecksum) + xsum = checksum.Checksum(b[TCPChecksumOffset+2:b.DataOffset()], xsum) + return xsum +} + +// IsChecksumValid returns true iff the TCP header's checksum is valid. +func (b TCP) IsChecksumValid(src, dst tcpip.Address, payloadChecksum, payloadLength uint16) bool { + xsum := PseudoHeaderChecksum(TCPProtocolNumber, src, dst, uint16(b.DataOffset())+payloadLength) + xsum = checksum.Combine(xsum, payloadChecksum) + // return b.CalculateChecksum(xsum) == 0xffff + return checksum.Checksum(b[:b.DataOffset()], xsum) == 0xffff +} + +// Options returns a slice that holds the unparsed TCP options in the segment. +func (b TCP) Options() []byte { + return b[TCPMinimumSize:b.DataOffset()] +} + +// ParsedOptions returns a TCPOptions structure which parses and caches the TCP +// option values in the TCP segment. NOTE: Invoking this function repeatedly is +// expensive as it reparses the options on each invocation. +func (b TCP) ParsedOptions() TCPOptions { + return ParseTCPOptions(b.Options()) +} + +func (b TCP) encodeSubset(seq, ack uint32, flags TCPFlags, rcvwnd uint16) { + binary.BigEndian.PutUint32(b[TCPSeqNumOffset:], seq) + binary.BigEndian.PutUint32(b[TCPAckNumOffset:], ack) + b[TCPFlagsOffset] = uint8(flags) + binary.BigEndian.PutUint16(b[TCPWinSizeOffset:], rcvwnd) +} + +// Encode encodes all the fields of the TCP header. +func (b TCP) Encode(t *TCPFields) { + b.encodeSubset(t.SeqNum, t.AckNum, t.Flags, t.WindowSize) + b.SetSourcePort(t.SrcPort) + b.SetDestinationPort(t.DstPort) + b.SetDataOffset(t.DataOffset) + b.SetChecksum(t.Checksum) + b.SetUrgentPointer(t.UrgentPointer) +} + +// EncodePartial updates a subset of the fields of the TCP header. It is useful +// in cases when similar segments are produced. +func (b TCP) EncodePartial(partialChecksum, length uint16, seqnum, acknum uint32, flags TCPFlags, rcvwnd uint16) { + // Add the total length and "flags" field contributions to the checksum. + // We don't use the flags field directly from the header because it's a + // one-byte field with an odd offset, so it would be accounted for + // incorrectly by the Checksum routine. + tmp := make([]byte, 4) + binary.BigEndian.PutUint16(tmp, length) + binary.BigEndian.PutUint16(tmp[2:], uint16(flags)) + xsum := checksum.Checksum(tmp, partialChecksum) + + // Encode the passed-in fields. + b.encodeSubset(seqnum, acknum, flags, rcvwnd) + + // Add the contributions of the passed-in fields to the checksum. + xsum = checksum.Checksum(b[TCPSeqNumOffset:TCPSeqNumOffset+8], xsum) + xsum = checksum.Checksum(b[TCPWinSizeOffset:TCPWinSizeOffset+2], xsum) + + // Encode the checksum. + b.SetChecksum(^xsum) +} + +// SetSourcePortWithChecksumUpdate implements ChecksummableTransport. +func (b TCP) SetSourcePortWithChecksumUpdate(new uint16) { + old := b.SourcePort() + b.SetSourcePort(new) + b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) +} + +// SetDestinationPortWithChecksumUpdate implements ChecksummableTransport. +func (b TCP) SetDestinationPortWithChecksumUpdate(new uint16) { + old := b.DestinationPort() + b.SetDestinationPort(new) + b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) +} + +// UpdateChecksumPseudoHeaderAddress implements ChecksummableTransport. +func (b TCP) UpdateChecksumPseudoHeaderAddress(old, new tcpip.Address, fullChecksum bool) { + xsum := b.Checksum() + if fullChecksum { + xsum = ^xsum + } + + xsum = checksumUpdate2ByteAlignedAddress(xsum, old, new) + if fullChecksum { + xsum = ^xsum + } + + b.SetChecksum(xsum) +} + +// ParseSynOptions parses the options received in a SYN segment and returns the +// relevant ones. opts should point to the option part of the TCP header. +func ParseSynOptions(opts []byte, isAck bool) TCPSynOptions { + limit := len(opts) + + synOpts := TCPSynOptions{ + // Per RFC 1122, page 85: "If an MSS option is not received at + // connection setup, TCP MUST assume a default send MSS of 536." + MSS: TCPDefaultMSS, + // If no window scale option is specified, WS in options is + // returned as -1; this is because the absence of the option + // indicates that the we cannot use window scaling on the + // receive end either. + WS: -1, + } + + for i := 0; i < limit; { + switch opts[i] { + case TCPOptionEOL: + i = limit + case TCPOptionNOP: + i++ + case TCPOptionMSS: + if i+4 > limit || opts[i+1] != 4 { + return synOpts + } + mss := uint16(opts[i+2])<<8 | uint16(opts[i+3]) + if mss == 0 { + return synOpts + } + synOpts.MSS = mss + if mss < TCPMinimumSendMSS { + synOpts.MSS = TCPMinimumSendMSS + } + i += 4 + + case TCPOptionWS: + if i+3 > limit || opts[i+1] != 3 { + return synOpts + } + ws := int(opts[i+2]) + if ws > MaxWndScale { + ws = MaxWndScale + } + synOpts.WS = ws + i += 3 + + case TCPOptionTS: + if i+10 > limit || opts[i+1] != 10 { + return synOpts + } + synOpts.TSVal = binary.BigEndian.Uint32(opts[i+2:]) + if isAck { + // If the segment is a SYN-ACK then store the Timestamp Echo Reply + // in the segment. + synOpts.TSEcr = binary.BigEndian.Uint32(opts[i+6:]) + } + synOpts.TS = true + i += 10 + case TCPOptionSACKPermitted: + if i+2 > limit || opts[i+1] != 2 { + return synOpts + } + synOpts.SACKPermitted = true + i += 2 + + default: + // We don't recognize this option, just skip over it. + if i+2 > limit { + return synOpts + } + l := int(opts[i+1]) + // If the length is incorrect or if l+i overflows the + // total options length then return false. + if l < 2 || i+l > limit { + return synOpts + } + i += l + } + } + + return synOpts +} + +// ParseTCPOptions extracts and stores all known options in the provided byte +// slice in a TCPOptions structure. +func ParseTCPOptions(b []byte) TCPOptions { + opts := TCPOptions{} + limit := len(b) + for i := 0; i < limit; { + switch b[i] { + case TCPOptionEOL: + i = limit + case TCPOptionNOP: + i++ + case TCPOptionTS: + if i+10 > limit || (b[i+1] != 10) { + return opts + } + opts.TS = true + opts.TSVal = binary.BigEndian.Uint32(b[i+2:]) + opts.TSEcr = binary.BigEndian.Uint32(b[i+6:]) + i += 10 + case TCPOptionSACK: + if i+2 > limit { + // Malformed SACK block, just return and stop parsing. + return opts + } + sackOptionLen := int(b[i+1]) + if i+sackOptionLen > limit || (sackOptionLen-2)%8 != 0 { + // Malformed SACK block, just return and stop parsing. + return opts + } + numBlocks := (sackOptionLen - 2) / 8 + opts.SACKBlocks = []SACKBlock{} + for j := 0; j < numBlocks; j++ { + start := binary.BigEndian.Uint32(b[i+2+j*8:]) + end := binary.BigEndian.Uint32(b[i+2+j*8+4:]) + opts.SACKBlocks = append(opts.SACKBlocks, SACKBlock{ + Start: seqnum.Value(start), + End: seqnum.Value(end), + }) + } + i += sackOptionLen + default: + // We don't recognize this option, just skip over it. + if i+2 > limit { + return opts + } + l := int(b[i+1]) + // If the length is incorrect or if l+i overflows the + // total options length then return false. + if l < 2 || i+l > limit { + return opts + } + i += l + } + } + return opts +} + +// EncodeMSSOption encodes the MSS TCP option with the provided MSS values in +// the supplied buffer. If the provided buffer is not large enough then it just +// returns without encoding anything. It returns the number of bytes written to +// the provided buffer. +func EncodeMSSOption(mss uint32, b []byte) int { + if len(b) < TCPOptionMSSLength { + return 0 + } + b[0], b[1], b[2], b[3] = TCPOptionMSS, TCPOptionMSSLength, byte(mss>>8), byte(mss) + return TCPOptionMSSLength +} + +// EncodeWSOption encodes the WS TCP option with the WS value in the +// provided buffer. If the provided buffer is not large enough then it just +// returns without encoding anything. It returns the number of bytes written to +// the provided buffer. +func EncodeWSOption(ws int, b []byte) int { + if len(b) < TCPOptionWSLength { + return 0 + } + b[0], b[1], b[2] = TCPOptionWS, TCPOptionWSLength, uint8(ws) + return int(b[1]) +} + +// EncodeTSOption encodes the provided tsVal and tsEcr values as a TCP timestamp +// option into the provided buffer. If the buffer is smaller than expected it +// just returns without encoding anything. It returns the number of bytes +// written to the provided buffer. +func EncodeTSOption(tsVal, tsEcr uint32, b []byte) int { + if len(b) < TCPOptionTSLength { + return 0 + } + b[0], b[1] = TCPOptionTS, TCPOptionTSLength + binary.BigEndian.PutUint32(b[2:], tsVal) + binary.BigEndian.PutUint32(b[6:], tsEcr) + return int(b[1]) +} + +// EncodeSACKPermittedOption encodes a SACKPermitted option into the provided +// buffer. If the buffer is smaller than required it just returns without +// encoding anything. It returns the number of bytes written to the provided +// buffer. +func EncodeSACKPermittedOption(b []byte) int { + if len(b) < TCPOptionSackPermittedLength { + return 0 + } + + b[0], b[1] = TCPOptionSACKPermitted, TCPOptionSackPermittedLength + return int(b[1]) +} + +// EncodeSACKBlocks encodes the provided SACK blocks as a TCP SACK option block +// in the provided slice. It tries to fit in as many blocks as possible based on +// number of bytes available in the provided buffer. It returns the number of +// bytes written to the provided buffer. +func EncodeSACKBlocks(sackBlocks []SACKBlock, b []byte) int { + if len(sackBlocks) == 0 { + return 0 + } + l := len(sackBlocks) + if l > TCPMaxSACKBlocks { + l = TCPMaxSACKBlocks + } + if ll := (len(b) - 2) / 8; ll < l { + l = ll + } + if l == 0 { + // There is not enough space in the provided buffer to add + // any SACK blocks. + return 0 + } + b[0] = TCPOptionSACK + b[1] = byte(l*8 + 2) + for i := 0; i < l; i++ { + binary.BigEndian.PutUint32(b[i*8+2:], uint32(sackBlocks[i].Start)) + binary.BigEndian.PutUint32(b[i*8+6:], uint32(sackBlocks[i].End)) + } + return int(b[1]) +} + +// EncodeNOP adds an explicit NOP to the option list. +func EncodeNOP(b []byte) int { + if len(b) == 0 { + return 0 + } + b[0] = TCPOptionNOP + return 1 +} + +// AddTCPOptionPadding adds the required number of TCPOptionNOP to quad align +// the option buffer. It adds padding bytes after the offset specified and +// returns the number of padding bytes added. The passed in options slice +// must have space for the padding bytes. +func AddTCPOptionPadding(options []byte, offset int) int { + paddingToAdd := -offset & 3 + // Now add any padding bytes that might be required to quad align the + // options. + for i := offset; i < offset+paddingToAdd; i++ { + options[i] = TCPOptionNOP + } + return paddingToAdd +} + +// Acceptable checks if a segment that starts at segSeq and has length segLen is +// "acceptable" for arriving in a receive window that starts at rcvNxt and ends +// before rcvAcc, according to the table on page 26 and 69 of RFC 793. +func Acceptable(segSeq seqnum.Value, segLen seqnum.Size, rcvNxt, rcvAcc seqnum.Value) bool { + if rcvNxt == rcvAcc { + return segLen == 0 && segSeq == rcvNxt + } + if segLen == 0 { + // rcvWnd is incremented by 1 because that is Linux's behavior despite the + // RFC. + return segSeq.InRange(rcvNxt, rcvAcc.Add(1)) + } + // Page 70 of RFC 793 allows packets that can be made "acceptable" by trimming + // the payload, so we'll accept any payload that overlaps the receive window. + // segSeq < rcvAcc is more correct according to RFC, however, Linux does it + // differently, it uses segSeq <= rcvAcc, we'd want to keep the same behavior + // as Linux. + return rcvNxt.LessThan(segSeq.Add(segLen)) && segSeq.LessThanEq(rcvAcc) +} + +// TCPValid returns true if the pkt has a valid TCP header. It checks whether: +// - The data offset is too small. +// - The data offset is too large. +// - The checksum is invalid. +// +// TCPValid corresponds to net/netfilter/nf_conntrack_proto_tcp.c:tcp_error. +func TCPValid(hdr TCP, payloadChecksum func() uint16, payloadSize uint16, srcAddr, dstAddr tcpip.Address, skipChecksumValidation bool) (csum uint16, csumValid, ok bool) { + if offset := int(hdr.DataOffset()); offset < TCPMinimumSize || offset > len(hdr) { + return + } + + if skipChecksumValidation { + csumValid = true + } else { + csum = hdr.Checksum() + csumValid = hdr.IsChecksumValid(srcAddr, dstAddr, payloadChecksum(), payloadSize) + } + return csum, csumValid, true +} diff --git a/pkg/tcpip/header/udp.go b/pkg/tcpip/header/udp.go new file mode 100644 index 0000000..2fc93a1 --- /dev/null +++ b/pkg/tcpip/header/udp.go @@ -0,0 +1,199 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import ( + "encoding/binary" + "math" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" +) + +const ( + udpSrcPort = 0 + udpDstPort = 2 + udpLength = 4 + udpChecksum = 6 +) + +const ( + // UDPMaximumPacketSize is the largest possible UDP packet. + UDPMaximumPacketSize = 0xffff +) + +// UDPFields contains the fields of a UDP packet. It is used to describe the +// fields of a packet that needs to be encoded. +type UDPFields struct { + // SrcPort is the "source port" field of a UDP packet. + SrcPort uint16 + + // DstPort is the "destination port" field of a UDP packet. + DstPort uint16 + + // Length is the "length" field of a UDP packet. + Length uint16 + + // Checksum is the "checksum" field of a UDP packet. + Checksum uint16 +} + +// UDP represents a UDP header stored in a byte array. +type UDP []byte + +const ( + // UDPMinimumSize is the minimum size of a valid UDP packet. + UDPMinimumSize = 8 + + // UDPMaximumSize is the maximum size of a valid UDP packet. The length field + // in the UDP header is 16 bits as per RFC 768. + UDPMaximumSize = math.MaxUint16 + + // UDPProtocolNumber is UDP's transport protocol number. + UDPProtocolNumber tcpip.TransportProtocolNumber = 17 +) + +// SourcePort returns the "source port" field of the UDP header. +func (b UDP) SourcePort() uint16 { + return binary.BigEndian.Uint16(b[udpSrcPort:]) +} + +// DestinationPort returns the "destination port" field of the UDP header. +func (b UDP) DestinationPort() uint16 { + return binary.BigEndian.Uint16(b[udpDstPort:]) +} + +// Length returns the "length" field of the UDP header. +func (b UDP) Length() uint16 { + return binary.BigEndian.Uint16(b[udpLength:]) +} + +// Payload returns the data contained in the UDP datagram. +func (b UDP) Payload() []byte { + return b[UDPMinimumSize:] +} + +// Checksum returns the "checksum" field of the UDP header. +func (b UDP) Checksum() uint16 { + return binary.BigEndian.Uint16(b[udpChecksum:]) +} + +// SetSourcePort sets the "source port" field of the UDP header. +func (b UDP) SetSourcePort(port uint16) { + binary.BigEndian.PutUint16(b[udpSrcPort:], port) +} + +// SetDestinationPort sets the "destination port" field of the UDP header. +func (b UDP) SetDestinationPort(port uint16) { + binary.BigEndian.PutUint16(b[udpDstPort:], port) +} + +// SetChecksum sets the "checksum" field of the UDP header. +func (b UDP) SetChecksum(xsum uint16) { + checksum.Put(b[udpChecksum:], xsum) +} + +// SetLength sets the "length" field of the UDP header. +func (b UDP) SetLength(length uint16) { + binary.BigEndian.PutUint16(b[udpLength:], length) +} + +// CalculateChecksum calculates the checksum of the UDP packet, given the +// checksum of the network-layer pseudo-header and the checksum of the payload. +func (b UDP) CalculateChecksum(partialChecksum uint16) uint16 { + // Calculate the rest of the checksum. + // return checksum.Checksum(b[:UDPMinimumSize], partialChecksum) + xsum := checksum.Checksum(b[:udpChecksum], partialChecksum) + xsum = checksum.Checksum(b[udpChecksum+2:UDPMinimumSize], xsum) + return xsum +} + +// IsChecksumValid returns true iff the UDP header's checksum is valid. +func (b UDP) IsChecksumValid(src, dst tcpip.Address, payloadChecksum uint16) bool { + xsum := PseudoHeaderChecksum(UDPProtocolNumber, dst, src, b.Length()) + xsum = checksum.Combine(xsum, payloadChecksum) + // return b.CalculateChecksum(xsum) == 0xffff + return checksum.Checksum(b[:UDPMinimumSize], xsum) == 0xffff +} + +// Encode encodes all the fields of the UDP header. +func (b UDP) Encode(u *UDPFields) { + b.SetSourcePort(u.SrcPort) + b.SetDestinationPort(u.DstPort) + b.SetLength(u.Length) + b.SetChecksum(u.Checksum) +} + +// SetSourcePortWithChecksumUpdate implements ChecksummableTransport. +func (b UDP) SetSourcePortWithChecksumUpdate(new uint16) { + old := b.SourcePort() + b.SetSourcePort(new) + b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) +} + +// SetDestinationPortWithChecksumUpdate implements ChecksummableTransport. +func (b UDP) SetDestinationPortWithChecksumUpdate(new uint16) { + old := b.DestinationPort() + b.SetDestinationPort(new) + b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) +} + +// UpdateChecksumPseudoHeaderAddress implements ChecksummableTransport. +func (b UDP) UpdateChecksumPseudoHeaderAddress(old, new tcpip.Address, fullChecksum bool) { + xsum := b.Checksum() + if fullChecksum { + xsum = ^xsum + } + + xsum = checksumUpdate2ByteAlignedAddress(xsum, old, new) + if fullChecksum { + xsum = ^xsum + } + + b.SetChecksum(xsum) +} + +// UDPValid returns true if the pkt has a valid UDP header. It checks whether: +// - The length field is too small. +// - The length field is too large. +// - The checksum is invalid. +// +// UDPValid corresponds to net/netfilter/nf_conntrack_proto_udp.c:udp_error. +func UDPValid(hdr UDP, payloadChecksum func() uint16, payloadSize uint16, netProto tcpip.NetworkProtocolNumber, srcAddr, dstAddr tcpip.Address, skipChecksumValidation bool) (lengthValid, csumValid bool) { + if length := hdr.Length(); length > payloadSize+UDPMinimumSize || length < UDPMinimumSize { + return false, false + } + + if skipChecksumValidation { + return true, true + } + + // On IPv4, UDP checksum is optional, and a zero value means the transmitter + // omitted the checksum generation, as per RFC 768: + // + // An all zero transmitted checksum value means that the transmitter + // generated no checksum (for debugging or for higher level protocols that + // don't care). + // + // On IPv6, UDP checksum is not optional, as per RFC 2460 Section 8.1: + // + // Unlike IPv4, when UDP packets are originated by an IPv6 node, the UDP + // checksum is not optional. + if netProto == IPv4ProtocolNumber && hdr.Checksum() == 0 { + return true, true + } + + return true, hdr.IsChecksumValid(srcAddr, dstAddr, payloadChecksum()) +} diff --git a/pkg/tcpip/header/virtionet.go b/pkg/tcpip/header/virtionet.go new file mode 100644 index 0000000..0958c68 --- /dev/null +++ b/pkg/tcpip/header/virtionet.go @@ -0,0 +1,94 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package header + +import "encoding/binary" + +// These constants are declared in linux/virtio_net.h. +const ( + _VIRTIO_NET_HDR_F_NEEDS_CSUM = 1 + _VIRTIO_NET_HDR_GSO_NONE = 0 + _VIRTIO_NET_HDR_GSO_TCPV4 = 1 + _VIRTIO_NET_HDR_GSO_TCPV6 = 4 +) + +const ( + // VirtioNetHeaderSize is the size of VirtioNetHeader in bytes. + VirtioNetHeaderSize = 10 +) + +// Offsets for fields in the virtio net header. +const ( + flags = 0 + gsoType = 1 + hdrLen = 2 + gsoSize = 4 + csumStart = 6 + csumOffset = 8 +) + +// VirtioNetHeaderFields is the Go equivalent of the struct declared in +// linux/virtio_net.h. +type VirtioNetHeaderFields struct { + Flags uint8 + GSOType uint8 + HdrLen uint16 + GSOSize uint16 + CSumStart uint16 + CSumOffset uint16 +} + +// VirtioNetHeader represents a virtio net header stored in a byte array. +type VirtioNetHeader []byte + +// Flags returns the "flags" field of the virtio net header. +func (v VirtioNetHeader) Flags() uint8 { + return uint8(v[flags]) +} + +// GSOType returns the "gsoType" field of the virtio net header. +func (v VirtioNetHeader) GSOType() uint8 { + return uint8(v[gsoType]) +} + +// HdrLen returns the "hdrLen" field of the virtio net header. +func (v VirtioNetHeader) HdrLen() uint16 { + return binary.BigEndian.Uint16(v[hdrLen:]) +} + +// GSOSize returns the "gsoSize" field of the virtio net header. +func (v VirtioNetHeader) GSOSize() uint16 { + return binary.BigEndian.Uint16(v[gsoSize:]) +} + +// CSumStart returns the "csumStart" field of the virtio net header. +func (v VirtioNetHeader) CSumStart() uint16 { + return binary.BigEndian.Uint16(v[csumStart:]) +} + +// CSumOffset returns the "csumOffset" field of the virtio net header. +func (v VirtioNetHeader) CSumOffset() uint16 { + return binary.BigEndian.Uint16(v[csumOffset:]) +} + +// Encode encodes all the fields of the virtio net header. +func (v VirtioNetHeader) Encode(f *VirtioNetHeaderFields) { + v[flags] = uint8(f.Flags) + v[gsoType] = uint8(f.GSOType) + binary.LittleEndian.PutUint16(v[hdrLen:], f.HdrLen) + binary.LittleEndian.PutUint16(v[gsoSize:], f.GSOSize) + binary.LittleEndian.PutUint16(v[csumStart:], f.CSumStart) + binary.LittleEndian.PutUint16(v[csumOffset:], f.CSumOffset) +} diff --git a/pkg/tcpip/internal/tcp/tcp.go b/pkg/tcpip/internal/tcp/tcp.go new file mode 100644 index 0000000..15124d2 --- /dev/null +++ b/pkg/tcpip/internal/tcp/tcp.go @@ -0,0 +1,48 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tcp contains internal type definitions that are not expected to be +// used by anyone else outside pkg/tcpip. +package tcp + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// TSOffset is an offset applied to the value of the TSVal field in the TCP +// Timestamp option. +// +// +stateify savable +type TSOffset struct { + milliseconds uint32 +} + +// NewTSOffset creates a new TSOffset from milliseconds. +func NewTSOffset(milliseconds uint32) TSOffset { + return TSOffset{ + milliseconds: milliseconds, + } +} + +// TSVal applies the offset to now and returns the timestamp in milliseconds. +func (offset TSOffset) TSVal(now tcpip.MonotonicTime) uint32 { + return uint32(now.Sub(tcpip.MonotonicTime{}).Milliseconds()) + offset.milliseconds +} + +// Elapsed calculates the elapsed time given now and the echoed back timestamp. +func (offset TSOffset) Elapsed(now tcpip.MonotonicTime, tsEcr uint32) time.Duration { + return time.Duration(offset.TSVal(now)-tsEcr) * time.Millisecond +} diff --git a/pkg/tcpip/internal/tcp/tcp_state_autogen.go b/pkg/tcpip/internal/tcp/tcp_state_autogen.go new file mode 100644 index 0000000..ab8f8a1 --- /dev/null +++ b/pkg/tcpip/internal/tcp/tcp_state_autogen.go @@ -0,0 +1,38 @@ +// automatically generated by stateify. + +package tcp + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (offset *TSOffset) StateTypeName() string { + return "pkg/tcpip/internal/tcp.TSOffset" +} + +func (offset *TSOffset) StateFields() []string { + return []string{ + "milliseconds", + } +} + +func (offset *TSOffset) beforeSave() {} + +// +checklocksignore +func (offset *TSOffset) StateSave(stateSinkObject state.Sink) { + offset.beforeSave() + stateSinkObject.Save(0, &offset.milliseconds) +} + +func (offset *TSOffset) afterLoad(context.Context) {} + +// +checklocksignore +func (offset *TSOffset) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &offset.milliseconds) +} + +func init() { + state.Register((*TSOffset)(nil)) +} diff --git a/pkg/tcpip/link/channel/channel.go b/pkg/tcpip/link/channel/channel.go new file mode 100644 index 0000000..6ab59cb --- /dev/null +++ b/pkg/tcpip/link/channel/channel.go @@ -0,0 +1,321 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package channel provides the implementation of channel-based data-link layer +// endpoints. Such endpoints allow injection of inbound packets and store +// outbound packets in a channel. +package channel + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// Notification is the interface for receiving notification from the packet +// queue. +type Notification interface { + // WriteNotify will be called when a write happens to the queue. + WriteNotify() +} + +// NotificationHandle is an opaque handle to the registered notification target. +// It can be used to unregister the notification when no longer interested. +// +// +stateify savable +type NotificationHandle struct { + n Notification +} + +type queue struct { + // c is the outbound packet channel. + c chan *stack.PacketBuffer + mu queueRWMutex + // +checklocks:mu + notify []*NotificationHandle + // +checklocks:mu + closed bool +} + +func (q *queue) Close() { + q.mu.Lock() + defer q.mu.Unlock() + if !q.closed { + close(q.c) + } + q.closed = true +} + +func (q *queue) Read() *stack.PacketBuffer { + select { + case p := <-q.c: + return p + default: + return nil + } +} + +func (q *queue) ReadContext(ctx context.Context) *stack.PacketBuffer { + select { + case pkt := <-q.c: + return pkt + case <-ctx.Done(): + return nil + } +} + +func (q *queue) Write(pkt *stack.PacketBuffer) tcpip.Error { + // q holds the PacketBuffer. + q.mu.RLock() + if q.closed { + q.mu.RUnlock() + return &tcpip.ErrClosedForSend{} + } + + wrote := false + p := pkt.Clone() + select { + case q.c <- p: + wrote = true + default: + p.DecRef() + } + notify := q.notify + q.mu.RUnlock() + + if wrote { + // Send notification outside of lock. + for _, h := range notify { + h.n.WriteNotify() + } + return nil + } + return &tcpip.ErrNoBufferSpace{} +} + +func (q *queue) Num() int { + return len(q.c) +} + +func (q *queue) AddNotify(notify Notification) *NotificationHandle { + q.mu.Lock() + defer q.mu.Unlock() + h := &NotificationHandle{n: notify} + q.notify = append(q.notify, h) + return h +} + +func (q *queue) RemoveNotify(handle *NotificationHandle) { + q.mu.Lock() + defer q.mu.Unlock() + // Make a copy, since we reads the array outside of lock when notifying. + notify := make([]*NotificationHandle, 0, len(q.notify)) + for _, h := range q.notify { + if h != handle { + notify = append(notify, h) + } + } + q.notify = notify +} + +var ( + _ stack.LinkEndpoint = (*Endpoint)(nil) + _ stack.GSOEndpoint = (*Endpoint)(nil) +) + +// Endpoint is link layer endpoint that stores outbound packets in a channel +// and allows injection of inbound packets. +// +// +stateify savable +type Endpoint struct { + LinkEPCapabilities stack.LinkEndpointCapabilities + SupportedGSOKind stack.SupportedGSO + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher + // +checklocks:mu + linkAddr tcpip.LinkAddress + // +checklocks:mu + mtu uint32 + + // Outbound packet queue. + q *queue +} + +// New creates a new channel endpoint. +func New(size int, mtu uint32, linkAddr tcpip.LinkAddress) *Endpoint { + return &Endpoint{ + q: &queue{ + c: make(chan *stack.PacketBuffer, size), + }, + mtu: mtu, + linkAddr: linkAddr, + } +} + +// Close closes e. Further packet injections will return an error, and all pending +// packets are discarded. Close may be called concurrently with WritePackets. +func (e *Endpoint) Close() { + e.q.Close() + e.Drain() +} + +// Read does non-blocking read one packet from the outbound packet queue. +func (e *Endpoint) Read() *stack.PacketBuffer { + return e.q.Read() +} + +// ReadContext does blocking read for one packet from the outbound packet queue. +// It can be cancelled by ctx, and in this case, it returns nil. +func (e *Endpoint) ReadContext(ctx context.Context) *stack.PacketBuffer { + return e.q.ReadContext(ctx) +} + +// Drain removes all outbound packets from the channel and counts them. +func (e *Endpoint) Drain() int { + c := 0 + for pkt := e.Read(); pkt != nil; pkt = e.Read() { + pkt.DecRef() + c++ + } + return c +} + +// NumQueued returns the number of packet queued for outbound. +func (e *Endpoint) NumQueued() int { + return e.q.Num() +} + +// InjectInbound injects an inbound packet. If the endpoint is not attached, the +// packet is not delivered. +func (e *Endpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverNetworkPacket(protocol, pkt) + } +} + +// Attach saves the stack network-layer dispatcher for use later when packets +// are injected. +func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + defer e.mu.Unlock() + e.dispatcher = dispatcher +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *Endpoint) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.dispatcher != nil +} + +// MTU implements stack.LinkEndpoint.MTU. +func (e *Endpoint) MTU() uint32 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mtu +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. +func (e *Endpoint) SetMTU(mtu uint32) { + e.mu.Lock() + defer e.mu.Unlock() + e.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities { + return e.LinkEPCapabilities +} + +// GSOMaxSize implements stack.GSOEndpoint. +func (*Endpoint) GSOMaxSize() uint32 { + return 1 << 15 +} + +// SupportedGSO implements stack.GSOEndpoint. +func (e *Endpoint) SupportedGSO() stack.SupportedGSO { + return e.SupportedGSOKind +} + +// MaxHeaderLength returns the maximum size of the link layer header. Given it +// doesn't have a header, it just returns 0. +func (*Endpoint) MaxHeaderLength() uint16 { + return 0 +} + +// LinkAddress returns the link address of this endpoint. +func (e *Endpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.linkAddr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.linkAddr = addr +} + +// WritePackets stores outbound packets into the channel. +// Multiple concurrent calls are permitted. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + n := 0 + for _, pkt := range pkts.AsSlice() { + if err := e.q.Write(pkt); err != nil { + if _, ok := err.(*tcpip.ErrNoBufferSpace); !ok && n == 0 { + return 0, err + } + break + } + n++ + } + + return n, nil +} + +// Wait implements stack.LinkEndpoint.Wait. +func (*Endpoint) Wait() {} + +// AddNotify adds a notification target for receiving event about outgoing +// packets. +func (e *Endpoint) AddNotify(notify Notification) *NotificationHandle { + return e.q.AddNotify(notify) +} + +// RemoveNotify removes handle from the list of notification targets. +func (e *Endpoint) RemoveNotify(handle *NotificationHandle) { + e.q.RemoveNotify(handle) +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (*Endpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareNone +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (*Endpoint) AddHeader(*stack.PacketBuffer) {} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (*Endpoint) ParseHeader(*stack.PacketBuffer) bool { return true } + +// SetOnCloseAction implements stack.LinkEndpoint. +func (*Endpoint) SetOnCloseAction(func()) {} diff --git a/pkg/tcpip/link/channel/channel_state_autogen.go b/pkg/tcpip/link/channel/channel_state_autogen.go new file mode 100644 index 0000000..15cec72 --- /dev/null +++ b/pkg/tcpip/link/channel/channel_state_autogen.go @@ -0,0 +1,79 @@ +// automatically generated by stateify. + +package channel + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (n *NotificationHandle) StateTypeName() string { + return "pkg/tcpip/link/channel.NotificationHandle" +} + +func (n *NotificationHandle) StateFields() []string { + return []string{ + "n", + } +} + +func (n *NotificationHandle) beforeSave() {} + +// +checklocksignore +func (n *NotificationHandle) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.n) +} + +func (n *NotificationHandle) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NotificationHandle) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.n) +} + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/channel.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "LinkEPCapabilities", + "SupportedGSOKind", + "dispatcher", + "linkAddr", + "mtu", + "q", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.LinkEPCapabilities) + stateSinkObject.Save(1, &e.SupportedGSOKind) + stateSinkObject.Save(2, &e.dispatcher) + stateSinkObject.Save(3, &e.linkAddr) + stateSinkObject.Save(4, &e.mtu) + stateSinkObject.Save(5, &e.q) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.LinkEPCapabilities) + stateSourceObject.Load(1, &e.SupportedGSOKind) + stateSourceObject.Load(2, &e.dispatcher) + stateSourceObject.Load(3, &e.linkAddr) + stateSourceObject.Load(4, &e.mtu) + stateSourceObject.Load(5, &e.q) +} + +func init() { + state.Register((*NotificationHandle)(nil)) + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/channel/endpoint_mutex.go b/pkg/tcpip/link/channel/endpoint_mutex.go new file mode 100644 index 0000000..8c834d2 --- /dev/null +++ b/pkg/tcpip/link/channel/endpoint_mutex.go @@ -0,0 +1,96 @@ +package channel + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/channel/queue_mutex.go b/pkg/tcpip/link/channel/queue_mutex.go new file mode 100644 index 0000000..c7f2f8f --- /dev/null +++ b/pkg/tcpip/link/channel/queue_mutex.go @@ -0,0 +1,96 @@ +package channel + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type queueRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var queuelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type queuelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *queueRWMutex) Lock() { + locking.AddGLock(queueprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *queueRWMutex) NestedLock(i queuelockNameIndex) { + locking.AddGLock(queueprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *queueRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(queueprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *queueRWMutex) NestedUnlock(i queuelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(queueprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *queueRWMutex) RLock() { + locking.AddGLock(queueprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *queueRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(queueprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *queueRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *queueRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *queueRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var queueprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func queueinitLockNames() {} + +func init() { + queueinitLockNames() + queueprefixIndex = locking.NewMutexClass(reflect.TypeOf(queueRWMutex{}), queuelockNames) +} diff --git a/pkg/tcpip/link/ethernet/ethernet.go b/pkg/tcpip/link/ethernet/ethernet.go new file mode 100644 index 0000000..80f3cdb --- /dev/null +++ b/pkg/tcpip/link/ethernet/ethernet.go @@ -0,0 +1,121 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ethernet provides an implementation of an ethernet link endpoint that +// wraps an inner link endpoint. +package ethernet + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/link/nested" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var ( + _ stack.NetworkDispatcher = (*Endpoint)(nil) + _ stack.LinkEndpoint = (*Endpoint)(nil) +) + +// New returns an ethernet link endpoint that wraps an inner link endpoint. +func New(ep stack.LinkEndpoint) *Endpoint { + var e Endpoint + e.Endpoint.Init(ep, &e) + return &e +} + +// Endpoint is an ethernet endpoint. +// +// It adds an ethernet header to packets before sending them out through its +// inner link endpoint and consumes an ethernet header before sending the +// packet to the stack. +// +// +stateify savable +type Endpoint struct { + nested.Endpoint +} + +// LinkAddress implements stack.LinkEndpoint. +func (e *Endpoint) LinkAddress() tcpip.LinkAddress { + if l := e.Endpoint.LinkAddress(); len(l) != 0 { + return l + } + return header.UnspecifiedEthernetAddress +} + +// MTU implements stack.LinkEndpoint. +func (e *Endpoint) MTU() uint32 { + return e.Endpoint.MTU() +} + +// DeliverNetworkPacket implements stack.NetworkDispatcher. +func (e *Endpoint) DeliverNetworkPacket(_ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + if !e.ParseHeader(pkt) { + return + } + eth := header.Ethernet(pkt.LinkHeader().Slice()) + dst := eth.DestinationAddress() + if dst == header.EthernetBroadcastAddress { + pkt.PktType = tcpip.PacketBroadcast + } else if header.IsMulticastEthernetAddress(dst) { + pkt.PktType = tcpip.PacketMulticast + } else if dst == e.LinkAddress() { + pkt.PktType = tcpip.PacketHost + } else { + pkt.PktType = tcpip.PacketOtherHost + } + + // Note, there is no need to check the destination link address here since + // the ethernet hardware filters frames based on their destination addresses. + e.Endpoint.DeliverNetworkPacket(eth.Type() /* protocol */, pkt) +} + +// Capabilities implements stack.LinkEndpoint. +func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities { + c := e.Endpoint.Capabilities() + if c&stack.CapabilityLoopback == 0 { + c |= stack.CapabilityResolutionRequired + } + return c +} + +// MaxHeaderLength implements stack.LinkEndpoint. +func (e *Endpoint) MaxHeaderLength() uint16 { + return header.EthernetMinimumSize + e.Endpoint.MaxHeaderLength() +} + +// ARPHardwareType implements stack.LinkEndpoint. +func (e *Endpoint) ARPHardwareType() header.ARPHardwareType { + if a := e.Endpoint.ARPHardwareType(); a != header.ARPHardwareNone { + return a + } + return header.ARPHardwareEther +} + +// AddHeader implements stack.LinkEndpoint. +func (*Endpoint) AddHeader(pkt *stack.PacketBuffer) { + eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize)) + fields := header.EthernetFields{ + SrcAddr: pkt.EgressRoute.LocalLinkAddress, + DstAddr: pkt.EgressRoute.RemoteLinkAddress, + Type: pkt.NetworkProtocolNumber, + } + eth.Encode(&fields) +} + +// ParseHeader implements stack.LinkEndpoint. +func (*Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + _, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize) + return ok +} diff --git a/pkg/tcpip/link/ethernet/ethernet_state_autogen.go b/pkg/tcpip/link/ethernet/ethernet_state_autogen.go new file mode 100644 index 0000000..69a65d9 --- /dev/null +++ b/pkg/tcpip/link/ethernet/ethernet_state_autogen.go @@ -0,0 +1,38 @@ +// automatically generated by stateify. + +package ethernet + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/ethernet.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "Endpoint", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.Endpoint) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.Endpoint) +} + +func init() { + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/fdbased/endpoint.go b/pkg/tcpip/link/fdbased/endpoint.go new file mode 100644 index 0000000..1561ca7 --- /dev/null +++ b/pkg/tcpip/link/fdbased/endpoint.go @@ -0,0 +1,906 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package fdbased provides the implementation of data-link layer endpoints +// backed by boundary-preserving file descriptors (e.g., TUN devices, +// seqpacket/datagram sockets). +// +// FD based endpoints can be used in the networking stack by calling New() to +// create a new endpoint, and then passing it as an argument to +// Stack.CreateNIC(). +// +// FD based endpoints can use more than one file descriptor to read incoming +// packets. If there are more than one FDs specified and the underlying FD is an +// AF_PACKET then the endpoint will enable FANOUT mode on the socket so that the +// host kernel will consistently hash the packets to the sockets. This ensures +// that packets for the same TCP streams are not reordered. +// +// Similarly if more than one FD's are specified where the underlying FD is not +// AF_PACKET then it's the caller's responsibility to ensure that all inbound +// packets on the descriptors are consistently 5 tuple hashed to one of the +// descriptors to prevent TCP reordering. +// +// Since netstack today does not compute 5 tuple hashes for outgoing packets we +// only use the first FD to write outbound packets. Once 5 tuple hashes for +// all outbound packets are available we will make use of all underlying FD's to +// write outbound packets. +package fdbased + +import ( + "fmt" + "runtime" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "golang.org/x/sys/unix" +) + +// linkDispatcher reads packets from the link FD and dispatches them to the +// NetworkDispatcher. +type linkDispatcher interface { + Stop() + dispatch() (bool, tcpip.Error) + release() +} + +// PacketDispatchMode are the various supported methods of receiving and +// dispatching packets from the underlying FD. +type PacketDispatchMode int + +// BatchSize is the number of packets to write in each syscall. It is 47 +// because when GVisorGSO is in use then a single 65KB TCP segment can get +// split into 46 segments of 1420 bytes and a single 216 byte segment. +const BatchSize = 47 + +const ( + // Readv is the default dispatch mode and is the least performant of the + // dispatch options but the one that is supported by all underlying FD + // types. + Readv PacketDispatchMode = iota + // RecvMMsg enables use of recvmmsg() syscall instead of readv() to + // read inbound packets. This reduces # of syscalls needed to process + // packets. + // + // NOTE: recvmmsg() is only supported for sockets, so if the underlying + // FD is not a socket then the code will still fall back to the readv() + // path. + RecvMMsg + // PacketMMap enables use of PACKET_RX_RING to receive packets from the + // NIC. PacketMMap requires that the underlying FD be an AF_PACKET. The + // primary use-case for this is runsc which uses an AF_PACKET FD to + // receive packets from the veth device. + PacketMMap +) + +func (p PacketDispatchMode) String() string { + switch p { + case Readv: + return "Readv" + case RecvMMsg: + return "RecvMMsg" + case PacketMMap: + return "PacketMMap" + default: + return fmt.Sprintf("unknown packet dispatch mode '%d'", p) + } +} + +var ( + _ stack.LinkEndpoint = (*endpoint)(nil) + _ stack.GSOEndpoint = (*endpoint)(nil) +) + +// +stateify savable +type fdInfo struct { + fd int + isSocket bool +} + +// +stateify savable +type endpoint struct { + // fds is the set of file descriptors each identifying one inbound/outbound + // channel. The endpoint will dispatch from all inbound channels as well as + // hash outbound packets to specific channels based on the packet hash. + fds []fdInfo + + // hdrSize specifies the link-layer header size. If set to 0, no header + // is added/removed; otherwise an ethernet header is used. + hdrSize int + + // caps holds the endpoint capabilities. + caps stack.LinkEndpointCapabilities + + // closed is a function to be called when the FD's peer (if any) closes + // its end of the communication pipe. + closed func(tcpip.Error) `state:"nosave"` + + inboundDispatchers []linkDispatcher + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher + + // packetDispatchMode controls the packet dispatcher used by this + // endpoint. + packetDispatchMode PacketDispatchMode + + // gsoMaxSize is the maximum GSO packet size. It is zero if GSO is + // disabled. + gsoMaxSize uint32 + + // wg keeps track of running goroutines. + wg sync.WaitGroup `state:"nosave"` + + // gsoKind is the supported kind of GSO. + gsoKind stack.SupportedGSO + + // maxSyscallHeaderBytes has the same meaning as + // Options.MaxSyscallHeaderBytes. + maxSyscallHeaderBytes uintptr + + // writevMaxIovs is the maximum number of iovecs that may be passed to + // rawfile.NonBlockingWriteIovec, as possibly limited by + // maxSyscallHeaderBytes. (No analogous limit is defined for + // rawfile.NonBlockingSendMMsg, since in that case the maximum number of + // iovecs also depends on the number of mmsghdrs. Instead, if sendBatch + // encounters a packet whose iovec count is limited by + // maxSyscallHeaderBytes, it falls back to writing the packet using writev + // via WritePacket.) + writevMaxIovs int + + // addr is the address of the endpoint. + // + // +checklocks:mu + addr tcpip.LinkAddress + + // mtu (maximum transmission unit) is the maximum size of a packet. + // +checklocks:mu + mtu uint32 +} + +// Options specify the details about the fd-based endpoint to be created. +// +// +stateify savable +type Options struct { + // FDs is a set of FDs used to read/write packets. + FDs []int + + // MTU is the mtu to use for this endpoint. + MTU uint32 + + // EthernetHeader if true, indicates that the endpoint should read/write + // ethernet frames instead of IP packets. + EthernetHeader bool + + // ClosedFunc is a function to be called when an endpoint's peer (if + // any) closes its end of the communication pipe. + ClosedFunc func(tcpip.Error) + + // Address is the link address for this endpoint. Only used if + // EthernetHeader is true. + Address tcpip.LinkAddress + + // SaveRestore if true, indicates that this NIC capability set should + // include CapabilitySaveRestore + SaveRestore bool + + // DisconnectOk if true, indicates that this NIC capability set should + // include CapabilityDisconnectOk. + DisconnectOk bool + + // GSOMaxSize is the maximum GSO packet size. It is zero if GSO is + // disabled. + GSOMaxSize uint32 + + // GVisorGSOEnabled indicates whether Gvisor GSO is enabled or not. + GVisorGSOEnabled bool + + // PacketDispatchMode specifies the type of inbound dispatcher to be + // used for this endpoint. + PacketDispatchMode PacketDispatchMode + + // TXChecksumOffload if true, indicates that this endpoints capability + // set should include CapabilityTXChecksumOffload. + TXChecksumOffload bool + + // RXChecksumOffload if true, indicates that this endpoints capability + // set should include CapabilityRXChecksumOffload. + RXChecksumOffload bool + + // If MaxSyscallHeaderBytes is non-zero, it is the maximum number of bytes + // of struct iovec, msghdr, and mmsghdr that may be passed by each host + // system call. + MaxSyscallHeaderBytes int + + // InterfaceIndex is the interface index of the underlying device. + InterfaceIndex int + + // GRO enables generic receive offload. + GRO bool + + // ProcessorsPerChannel is the number of goroutines used to handle packets + // from each FD. + ProcessorsPerChannel int +} + +// fanoutID is used for AF_PACKET based endpoints to enable PACKET_FANOUT +// support in the host kernel. This allows us to use multiple FD's to receive +// from the same underlying NIC. The fanoutID needs to be the same for a given +// set of FD's that point to the same NIC. Trying to set the PACKET_FANOUT +// option for an FD with a fanoutID already in use by another FD for a different +// NIC will return an EINVAL. +// +// Since fanoutID must be unique within the network namespace, we start with +// the PID to avoid collisions. The only way to be sure of avoiding collisions +// is to run in a new network namespace. +var fanoutID atomicbitops.Int32 = atomicbitops.FromInt32(int32(unix.Getpid())) + +// New creates a new fd-based endpoint. +// +// Makes fd non-blocking, but does not take ownership of fd, which must remain +// open for the lifetime of the returned endpoint (until after the endpoint has +// stopped being using and Wait returns). +func New(opts *Options) (stack.LinkEndpoint, error) { + caps := stack.LinkEndpointCapabilities(0) + if opts.RXChecksumOffload { + caps |= stack.CapabilityRXChecksumOffload + } + + if opts.TXChecksumOffload { + caps |= stack.CapabilityTXChecksumOffload + } + + hdrSize := 0 + if opts.EthernetHeader { + hdrSize = header.EthernetMinimumSize + caps |= stack.CapabilityResolutionRequired + } + + if opts.SaveRestore { + caps |= stack.CapabilitySaveRestore + } + + if opts.DisconnectOk { + caps |= stack.CapabilityDisconnectOk + } + + if len(opts.FDs) == 0 { + return nil, fmt.Errorf("opts.FD is empty, at least one FD must be specified") + } + + if opts.MaxSyscallHeaderBytes < 0 { + return nil, fmt.Errorf("opts.MaxSyscallHeaderBytes is negative") + } + + e := &endpoint{ + mtu: opts.MTU, + caps: caps, + closed: opts.ClosedFunc, + addr: opts.Address, + hdrSize: hdrSize, + packetDispatchMode: opts.PacketDispatchMode, + maxSyscallHeaderBytes: uintptr(opts.MaxSyscallHeaderBytes), + writevMaxIovs: rawfile.MaxIovs, + } + if e.maxSyscallHeaderBytes != 0 { + if max := int(e.maxSyscallHeaderBytes / rawfile.SizeofIovec); max < e.writevMaxIovs { + e.writevMaxIovs = max + } + } + + // Increment fanoutID to ensure that we don't re-use the same fanoutID + // for the next endpoint. + fid := fanoutID.Add(1) + + // Create per channel dispatchers. + for _, fd := range opts.FDs { + if err := unix.SetNonblock(fd, true); err != nil { + return nil, fmt.Errorf("unix.SetNonblock(%v) failed: %v", fd, err) + } + + isSocket, err := isSocketFD(fd) + if err != nil { + return nil, err + } + e.fds = append(e.fds, fdInfo{fd: fd, isSocket: isSocket}) + if opts.GSOMaxSize != 0 { + if opts.GVisorGSOEnabled { + e.gsoKind = stack.GVisorGSOSupported + } else { + e.gsoKind = stack.HostGSOSupported + } + e.gsoMaxSize = opts.GSOMaxSize + } + if opts.ProcessorsPerChannel == 0 { + opts.ProcessorsPerChannel = max(1, runtime.GOMAXPROCS(0)/len(opts.FDs)) + } + + inboundDispatcher, err := createInboundDispatcher(e, fd, isSocket, fid, opts) + if err != nil { + return nil, fmt.Errorf("createInboundDispatcher(...) = %v", err) + } + e.inboundDispatchers = append(e.inboundDispatchers, inboundDispatcher) + } + + return e, nil +} + +func createInboundDispatcher(e *endpoint, fd int, isSocket bool, fID int32, opts *Options) (linkDispatcher, error) { + // By default use the readv() dispatcher as it works with all kinds of + // FDs (tap/tun/unix domain sockets and af_packet). + inboundDispatcher, err := newReadVDispatcher(fd, e, opts) + if err != nil { + return nil, fmt.Errorf("newReadVDispatcher(%d, %+v) = %v", fd, e, err) + } + + if isSocket { + sa, err := unix.Getsockname(fd) + if err != nil { + return nil, fmt.Errorf("unix.Getsockname(%d) = %v", fd, err) + } + switch sa.(type) { + case *unix.SockaddrLinklayer: + // Enable PACKET_FANOUT mode if the underlying socket is of type + // AF_PACKET. We do not enable PACKET_FANOUT_FLAG_DEFRAG as that will + // prevent gvisor from receiving fragmented packets and the host does the + // reassembly on our behalf before delivering the fragments. This makes it + // hard to test fragmentation reassembly code in Netstack. + // + // See: include/uapi/linux/if_packet.h (struct fanout_args). + // + // NOTE: We are using SetSockOptInt here even though the underlying + // option is actually a struct. The code follows the example in the + // kernel documentation as described at the link below: + // + // See: https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt + // + // This works out because the actual implementation for the option zero + // initializes the structure and will initialize the max_members field + // to a proper value if zero. + // + // See: https://github.com/torvalds/linux/blob/7acac4b3196caee5e21fb5ea53f8bc124e6a16fc/net/packet/af_packet.c#L3881 + const fanoutType = unix.PACKET_FANOUT_HASH + fanoutArg := (int(fID) & 0xffff) | fanoutType<<16 + if err := unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT, fanoutArg); err != nil { + return nil, fmt.Errorf("failed to enable PACKET_FANOUT option: %v", err) + } + } + + switch e.packetDispatchMode { + case PacketMMap: + inboundDispatcher, err = newPacketMMapDispatcher(fd, e, opts) + if err != nil { + return nil, fmt.Errorf("newPacketMMapDispatcher(%d, %+v) = %v", fd, e, err) + } + case RecvMMsg: + // If the provided FD is a socket then we optimize + // packet reads by using recvmmsg() instead of read() to + // read packets in a batch. + inboundDispatcher, err = newRecvMMsgDispatcher(fd, e, opts) + if err != nil { + return nil, fmt.Errorf("newRecvMMsgDispatcher(%d, %+v) = %v", fd, e, err) + } + case Readv: + default: + return nil, fmt.Errorf("unknown dispatch mode %d", e.packetDispatchMode) + } + } + return inboundDispatcher, nil +} + +func isSocketFD(fd int) (bool, error) { + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return false, fmt.Errorf("unix.Fstat(%v,...) failed: %v", fd, err) + } + return (stat.Mode & unix.S_IFSOCK) == unix.S_IFSOCK, nil +} + +// Attach launches the goroutine that reads packets from the file descriptor and +// dispatches them via the provided dispatcher. If one is already attached, +// then nothing happens. +// +// Attach implements stack.LinkEndpoint.Attach. +func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + + // nil means the NIC is being removed. + if dispatcher == nil && e.dispatcher != nil { + for _, dispatcher := range e.inboundDispatchers { + dispatcher.Stop() + } + e.dispatcher = nil + // NOTE(gvisor.dev/issue/11456): Unlock e.mu before e.Wait(). + e.mu.Unlock() + e.Wait() + return + } + defer e.mu.Unlock() + if dispatcher != nil && e.dispatcher == nil { + e.dispatcher = dispatcher + // Link endpoints are not savable. When transportation endpoints are + // saved, they stop sending outgoing packets and all incoming packets + // are rejected. + for i := range e.inboundDispatchers { + e.wg.Add(1) + go func(i int) { // S/R-SAFE: See above. + e.dispatchLoop(e.inboundDispatchers[i]) + e.wg.Done() + }(i) + } + } +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *endpoint) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.dispatcher != nil +} + +// MTU implements stack.LinkEndpoint.MTU. +func (e *endpoint) MTU() uint32 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mtu +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. +func (e *endpoint) SetMTU(mtu uint32) { + e.mu.Lock() + defer e.mu.Unlock() + e.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (e *endpoint) Capabilities() stack.LinkEndpointCapabilities { + return e.caps +} + +// MaxHeaderLength returns the maximum size of the link-layer header. +func (e *endpoint) MaxHeaderLength() uint16 { + return uint16(e.hdrSize) +} + +// LinkAddress returns the link address of this endpoint. +func (e *endpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.addr = addr +} + +// Wait implements stack.LinkEndpoint.Wait. It waits for the endpoint to stop +// reading from its FD. +func (e *endpoint) Wait() { + e.wg.Wait() +} + +// virtioNetHdr is declared in linux/virtio_net.h. +type virtioNetHdr struct { + flags uint8 + gsoType uint8 + hdrLen uint16 + gsoSize uint16 + csumStart uint16 + csumOffset uint16 +} + +// marshal serializes h to a newly-allocated byte slice, in little-endian byte +// order. +// +// Note: Virtio v1.0 onwards specifies little-endian as the byte ordering used +// for general serialization. This makes it difficult to use go-marshal for +// virtio types, as go-marshal implicitly uses the native byte ordering. +func (h *virtioNetHdr) marshal() []byte { + buf := [virtioNetHdrSize]byte{ + 0: byte(h.flags), + 1: byte(h.gsoType), + + // Manually lay out the fields in little-endian byte order. Little endian => + // least significant bit goes to the lower address. + + 2: byte(h.hdrLen), + 3: byte(h.hdrLen >> 8), + + 4: byte(h.gsoSize), + 5: byte(h.gsoSize >> 8), + + 6: byte(h.csumStart), + 7: byte(h.csumStart >> 8), + + 8: byte(h.csumOffset), + 9: byte(h.csumOffset >> 8), + } + return buf[:] +} + +// These constants are declared in linux/virtio_net.h. +const ( + _VIRTIO_NET_HDR_F_NEEDS_CSUM = 1 + + _VIRTIO_NET_HDR_GSO_TCPV4 = 1 + _VIRTIO_NET_HDR_GSO_TCPV6 = 4 +) + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *endpoint) AddHeader(pkt *stack.PacketBuffer) { + if e.hdrSize > 0 { + // Add ethernet header if needed. + eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize)) + eth.Encode(&header.EthernetFields{ + SrcAddr: pkt.EgressRoute.LocalLinkAddress, + DstAddr: pkt.EgressRoute.RemoteLinkAddress, + Type: pkt.NetworkProtocolNumber, + }) + } +} + +func (e *endpoint) parseHeader(pkt *stack.PacketBuffer) (header.Ethernet, bool) { + if e.hdrSize <= 0 { + return nil, true + } + hdrBytes, ok := pkt.LinkHeader().Consume(e.hdrSize) + if !ok { + return nil, false + } + hdr := header.Ethernet(hdrBytes) + pkt.NetworkProtocolNumber = hdr.Type() + return hdr, true +} + +// parseInboundHeader parses the link header of pkt and returns true if the +// header is well-formed and sent to this endpoint's MAC or the broadcast +// address. +func (e *endpoint) parseInboundHeader(pkt *stack.PacketBuffer, wantAddr tcpip.LinkAddress) bool { + hdr, ok := e.parseHeader(pkt) + if !ok || e.hdrSize <= 0 { + return ok + } + dstAddr := hdr.DestinationAddress() + // Per RFC 9542 2.1 on the least significant bit of the first octet of + // a MAC address: "If it is zero, the MAC address is unicast. If it is + // a one, the address is groupcast (multicast or broadcast)." Multicast + // and broadcast are the same thing to ethernet; they are both sent to + // everyone. + return dstAddr == wantAddr || byte(dstAddr[0])&0x01 == 1 +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (e *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + _, ok := e.parseHeader(pkt) + return ok +} + +// writePacket writes outbound packets to the file descriptor. If it is not +// currently writable, the packet is dropped. +func (e *endpoint) writePacket(pkt *stack.PacketBuffer) tcpip.Error { + fdInfo := e.fds[pkt.Hash%uint32(len(e.fds))] + fd := fdInfo.fd + var vnetHdrBuf []byte + if e.gsoKind == stack.HostGSOSupported { + vnetHdr := virtioNetHdr{} + if pkt.GSOOptions.Type != stack.GSONone { + vnetHdr.hdrLen = uint16(pkt.HeaderSize()) + if pkt.GSOOptions.NeedsCsum { + vnetHdr.flags = _VIRTIO_NET_HDR_F_NEEDS_CSUM + vnetHdr.csumStart = pkt.GSOOptions.L3HdrLen + vnetHdr.csumOffset = pkt.GSOOptions.CsumOffset + } + if uint16(pkt.Data().Size()) > pkt.GSOOptions.MSS { + switch pkt.GSOOptions.Type { + case stack.GSOTCPv4: + vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV4 + case stack.GSOTCPv6: + vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV6 + default: + panic(fmt.Sprintf("Unknown gso type: %v", pkt.GSOOptions.Type)) + } + vnetHdr.gsoSize = pkt.GSOOptions.MSS + } + } + vnetHdrBuf = vnetHdr.marshal() + } + + views := pkt.AsSlices() + numIovecs := len(views) + if len(vnetHdrBuf) != 0 { + numIovecs++ + } + if numIovecs > e.writevMaxIovs { + numIovecs = e.writevMaxIovs + } + + // Allocate small iovec arrays on the stack. + var iovecsArr [8]unix.Iovec + iovecs := iovecsArr[:0] + if numIovecs > len(iovecsArr) { + iovecs = make([]unix.Iovec, 0, numIovecs) + } + iovecs = rawfile.AppendIovecFromBytes(iovecs, vnetHdrBuf, numIovecs) + for _, v := range views { + iovecs = rawfile.AppendIovecFromBytes(iovecs, v, numIovecs) + } + if errno := rawfile.NonBlockingWriteIovec(fd, iovecs); errno != 0 { + return tcpip.TranslateErrno(errno) + } + return nil +} + +func (e *endpoint) sendBatch(batchFDInfo fdInfo, pkts []*stack.PacketBuffer) (int, tcpip.Error) { + // Degrade to writePacket if underlying fd is not a socket. + if !batchFDInfo.isSocket { + var written int + var err tcpip.Error + for written < len(pkts) { + if err = e.writePacket(pkts[written]); err != nil { + break + } + written++ + } + return written, err + } + + // Send a batch of packets through batchFD. + batchFD := batchFDInfo.fd + mmsgHdrsStorage := make([]rawfile.MMsgHdr, 0, len(pkts)) + packets := 0 + for packets < len(pkts) { + mmsgHdrs := mmsgHdrsStorage + batch := pkts[packets:] + syscallHeaderBytes := uintptr(0) + for _, pkt := range batch { + var vnetHdrBuf []byte + if e.gsoKind == stack.HostGSOSupported { + vnetHdr := virtioNetHdr{} + if pkt.GSOOptions.Type != stack.GSONone { + vnetHdr.hdrLen = uint16(pkt.HeaderSize()) + if pkt.GSOOptions.NeedsCsum { + vnetHdr.flags = _VIRTIO_NET_HDR_F_NEEDS_CSUM + vnetHdr.csumStart = pkt.GSOOptions.L3HdrLen + vnetHdr.csumOffset = pkt.GSOOptions.CsumOffset + } + if pkt.GSOOptions.Type != stack.GSONone && uint16(pkt.Data().Size()) > pkt.GSOOptions.MSS { + switch pkt.GSOOptions.Type { + case stack.GSOTCPv4: + vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV4 + case stack.GSOTCPv6: + vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV6 + default: + panic(fmt.Sprintf("Unknown gso type: %v", pkt.GSOOptions.Type)) + } + vnetHdr.gsoSize = pkt.GSOOptions.MSS + } + } + vnetHdrBuf = vnetHdr.marshal() + } + + views, offset := pkt.AsViewList() + var skipped int + var view *buffer.View + for view = views.Front(); view != nil && offset >= view.Size(); view = view.Next() { + offset -= view.Size() + skipped++ + } + + // We've made it to the usable views. + numIovecs := views.Len() - skipped + if len(vnetHdrBuf) != 0 { + numIovecs++ + } + if numIovecs > rawfile.MaxIovs { + numIovecs = rawfile.MaxIovs + } + if e.maxSyscallHeaderBytes != 0 { + syscallHeaderBytes += rawfile.SizeofMMsgHdr + uintptr(numIovecs)*rawfile.SizeofIovec + if syscallHeaderBytes > e.maxSyscallHeaderBytes { + // We can't fit this packet into this call to sendmmsg(). + // We could potentially do so if we reduced numIovecs + // further, but this might incur considerable extra + // copying. Leave it to the next batch instead. + break + } + } + + // We can't easily allocate iovec arrays on the stack here since + // they will escape this loop iteration via mmsgHdrs. + iovecs := make([]unix.Iovec, 0, numIovecs) + iovecs = rawfile.AppendIovecFromBytes(iovecs, vnetHdrBuf, numIovecs) + // At most one slice has a non-zero offset. + iovecs = rawfile.AppendIovecFromBytes(iovecs, view.AsSlice()[offset:], numIovecs) + for view = view.Next(); view != nil; view = view.Next() { + iovecs = rawfile.AppendIovecFromBytes(iovecs, view.AsSlice(), numIovecs) + } + + var mmsgHdr rawfile.MMsgHdr + mmsgHdr.Msg.Iov = &iovecs[0] + mmsgHdr.Msg.SetIovlen(len(iovecs)) + mmsgHdrs = append(mmsgHdrs, mmsgHdr) + } + + if len(mmsgHdrs) == 0 { + // We can't fit batch[0] into a mmsghdr while staying under + // e.maxSyscallHeaderBytes. Use WritePacket, which will avoid the + // mmsghdr (by using writev) and re-buffer iovecs more aggressively + // if necessary (by using e.writevMaxIovs instead of + // rawfile.MaxIovs). + pkt := batch[0] + if err := e.writePacket(pkt); err != nil { + return packets, err + } + packets++ + } else { + for len(mmsgHdrs) > 0 { + sent, errno := rawfile.NonBlockingSendMMsg(batchFD, mmsgHdrs) + if errno != 0 { + return packets, tcpip.TranslateErrno(errno) + } + packets += sent + mmsgHdrs = mmsgHdrs[sent:] + } + } + } + + return packets, nil +} + +// WritePackets writes outbound packets to the underlying file descriptors. If +// one is not currently writable, the packet is dropped. +// +// Being a batch API, each packet in pkts should have the following +// fields populated: +// - pkt.EgressRoute +// - pkt.GSOOptions +// - pkt.NetworkProtocolNumber +func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + // Preallocate to avoid repeated reallocation as we append to batch. + batch := make([]*stack.PacketBuffer, 0, BatchSize) + batchFDInfo := fdInfo{fd: -1, isSocket: false} + sentPackets := 0 + for _, pkt := range pkts.AsSlice() { + if len(batch) == 0 { + batchFDInfo = e.fds[pkt.Hash%uint32(len(e.fds))] + } + pktFDInfo := e.fds[pkt.Hash%uint32(len(e.fds))] + if sendNow := pktFDInfo != batchFDInfo; !sendNow { + batch = append(batch, pkt) + continue + } + n, err := e.sendBatch(batchFDInfo, batch) + sentPackets += n + if err != nil { + return sentPackets, err + } + batch = batch[:0] + batch = append(batch, pkt) + batchFDInfo = pktFDInfo + } + + if len(batch) != 0 { + n, err := e.sendBatch(batchFDInfo, batch) + sentPackets += n + if err != nil { + return sentPackets, err + } + } + return sentPackets, nil +} + +// InjectOutbound implements stack.InjectableEndpoint.InjectOutbound. +func (e *endpoint) InjectOutbound(dest tcpip.Address, packet *buffer.View) tcpip.Error { + if errno := rawfile.NonBlockingWrite(e.fds[0].fd, packet.AsSlice()); errno != 0 { + return tcpip.TranslateErrno(errno) + } + return nil +} + +// dispatchLoop reads packets from the file descriptor in a loop and dispatches +// them to the network stack. +func (e *endpoint) dispatchLoop(inboundDispatcher linkDispatcher) tcpip.Error { + for { + cont, err := inboundDispatcher.dispatch() + if err != nil || !cont { + if e.closed != nil { + e.closed(err) + } + inboundDispatcher.release() + return err + } + } +} + +// GSOMaxSize implements stack.GSOEndpoint. +func (e *endpoint) GSOMaxSize() uint32 { + return e.gsoMaxSize +} + +// SupportedGSO implements stack.GSOEndpoint. +func (e *endpoint) SupportedGSO() stack.SupportedGSO { + return e.gsoKind +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (e *endpoint) ARPHardwareType() header.ARPHardwareType { + if e.hdrSize > 0 { + return header.ARPHardwareEther + } + return header.ARPHardwareNone +} + +// Close implements stack.LinkEndpoint. +func (e *endpoint) Close() {} + +// SetOnCloseAction implements stack.LinkEndpoint. +func (*endpoint) SetOnCloseAction(func()) {} + +// InjectableEndpoint is an injectable fd-based endpoint. The endpoint writes +// to the FD, but does not read from it. All reads come from injected packets. +// +// +stateify savable +type InjectableEndpoint struct { + endpoint + + mu injectableEndpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher +} + +// Attach saves the stack network-layer dispatcher for use later when packets +// are injected. +func (e *InjectableEndpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + defer e.mu.Unlock() + e.dispatcher = dispatcher +} + +// InjectInbound injects an inbound packet. If the endpoint is not attached, the +// packet is not delivered. +func (e *InjectableEndpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverNetworkPacket(protocol, pkt) + } +} + +// NewInjectable creates a new fd-based InjectableEndpoint. +func NewInjectable(fd int, mtu uint32, capabilities stack.LinkEndpointCapabilities) (*InjectableEndpoint, error) { + unix.SetNonblock(fd, true) + isSocket, err := isSocketFD(fd) + if err != nil { + return nil, err + } + + return &InjectableEndpoint{endpoint: endpoint{ + fds: []fdInfo{{fd: fd, isSocket: isSocket}}, + mtu: mtu, + caps: capabilities, + writevMaxIovs: rawfile.MaxIovs, + }}, nil +} diff --git a/pkg/tcpip/link/fdbased/endpoint_mutex.go b/pkg/tcpip/link/fdbased/endpoint_mutex.go new file mode 100644 index 0000000..d05b264 --- /dev/null +++ b/pkg/tcpip/link/fdbased/endpoint_mutex.go @@ -0,0 +1,96 @@ +package fdbased + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/fdbased/endpoint_unsafe.go b/pkg/tcpip/link/fdbased/endpoint_unsafe.go new file mode 100644 index 0000000..904393f --- /dev/null +++ b/pkg/tcpip/link/fdbased/endpoint_unsafe.go @@ -0,0 +1,24 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package fdbased + +import ( + "unsafe" +) + +const virtioNetHdrSize = int(unsafe.Sizeof(virtioNetHdr{})) diff --git a/pkg/tcpip/link/fdbased/fdbased_nonlinux_state_autogen.go b/pkg/tcpip/link/fdbased/fdbased_nonlinux_state_autogen.go new file mode 100644 index 0000000..015f8f6 --- /dev/null +++ b/pkg/tcpip/link/fdbased/fdbased_nonlinux_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build !linux || (!amd64 && !arm64) +// +build !linux !amd64,!arm64 + +package fdbased diff --git a/pkg/tcpip/link/fdbased/fdbased_state_autogen.go b/pkg/tcpip/link/fdbased/fdbased_state_autogen.go new file mode 100644 index 0000000..c058305 --- /dev/null +++ b/pkg/tcpip/link/fdbased/fdbased_state_autogen.go @@ -0,0 +1,438 @@ +// automatically generated by stateify. + +//go:build linux && ((linux && amd64) || (linux && arm64)) && linux && linux +// +build linux +// +build linux,amd64 linux,arm64 +// +build linux +// +build linux + +package fdbased + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (f *fdInfo) StateTypeName() string { + return "pkg/tcpip/link/fdbased.fdInfo" +} + +func (f *fdInfo) StateFields() []string { + return []string{ + "fd", + "isSocket", + } +} + +func (f *fdInfo) beforeSave() {} + +// +checklocksignore +func (f *fdInfo) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.fd) + stateSinkObject.Save(1, &f.isSocket) +} + +func (f *fdInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (f *fdInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.fd) + stateSourceObject.Load(1, &f.isSocket) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/link/fdbased.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "fds", + "hdrSize", + "caps", + "inboundDispatchers", + "dispatcher", + "packetDispatchMode", + "gsoMaxSize", + "gsoKind", + "maxSyscallHeaderBytes", + "writevMaxIovs", + "addr", + "mtu", + } +} + +func (e *endpoint) beforeSave() {} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.fds) + stateSinkObject.Save(1, &e.hdrSize) + stateSinkObject.Save(2, &e.caps) + stateSinkObject.Save(3, &e.inboundDispatchers) + stateSinkObject.Save(4, &e.dispatcher) + stateSinkObject.Save(5, &e.packetDispatchMode) + stateSinkObject.Save(6, &e.gsoMaxSize) + stateSinkObject.Save(7, &e.gsoKind) + stateSinkObject.Save(8, &e.maxSyscallHeaderBytes) + stateSinkObject.Save(9, &e.writevMaxIovs) + stateSinkObject.Save(10, &e.addr) + stateSinkObject.Save(11, &e.mtu) +} + +func (e *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.fds) + stateSourceObject.Load(1, &e.hdrSize) + stateSourceObject.Load(2, &e.caps) + stateSourceObject.Load(3, &e.inboundDispatchers) + stateSourceObject.Load(4, &e.dispatcher) + stateSourceObject.Load(5, &e.packetDispatchMode) + stateSourceObject.Load(6, &e.gsoMaxSize) + stateSourceObject.Load(7, &e.gsoKind) + stateSourceObject.Load(8, &e.maxSyscallHeaderBytes) + stateSourceObject.Load(9, &e.writevMaxIovs) + stateSourceObject.Load(10, &e.addr) + stateSourceObject.Load(11, &e.mtu) +} + +func (o *Options) StateTypeName() string { + return "pkg/tcpip/link/fdbased.Options" +} + +func (o *Options) StateFields() []string { + return []string{ + "FDs", + "MTU", + "EthernetHeader", + "ClosedFunc", + "Address", + "SaveRestore", + "DisconnectOk", + "GSOMaxSize", + "GVisorGSOEnabled", + "PacketDispatchMode", + "TXChecksumOffload", + "RXChecksumOffload", + "MaxSyscallHeaderBytes", + "InterfaceIndex", + "GRO", + "ProcessorsPerChannel", + } +} + +func (o *Options) beforeSave() {} + +// +checklocksignore +func (o *Options) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.FDs) + stateSinkObject.Save(1, &o.MTU) + stateSinkObject.Save(2, &o.EthernetHeader) + stateSinkObject.Save(3, &o.ClosedFunc) + stateSinkObject.Save(4, &o.Address) + stateSinkObject.Save(5, &o.SaveRestore) + stateSinkObject.Save(6, &o.DisconnectOk) + stateSinkObject.Save(7, &o.GSOMaxSize) + stateSinkObject.Save(8, &o.GVisorGSOEnabled) + stateSinkObject.Save(9, &o.PacketDispatchMode) + stateSinkObject.Save(10, &o.TXChecksumOffload) + stateSinkObject.Save(11, &o.RXChecksumOffload) + stateSinkObject.Save(12, &o.MaxSyscallHeaderBytes) + stateSinkObject.Save(13, &o.InterfaceIndex) + stateSinkObject.Save(14, &o.GRO) + stateSinkObject.Save(15, &o.ProcessorsPerChannel) +} + +func (o *Options) afterLoad(context.Context) {} + +// +checklocksignore +func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.FDs) + stateSourceObject.Load(1, &o.MTU) + stateSourceObject.Load(2, &o.EthernetHeader) + stateSourceObject.Load(3, &o.ClosedFunc) + stateSourceObject.Load(4, &o.Address) + stateSourceObject.Load(5, &o.SaveRestore) + stateSourceObject.Load(6, &o.DisconnectOk) + stateSourceObject.Load(7, &o.GSOMaxSize) + stateSourceObject.Load(8, &o.GVisorGSOEnabled) + stateSourceObject.Load(9, &o.PacketDispatchMode) + stateSourceObject.Load(10, &o.TXChecksumOffload) + stateSourceObject.Load(11, &o.RXChecksumOffload) + stateSourceObject.Load(12, &o.MaxSyscallHeaderBytes) + stateSourceObject.Load(13, &o.InterfaceIndex) + stateSourceObject.Load(14, &o.GRO) + stateSourceObject.Load(15, &o.ProcessorsPerChannel) +} + +func (e *InjectableEndpoint) StateTypeName() string { + return "pkg/tcpip/link/fdbased.InjectableEndpoint" +} + +func (e *InjectableEndpoint) StateFields() []string { + return []string{ + "endpoint", + "dispatcher", + } +} + +func (e *InjectableEndpoint) beforeSave() {} + +// +checklocksignore +func (e *InjectableEndpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.endpoint) + stateSinkObject.Save(1, &e.dispatcher) +} + +func (e *InjectableEndpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *InjectableEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.endpoint) + stateSourceObject.Load(1, &e.dispatcher) +} + +func (d *packetMMapDispatcher) StateTypeName() string { + return "pkg/tcpip/link/fdbased.packetMMapDispatcher" +} + +func (d *packetMMapDispatcher) StateFields() []string { + return []string{ + "StopFD", + "fd", + "e", + "ringBuffer", + "ringOffset", + "mgr", + } +} + +func (d *packetMMapDispatcher) beforeSave() {} + +// +checklocksignore +func (d *packetMMapDispatcher) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.StopFD) + stateSinkObject.Save(1, &d.fd) + stateSinkObject.Save(2, &d.e) + stateSinkObject.Save(3, &d.ringBuffer) + stateSinkObject.Save(4, &d.ringOffset) + stateSinkObject.Save(5, &d.mgr) +} + +func (d *packetMMapDispatcher) afterLoad(context.Context) {} + +// +checklocksignore +func (d *packetMMapDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.StopFD) + stateSourceObject.Load(1, &d.fd) + stateSourceObject.Load(2, &d.e) + stateSourceObject.Load(3, &d.ringBuffer) + stateSourceObject.Load(4, &d.ringOffset) + stateSourceObject.Load(5, &d.mgr) +} + +func (b *iovecBuffer) StateTypeName() string { + return "pkg/tcpip/link/fdbased.iovecBuffer" +} + +func (b *iovecBuffer) StateFields() []string { + return []string{ + "views", + "sizes", + "skipsVnetHdr", + "pulledIndex", + } +} + +func (b *iovecBuffer) beforeSave() {} + +// +checklocksignore +func (b *iovecBuffer) StateSave(stateSinkObject state.Sink) { + b.beforeSave() + stateSinkObject.Save(0, &b.views) + stateSinkObject.Save(1, &b.sizes) + stateSinkObject.Save(2, &b.skipsVnetHdr) + stateSinkObject.Save(3, &b.pulledIndex) +} + +func (b *iovecBuffer) afterLoad(context.Context) {} + +// +checklocksignore +func (b *iovecBuffer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &b.views) + stateSourceObject.Load(1, &b.sizes) + stateSourceObject.Load(2, &b.skipsVnetHdr) + stateSourceObject.Load(3, &b.pulledIndex) +} + +func (d *readVDispatcher) StateTypeName() string { + return "pkg/tcpip/link/fdbased.readVDispatcher" +} + +func (d *readVDispatcher) StateFields() []string { + return []string{ + "StopFD", + "fd", + "e", + "buf", + "mgr", + } +} + +func (d *readVDispatcher) beforeSave() {} + +// +checklocksignore +func (d *readVDispatcher) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.StopFD) + stateSinkObject.Save(1, &d.fd) + stateSinkObject.Save(2, &d.e) + stateSinkObject.Save(3, &d.buf) + stateSinkObject.Save(4, &d.mgr) +} + +func (d *readVDispatcher) afterLoad(context.Context) {} + +// +checklocksignore +func (d *readVDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.StopFD) + stateSourceObject.Load(1, &d.fd) + stateSourceObject.Load(2, &d.e) + stateSourceObject.Load(3, &d.buf) + stateSourceObject.Load(4, &d.mgr) +} + +func (r *recvMMsgDispatcher) StateTypeName() string { + return "pkg/tcpip/link/fdbased.recvMMsgDispatcher" +} + +func (r *recvMMsgDispatcher) StateFields() []string { + return []string{ + "StopFD", + "fd", + "e", + "bufs", + "pkts", + "gro", + "mgr", + } +} + +func (r *recvMMsgDispatcher) beforeSave() {} + +// +checklocksignore +func (r *recvMMsgDispatcher) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.StopFD) + stateSinkObject.Save(1, &r.fd) + stateSinkObject.Save(2, &r.e) + stateSinkObject.Save(3, &r.bufs) + stateSinkObject.Save(4, &r.pkts) + stateSinkObject.Save(5, &r.gro) + stateSinkObject.Save(6, &r.mgr) +} + +// +checklocksignore +func (r *recvMMsgDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.StopFD) + stateSourceObject.Load(1, &r.fd) + stateSourceObject.Load(2, &r.e) + stateSourceObject.Load(3, &r.bufs) + stateSourceObject.Load(4, &r.pkts) + stateSourceObject.Load(5, &r.gro) + stateSourceObject.Load(6, &r.mgr) + stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) }) +} + +func (p *processor) StateTypeName() string { + return "pkg/tcpip/link/fdbased.processor" +} + +func (p *processor) StateFields() []string { + return []string{ + "pkts", + "e", + "gro", + "sleeper", + "packetWaker", + "closeWaker", + } +} + +func (p *processor) beforeSave() {} + +// +checklocksignore +func (p *processor) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.pkts) + stateSinkObject.Save(1, &p.e) + stateSinkObject.Save(2, &p.gro) + stateSinkObject.Save(3, &p.sleeper) + stateSinkObject.Save(4, &p.packetWaker) + stateSinkObject.Save(5, &p.closeWaker) +} + +func (p *processor) afterLoad(context.Context) {} + +// +checklocksignore +func (p *processor) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.pkts) + stateSourceObject.Load(1, &p.e) + stateSourceObject.Load(2, &p.gro) + stateSourceObject.Load(3, &p.sleeper) + stateSourceObject.Load(4, &p.packetWaker) + stateSourceObject.Load(5, &p.closeWaker) +} + +func (m *processorManager) StateTypeName() string { + return "pkg/tcpip/link/fdbased.processorManager" +} + +func (m *processorManager) StateFields() []string { + return []string{ + "processors", + "seed", + "e", + "ready", + } +} + +func (m *processorManager) beforeSave() {} + +// +checklocksignore +func (m *processorManager) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.processors) + stateSinkObject.Save(1, &m.seed) + stateSinkObject.Save(2, &m.e) + stateSinkObject.Save(3, &m.ready) +} + +// +checklocksignore +func (m *processorManager) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.processors) + stateSourceObject.Load(1, &m.seed) + stateSourceObject.Load(2, &m.e) + stateSourceObject.Load(3, &m.ready) + stateSourceObject.AfterLoad(func() { m.afterLoad(ctx) }) +} + +func init() { + state.Register((*fdInfo)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*Options)(nil)) + state.Register((*InjectableEndpoint)(nil)) + state.Register((*packetMMapDispatcher)(nil)) + state.Register((*iovecBuffer)(nil)) + state.Register((*readVDispatcher)(nil)) + state.Register((*recvMMsgDispatcher)(nil)) + state.Register((*processor)(nil)) + state.Register((*processorManager)(nil)) +} diff --git a/pkg/tcpip/link/fdbased/fdbased_unsafe_state_autogen.go b/pkg/tcpip/link/fdbased/fdbased_unsafe_state_autogen.go new file mode 100644 index 0000000..6a5ed4a --- /dev/null +++ b/pkg/tcpip/link/fdbased/fdbased_unsafe_state_autogen.go @@ -0,0 +1,7 @@ +// automatically generated by stateify. + +//go:build linux && ((linux && amd64) || (linux && arm64)) +// +build linux +// +build linux,amd64 linux,arm64 + +package fdbased diff --git a/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go b/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go new file mode 100644 index 0000000..2c813d8 --- /dev/null +++ b/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go @@ -0,0 +1,96 @@ +package fdbased + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type injectableEndpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var injectableEndpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type injectableEndpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *injectableEndpointRWMutex) Lock() { + locking.AddGLock(injectableEndpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *injectableEndpointRWMutex) NestedLock(i injectableEndpointlockNameIndex) { + locking.AddGLock(injectableEndpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *injectableEndpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(injectableEndpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *injectableEndpointRWMutex) NestedUnlock(i injectableEndpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(injectableEndpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *injectableEndpointRWMutex) RLock() { + locking.AddGLock(injectableEndpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *injectableEndpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(injectableEndpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *injectableEndpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *injectableEndpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *injectableEndpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var injectableEndpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func injectableEndpointinitLockNames() {} + +func init() { + injectableEndpointinitLockNames() + injectableEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(injectableEndpointRWMutex{}), injectableEndpointlockNames) +} diff --git a/pkg/tcpip/link/fdbased/mmap.go b/pkg/tcpip/link/fdbased/mmap.go new file mode 100644 index 0000000..9974e15 --- /dev/null +++ b/pkg/tcpip/link/fdbased/mmap.go @@ -0,0 +1,199 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build (linux && amd64) || (linux && arm64) +// +build linux,amd64 linux,arm64 + +package fdbased + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/link/stopfd" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "golang.org/x/sys/unix" +) + +const ( + tPacketAlignment = uintptr(16) + tpStatusKernel = 0 + tpStatusUser = 1 + tpStatusCopy = 2 + tpStatusLosing = 4 +) + +// We overallocate the frame size to accommodate space for the +// TPacketHdr+RawSockAddrLinkLayer+MAC header and any padding. +// +// Memory allocated for the ring buffer: tpBlockSize * tpBlockNR = 2 MiB +// +// NOTE: +// +// Frames need to be aligned at 16 byte boundaries. +// BlockSize needs to be page aligned. +// +// For details see PACKET_MMAP setting constraints in +// https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt +const ( + tpFrameSize = 65536 + 128 + tpBlockSize = tpFrameSize * 32 + tpBlockNR = 1 + tpFrameNR = (tpBlockSize * tpBlockNR) / tpFrameSize +) + +// tPacketAlign aligns the pointer v at a tPacketAlignment boundary. Direct +// translation of the TPACKET_ALIGN macro in . +func tPacketAlign(v uintptr) uintptr { + return (v + tPacketAlignment - 1) & uintptr(^(tPacketAlignment - 1)) +} + +// tPacketReq is the tpacket_req structure as described in +// https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt +type tPacketReq struct { + tpBlockSize uint32 + tpBlockNR uint32 + tpFrameSize uint32 + tpFrameNR uint32 +} + +// tPacketHdr is tpacket_hdr structure as described in +type tPacketHdr []byte + +const ( + tpStatusOffset = 0 + tpLenOffset = 8 + tpSnapLenOffset = 12 + tpMacOffset = 16 + tpNetOffset = 18 + tpSecOffset = 20 + tpUSecOffset = 24 +) + +func (t tPacketHdr) tpLen() uint32 { + return binary.LittleEndian.Uint32(t[tpLenOffset:]) +} + +func (t tPacketHdr) tpSnapLen() uint32 { + return binary.LittleEndian.Uint32(t[tpSnapLenOffset:]) +} + +func (t tPacketHdr) tpMac() uint16 { + return binary.LittleEndian.Uint16(t[tpMacOffset:]) +} + +func (t tPacketHdr) tpNet() uint16 { + return binary.LittleEndian.Uint16(t[tpNetOffset:]) +} + +func (t tPacketHdr) tpSec() uint32 { + return binary.LittleEndian.Uint32(t[tpSecOffset:]) +} + +func (t tPacketHdr) tpUSec() uint32 { + return binary.LittleEndian.Uint32(t[tpUSecOffset:]) +} + +func (t tPacketHdr) Payload() []byte { + return t[uint32(t.tpMac()) : uint32(t.tpMac())+t.tpSnapLen()] +} + +// packetMMapDispatcher uses PACKET_RX_RING's to read/dispatch inbound packets. +// See: mmap_amd64_unsafe.go for implementation details. +// +// +stateify savable +type packetMMapDispatcher struct { + stopfd.StopFD + // fd is the file descriptor used to send and receive packets. + fd int + + // e is the endpoint this dispatcher is attached to. + e *endpoint + + // ringBuffer is only used when PacketMMap dispatcher is used and points + // to the start of the mmapped PACKET_RX_RING buffer. + ringBuffer []byte + + // ringOffset is the current offset into the ring buffer where the next + // inbound packet will be placed by the kernel. + ringOffset int + + // mgr is the processor goroutine manager. + mgr *processorManager +} + +func (d *packetMMapDispatcher) release() { + d.mgr.close() +} + +func (d *packetMMapDispatcher) readMMappedPackets() (stack.PacketBufferList, bool, tcpip.Error) { + var pkts stack.PacketBufferList + hdr := tPacketHdr(d.ringBuffer[d.ringOffset*tpFrameSize:]) + for hdr.tpStatus()&tpStatusUser == 0 { + stopped, errno := rawfile.BlockingPollUntilStopped(d.EFD, d.fd, unix.POLLIN|unix.POLLERR) + if errno != 0 { + if errno == unix.EINTR { + continue + } + return pkts, stopped, tcpip.TranslateErrno(errno) + } + if stopped { + return pkts, true, nil + } + if hdr.tpStatus()&tpStatusCopy != 0 { + // This frame is truncated so skip it after flipping the + // buffer to the kernel. + hdr.setTPStatus(tpStatusKernel) + d.ringOffset = (d.ringOffset + 1) % tpFrameNR + hdr = (tPacketHdr)(d.ringBuffer[d.ringOffset*tpFrameSize:]) + continue + } + } + + for hdr.tpStatus()&tpStatusUser == 1 { + // Copy out the packet from the mmapped frame to a locally owned buffer. + pkts.PushBack(stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithView(buffer.NewViewWithData(hdr.Payload())), + })) + // Release packet to kernel. + hdr.setTPStatus(tpStatusKernel) + d.ringOffset = (d.ringOffset + 1) % tpFrameNR + hdr = tPacketHdr(d.ringBuffer[d.ringOffset*tpFrameSize:]) + } + return pkts, false, nil +} + +// dispatch reads packets from an mmaped ring buffer and dispatches them to the +// network stack. +func (d *packetMMapDispatcher) dispatch() (bool, tcpip.Error) { + pkts, stopped, err := d.readMMappedPackets() + defer pkts.Reset() + if err != nil || stopped { + return false, err + } + d.e.mu.RLock() + addr := d.e.addr + d.e.mu.RUnlock() + for _, pkt := range pkts.AsSlice() { + if d.e.parseInboundHeader(pkt, addr) { + d.mgr.queuePacket(pkt, d.e.hdrSize > 0) + } + } + if pkts.Len() > 0 { + d.mgr.wakeReady() + } + return true, nil +} diff --git a/pkg/tcpip/link/fdbased/mmap_nonlinux.go b/pkg/tcpip/link/fdbased/mmap_nonlinux.go new file mode 100644 index 0000000..c76c4b6 --- /dev/null +++ b/pkg/tcpip/link/fdbased/mmap_nonlinux.go @@ -0,0 +1,24 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux || (!amd64 && !arm64) +// +build !linux !amd64,!arm64 + +package fdbased + +// Stubbed out version for non-linux/non-amd64/non-arm64 platforms. + +func newPacketMMapDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) { + return nil, nil +} diff --git a/pkg/tcpip/link/fdbased/mmap_unsafe.go b/pkg/tcpip/link/fdbased/mmap_unsafe.go new file mode 100644 index 0000000..542e8b7 --- /dev/null +++ b/pkg/tcpip/link/fdbased/mmap_unsafe.go @@ -0,0 +1,92 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build (linux && amd64) || (linux && arm64) +// +build linux,amd64 linux,arm64 + +package fdbased + +import ( + "fmt" + "unsafe" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/tcpip/link/stopfd" + "golang.org/x/sys/unix" +) + +// tPacketHdrlen is the TPACKET_HDRLEN variable defined in . +var tPacketHdrlen = tPacketAlign(unsafe.Sizeof(tPacketHdr{}) + unsafe.Sizeof(unix.RawSockaddrLinklayer{})) + +// tpStatus returns the frame status field. +// The status is concurrently updated by the kernel as a result we must +// use atomic operations to prevent races. +func (t tPacketHdr) tpStatus() uint32 { + hdr := unsafe.Pointer(&t[0]) + statusPtr := unsafe.Pointer(uintptr(hdr) + uintptr(tpStatusOffset)) + return (*atomicbitops.Uint32)(statusPtr).Load() +} + +// setTPStatus set's the frame status to the provided status. +// The status is concurrently updated by the kernel as a result we must +// use atomic operations to prevent races. +func (t tPacketHdr) setTPStatus(status uint32) { + hdr := unsafe.Pointer(&t[0]) + statusPtr := unsafe.Pointer(uintptr(hdr) + uintptr(tpStatusOffset)) + (*atomicbitops.Uint32)(statusPtr).Store(status) +} + +func newPacketMMapDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) { + stopFD, err := stopfd.New() + if err != nil { + return nil, err + } + d := &packetMMapDispatcher{ + StopFD: stopFD, + fd: fd, + e: e, + } + pageSize := unix.Getpagesize() + if tpBlockSize%pageSize != 0 { + return nil, fmt.Errorf("tpBlockSize: %d is not page aligned, pagesize: %d", tpBlockSize, pageSize) + } + tReq := tPacketReq{ + tpBlockSize: uint32(tpBlockSize), + tpBlockNR: uint32(tpBlockNR), + tpFrameSize: uint32(tpFrameSize), + tpFrameNR: uint32(tpFrameNR), + } + // Setup PACKET_RX_RING. + if err := setsockopt(d.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, unsafe.Pointer(&tReq), unsafe.Sizeof(tReq)); err != nil { + return nil, fmt.Errorf("failed to enable PACKET_RX_RING: %v", err) + } + // Let's mmap the blocks. + sz := tpBlockSize * tpBlockNR + buf, err := unix.Mmap(d.fd, 0, sz, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + return nil, fmt.Errorf("unix.Mmap(...,0, %v, ...) failed = %v", sz, err) + } + d.mgr = newProcessorManager(opts, e) + d.mgr.start() + d.ringBuffer = buf + return d, nil +} + +func setsockopt(fd, level, name int, val unsafe.Pointer, vallen uintptr) error { + if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), uintptr(val), vallen, 0); errno != 0 { + return error(errno) + } + + return nil +} diff --git a/pkg/tcpip/link/fdbased/packet_dispatchers.go b/pkg/tcpip/link/fdbased/packet_dispatchers.go new file mode 100644 index 0000000..16c2f33 --- /dev/null +++ b/pkg/tcpip/link/fdbased/packet_dispatchers.go @@ -0,0 +1,330 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package fdbased + +import ( + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/link/stopfd" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/stack/gro" + "golang.org/x/sys/unix" +) + +// BufConfig defines the shape of the buffer used to read packets from the NIC. +// The duplication of 256 is intended so that the sum of the elements can cover +// the maximum packet size we expect to receive. See TestBufConfigMaxLength. +var BufConfig = []int{128, 256, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768} + +// +stateify savable +type iovecBuffer struct { + // buffer is the actual buffer that holds the packet contents. Some contents + // are reused across calls to pullBuffer if number of requested bytes is + // smaller than the number of bytes allocated in the buffer. + views []*buffer.View + + // iovecs are initialized with base pointers/len of the corresponding + // entries in the views defined above, except when GSO is enabled + // (skipsVnetHdr) then the first iovec points to a buffer for the vnet header + // which is stripped before the views are passed up the stack for further + // processing. + iovecs []unix.Iovec `state:"nosave"` + + // sizes is an array of buffer sizes for the underlying views. sizes is + // immutable. + sizes []int + + // skipsVnetHdr is true if virtioNetHdr is to skipped. + skipsVnetHdr bool + + // pulledIndex is the index of the last []byte buffer pulled from the + // underlying buffer storage during a call to pullBuffers. It is -1 + // if no buffer is pulled. + pulledIndex int +} + +func newIovecBuffer(sizes []int, skipsVnetHdr bool) *iovecBuffer { + b := &iovecBuffer{ + views: make([]*buffer.View, len(sizes)), + sizes: sizes, + skipsVnetHdr: skipsVnetHdr, + } + niov := len(b.views) + if b.skipsVnetHdr { + niov++ + } + b.iovecs = make([]unix.Iovec, niov) + return b +} + +func (b *iovecBuffer) nextIovecs() []unix.Iovec { + vnetHdrOff := 0 + if b.skipsVnetHdr { + var vnetHdr [virtioNetHdrSize]byte + // The kernel adds virtioNetHdr before each packet, but + // we don't use it, so we allocate a buffer for it, + // add it in iovecs but don't add it in a view. + b.iovecs[0] = unix.Iovec{Base: &vnetHdr[0]} + b.iovecs[0].SetLen(virtioNetHdrSize) + vnetHdrOff++ + } + + for i := range b.views { + if b.views[i] != nil { + break + } + v := buffer.NewViewSize(b.sizes[i]) + b.views[i] = v + b.iovecs[i+vnetHdrOff] = unix.Iovec{Base: v.BasePtr()} + b.iovecs[i+vnetHdrOff].SetLen(v.Size()) + } + return b.iovecs +} + +// pullBuffer extracts the enough underlying storage from b.buffer to hold n +// bytes. It removes this storage from b.buffer, returns a new buffer +// that holds the storage, and updates pulledIndex to indicate which part +// of b.buffer's storage must be reallocated during the next call to +// nextIovecs. +func (b *iovecBuffer) pullBuffer(n int) buffer.Buffer { + var views []*buffer.View + c := 0 + if b.skipsVnetHdr { + c += virtioNetHdrSize + if c >= n { + // Nothing in the packet. + return buffer.Buffer{} + } + } + // Remove the used views from the buffer. + for i, v := range b.views { + c += v.Size() + if c >= n { + b.views[i].CapLength(v.Size() - (c - n)) + views = append(views, b.views[:i+1]...) + break + } + } + for i := range views { + b.views[i] = nil + } + if b.skipsVnetHdr { + // Exclude the size of the vnet header. + n -= virtioNetHdrSize + } + pulled := buffer.Buffer{} + for _, v := range views { + pulled.Append(v) + } + pulled.Truncate(int64(n)) + return pulled +} + +func (b *iovecBuffer) release() { + for _, v := range b.views { + if v != nil { + v.Release() + v = nil + } + } +} + +// readVDispatcher uses readv() system call to read inbound packets and +// dispatches them. +// +// +stateify savable +type readVDispatcher struct { + stopfd.StopFD + // fd is the file descriptor used to send and receive packets. + fd int + + // e is the endpoint this dispatcher is attached to. + e *endpoint + + // buf is the iovec buffer that contains the packet contents. + buf *iovecBuffer + + // mgr is the processor goroutine manager. + mgr *processorManager +} + +func newReadVDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) { + stopFD, err := stopfd.New() + if err != nil { + return nil, err + } + d := &readVDispatcher{ + StopFD: stopFD, + fd: fd, + e: e, + } + skipsVnetHdr := d.e.gsoKind == stack.HostGSOSupported + d.buf = newIovecBuffer(BufConfig, skipsVnetHdr) + d.mgr = newProcessorManager(opts, e) + d.mgr.start() + return d, nil +} + +func (d *readVDispatcher) release() { + d.buf.release() + d.mgr.close() +} + +// dispatch reads one packet from the file descriptor and dispatches it. +func (d *readVDispatcher) dispatch() (bool, tcpip.Error) { + n, errno := rawfile.BlockingReadvUntilStopped(d.EFD, d.fd, d.buf.nextIovecs()) + if n <= 0 || errno != 0 { + return false, tcpip.TranslateErrno(errno) + } + + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: d.buf.pullBuffer(n), + }) + defer pkt.DecRef() + + d.e.mu.RLock() + addr := d.e.addr + d.e.mu.RUnlock() + if !d.e.parseInboundHeader(pkt, addr) { + return false, nil + } + d.mgr.queuePacket(pkt, d.e.hdrSize > 0) + d.mgr.wakeReady() + return true, nil +} + +// recvMMsgDispatcher uses the recvmmsg system call to read inbound packets and +// dispatches them. +// +// +stateify savable +type recvMMsgDispatcher struct { + stopfd.StopFD + // fd is the file descriptor used to send and receive packets. + fd int + + // e is the endpoint this dispatcher is attached to. + e *endpoint + + // bufs is an array of iovec buffers that contain packet contents. + bufs []*iovecBuffer + + // msgHdrs is an array of MMsgHdr objects where each MMsghdr is used to + // reference an array of iovecs in the iovecs field defined above. This + // array is passed as the parameter to recvmmsg call to retrieve + // potentially more than 1 packet per unix. + msgHdrs []rawfile.MMsgHdr `state:"nosave"` + + // pkts is reused to avoid allocations. + pkts stack.PacketBufferList + + // gro coalesces incoming packets to increase throughput. + gro gro.GRO + + // mgr is the processor goroutine manager. + mgr *processorManager +} + +const ( + // MaxMsgsPerRecv is the maximum number of packets we want to retrieve + // in a single RecvMMsg call. + MaxMsgsPerRecv = 8 +) + +func newRecvMMsgDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) { + stopFD, err := stopfd.New() + if err != nil { + return nil, err + } + d := &recvMMsgDispatcher{ + StopFD: stopFD, + fd: fd, + e: e, + bufs: make([]*iovecBuffer, MaxMsgsPerRecv), + msgHdrs: make([]rawfile.MMsgHdr, MaxMsgsPerRecv), + } + skipsVnetHdr := d.e.gsoKind == stack.HostGSOSupported + for i := range d.bufs { + d.bufs[i] = newIovecBuffer(BufConfig, skipsVnetHdr) + } + d.gro.Init(opts.GRO) + d.mgr = newProcessorManager(opts, e) + d.mgr.start() + + return d, nil +} + +func (d *recvMMsgDispatcher) release() { + for _, iov := range d.bufs { + iov.release() + } + d.mgr.close() +} + +// recvMMsgDispatch reads more than one packet at a time from the file +// descriptor and dispatches it. +func (d *recvMMsgDispatcher) dispatch() (bool, tcpip.Error) { + // Fill message headers. + for k := range d.msgHdrs { + if d.msgHdrs[k].Msg.Iovlen > 0 { + break + } + iovecs := d.bufs[k].nextIovecs() + iovLen := len(iovecs) + d.msgHdrs[k].Len = 0 + d.msgHdrs[k].Msg.Iov = &iovecs[0] + d.msgHdrs[k].Msg.SetIovlen(iovLen) + } + + nMsgs, errno := rawfile.BlockingRecvMMsgUntilStopped(d.EFD, d.fd, d.msgHdrs) + if errno != 0 { + return false, tcpip.TranslateErrno(errno) + } + if nMsgs == -1 { + return false, nil + } + + // Process each of received packets. + + d.e.mu.RLock() + addr := d.e.addr + dsp := d.e.dispatcher + d.e.mu.RUnlock() + + d.gro.Dispatcher = dsp + defer d.pkts.Reset() + + for k := 0; k < nMsgs; k++ { + n := int(d.msgHdrs[k].Len) + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: d.bufs[k].pullBuffer(n), + }) + d.pkts.PushBack(pkt) + + // Mark that this iovec has been processed. + d.msgHdrs[k].Msg.Iovlen = 0 + + if d.e.parseInboundHeader(pkt, addr) { + pkt.RXChecksumValidated = d.e.caps&stack.CapabilityRXChecksumOffload != 0 + d.mgr.queuePacket(pkt, d.e.hdrSize > 0) + } + } + d.mgr.wakeReady() + + return true, nil +} diff --git a/pkg/tcpip/link/fdbased/processor_mutex.go b/pkg/tcpip/link/fdbased/processor_mutex.go new file mode 100644 index 0000000..cd297d2 --- /dev/null +++ b/pkg/tcpip/link/fdbased/processor_mutex.go @@ -0,0 +1,64 @@ +package fdbased + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type processorMutex struct { + mu sync.Mutex +} + +var processorprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var processorlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type processorlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *processorMutex) Lock() { + locking.AddGLock(processorprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *processorMutex) NestedLock(i processorlockNameIndex) { + locking.AddGLock(processorprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *processorMutex) Unlock() { + locking.DelGLock(processorprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *processorMutex) NestedUnlock(i processorlockNameIndex) { + locking.DelGLock(processorprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func processorinitLockNames() {} + +func init() { + processorinitLockNames() + processorprefixIndex = locking.NewMutexClass(reflect.TypeOf(processorMutex{}), processorlockNames) +} diff --git a/pkg/tcpip/link/fdbased/processors.go b/pkg/tcpip/link/fdbased/processors.go new file mode 100644 index 0000000..47450f8 --- /dev/null +++ b/pkg/tcpip/link/fdbased/processors.go @@ -0,0 +1,278 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package fdbased + +import ( + "context" + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/rand" + "github.com/sagernet/gvisor/pkg/sleep" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/hash/jenkins" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/stack/gro" +) + +// +stateify savable +type processor struct { + mu processorMutex `state:"nosave"` + // +checklocks:mu + pkts stack.PacketBufferList + + e *endpoint + gro gro.GRO + sleeper sleep.Sleeper + packetWaker sleep.Waker + closeWaker sleep.Waker +} + +func (p *processor) start(wg *sync.WaitGroup) { + defer wg.Done() + defer p.sleeper.Done() + for { + switch w := p.sleeper.Fetch(true); { + case w == &p.packetWaker: + p.deliverPackets() + case w == &p.closeWaker: + p.mu.Lock() + p.pkts.Reset() + p.mu.Unlock() + return + } + } +} + +func (p *processor) deliverPackets() { + p.e.mu.RLock() + p.gro.Dispatcher = p.e.dispatcher + p.e.mu.RUnlock() + if p.gro.Dispatcher == nil { + p.mu.Lock() + p.pkts.Reset() + p.mu.Unlock() + return + } + + p.mu.Lock() + for p.pkts.Len() > 0 { + pkt := p.pkts.PopFront() + p.mu.Unlock() + p.gro.Enqueue(pkt) + pkt.DecRef() + p.mu.Lock() + } + p.mu.Unlock() + p.gro.Flush() +} + +// processorManager handles starting, closing, and queuing packets on processor +// goroutines. +// +// +stateify savable +type processorManager struct { + processors []processor + seed uint32 + wg sync.WaitGroup `state:"nosave"` + e *endpoint + ready []bool +} + +// newProcessorManager creates a new processor manager. +func newProcessorManager(opts *Options, e *endpoint) *processorManager { + m := &processorManager{} + m.seed = rand.Uint32() + m.ready = make([]bool, opts.ProcessorsPerChannel) + m.processors = make([]processor, opts.ProcessorsPerChannel) + m.e = e + m.wg.Add(opts.ProcessorsPerChannel) + + for i := range m.processors { + p := &m.processors[i] + p.sleeper.AddWaker(&p.packetWaker) + p.sleeper.AddWaker(&p.closeWaker) + p.gro.Init(opts.GRO) + p.e = e + } + + return m +} + +// start starts the processor goroutines if the processor manager is configured +// with more than one processor. +func (m *processorManager) start() { + for i := range m.processors { + p := &m.processors[i] + // Only start processor in a separate goroutine if we have multiple of them. + if len(m.processors) > 1 { + go p.start(&m.wg) + } + } +} + +// afterLoad is invoked by stateify. +func (m *processorManager) afterLoad(context.Context) { + m.wg.Add(len(m.processors)) + m.start() +} + +func (m *processorManager) connectionHash(cid *connectionID) uint32 { + var payload [4]byte + binary.LittleEndian.PutUint16(payload[0:], cid.srcPort) + binary.LittleEndian.PutUint16(payload[2:], cid.dstPort) + + h := jenkins.Sum32(m.seed) + h.Write(payload[:]) + h.Write(cid.srcAddr) + h.Write(cid.dstAddr) + return h.Sum32() +} + +// queuePacket queues a packet to be delivered to the appropriate processor. +func (m *processorManager) queuePacket(pkt *stack.PacketBuffer, hasEthHeader bool) { + var pIdx uint32 + cid, nonConnectionPkt := tcpipConnectionID(pkt) + if !hasEthHeader { + if nonConnectionPkt { + // If there's no eth header this should be a standard tcpip packet. If + // it isn't the packet is invalid so drop it. + return + } + pkt.NetworkProtocolNumber = cid.proto + } + if len(m.processors) == 1 || nonConnectionPkt { + // If the packet is not associated with an active connection, use the + // first processor. + pIdx = 0 + } else { + pIdx = m.connectionHash(&cid) % uint32(len(m.processors)) + } + p := &m.processors[pIdx] + p.mu.Lock() + defer p.mu.Unlock() + p.pkts.PushBack(pkt.IncRef()) + m.ready[pIdx] = true +} + +type connectionID struct { + srcAddr, dstAddr []byte + srcPort, dstPort uint16 + proto tcpip.NetworkProtocolNumber +} + +// tcpipConnectionID returns a tcpip connection id tuple based on the data found +// in the packet. It returns true if the packet is not associated with an active +// connection (e.g ARP, NDP, etc). The method assumes link headers have already +// been processed if they were present. +func tcpipConnectionID(pkt *stack.PacketBuffer) (connectionID, bool) { + var cid connectionID + h, ok := pkt.Data().PullUp(1) + if !ok { + // Skip this packet. + return cid, true + } + + const tcpSrcDstPortLen = 4 + switch header.IPVersion(h) { + case header.IPv4Version: + hdrLen := header.IPv4(h).HeaderLength() + h, ok = pkt.Data().PullUp(int(hdrLen) + tcpSrcDstPortLen) + if !ok { + return cid, true + } + ipHdr := header.IPv4(h[:hdrLen]) + tcpHdr := header.TCP(h[hdrLen:][:tcpSrcDstPortLen]) + + cid.srcAddr = ipHdr.SourceAddressSlice() + cid.dstAddr = ipHdr.DestinationAddressSlice() + // All fragment packets need to be processed by the same goroutine, so + // only record the TCP ports if this is not a fragment packet. + if ipHdr.IsValid(pkt.Data().Size()) && !ipHdr.More() && ipHdr.FragmentOffset() == 0 { + cid.srcPort = tcpHdr.SourcePort() + cid.dstPort = tcpHdr.DestinationPort() + } + cid.proto = header.IPv4ProtocolNumber + case header.IPv6Version: + h, ok = pkt.Data().PullUp(header.IPv6FixedHeaderSize + tcpSrcDstPortLen) + if !ok { + return cid, true + } + ipHdr := header.IPv6(h) + + var tcpHdr header.TCP + if tcpip.TransportProtocolNumber(ipHdr.NextHeader()) == header.TCPProtocolNumber { + tcpHdr = header.TCP(h[header.IPv6FixedHeaderSize:][:tcpSrcDstPortLen]) + } else { + // Slow path for IPv6 extension headers :(. + dataBuf := pkt.Data().ToBuffer() + dataBuf.TrimFront(header.IPv6MinimumSize) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(ipHdr.NextHeader()), dataBuf) + defer it.Release() + for { + hdr, done, err := it.Next() + if done || err != nil { + break + } + hdr.Release() + } + h, ok = pkt.Data().PullUp(int(it.HeaderOffset()) + tcpSrcDstPortLen) + if !ok { + return cid, true + } + tcpHdr = header.TCP(h[it.HeaderOffset():][:tcpSrcDstPortLen]) + } + cid.srcAddr = ipHdr.SourceAddressSlice() + cid.dstAddr = ipHdr.DestinationAddressSlice() + cid.srcPort = tcpHdr.SourcePort() + cid.dstPort = tcpHdr.DestinationPort() + cid.proto = header.IPv6ProtocolNumber + default: + return cid, true + } + return cid, false +} + +func (m *processorManager) close() { + if len(m.processors) < 2 { + return + } + for i := range m.processors { + p := &m.processors[i] + p.closeWaker.Assert() + } +} + +// wakeReady wakes up all processors that have a packet queued. If there is only +// one processor, the method delivers the packet inline without waking a +// goroutine. +func (m *processorManager) wakeReady() { + for i, ready := range m.ready { + if !ready { + continue + } + p := &m.processors[i] + if len(m.processors) > 1 { + p.packetWaker.Assert() + } else { + p.deliverPackets() + } + m.ready[i] = false + } +} diff --git a/pkg/tcpip/link/fdbased/save_restore.go b/pkg/tcpip/link/fdbased/save_restore.go new file mode 100644 index 0000000..a8f7f88 --- /dev/null +++ b/pkg/tcpip/link/fdbased/save_restore.go @@ -0,0 +1,26 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fdbased + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/rawfile" +) + +// afterLoad is invoked by stateify. +func (r *recvMMsgDispatcher) afterLoad(context.Context) { + r.msgHdrs = make([]rawfile.MMsgHdr, MaxMsgsPerRecv) +} diff --git a/pkg/tcpip/link/loopback/endpoint_mutex.go b/pkg/tcpip/link/loopback/endpoint_mutex.go new file mode 100644 index 0000000..2d34fc2 --- /dev/null +++ b/pkg/tcpip/link/loopback/endpoint_mutex.go @@ -0,0 +1,96 @@ +package loopback + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/loopback/loopback.go b/pkg/tcpip/link/loopback/loopback.go new file mode 100644 index 0000000..8d0c360 --- /dev/null +++ b/pkg/tcpip/link/loopback/loopback.go @@ -0,0 +1,146 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package loopback provides the implementation of loopback data-link layer +// endpoints. Such endpoints just turn outbound packets into inbound ones. +// +// Loopback endpoints can be used in the networking stack by calling New() to +// create a new endpoint, and then passing it as an argument to +// Stack.CreateNIC(). +package loopback + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + loopbackMTU = 65536 +) + +// +stateify savable +type endpoint struct { + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher + // +checklocks:mu + addr tcpip.LinkAddress + // +checklocks:mu + mtu uint32 +} + +// New creates a new loopback endpoint. This link-layer endpoint just turns +// outbound packets into inbound packets. +func New() stack.LinkEndpoint { + return &endpoint{ + mtu: loopbackMTU, + } +} + +// Attach implements stack.LinkEndpoint.Attach. It just saves the stack network- +// layer dispatcher for later use when packets need to be dispatched. +func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + defer e.mu.Unlock() + e.dispatcher = dispatcher +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *endpoint) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.dispatcher != nil +} + +// MTU implements stack.LinkEndpoint.MTU. +func (e *endpoint) MTU() uint32 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mtu +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. It has no impact. +func (e *endpoint) SetMTU(mtu uint32) { + e.mu.Lock() + defer e.mu.Unlock() + e.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. Loopback advertises +// itself as supporting checksum offload, but in reality it's just omitted. +func (*endpoint) Capabilities() stack.LinkEndpointCapabilities { + return stack.CapabilityRXChecksumOffload | stack.CapabilityTXChecksumOffload | stack.CapabilitySaveRestore | stack.CapabilityLoopback +} + +// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. Given that the +// loopback interface doesn't have a header, it just returns 0. +func (*endpoint) MaxHeaderLength() uint16 { + return 0 +} + +// LinkAddress returns the link address of this endpoint. +func (e *endpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.addr = addr +} + +// Wait implements stack.LinkEndpoint.Wait. +func (*endpoint) Wait() {} + +// WritePackets implements stack.LinkEndpoint.WritePackets. If the endpoint is +// not attached, the packets are not delivered. +func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + for _, pkt := range pkts.AsSlice() { + // In order to properly loop back to the inbound side we must create a + // fresh packet that only contains the underlying payload with no headers + // or struct fields set. + newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: pkt.ToBuffer(), + }) + if d != nil { + d.DeliverNetworkPacket(pkt.NetworkProtocolNumber, newPkt) + } + newPkt.DecRef() + } + return pkts.Len(), nil +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (*endpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareLoopback +} + +// AddHeader implements stack.LinkEndpoint. +func (*endpoint) AddHeader(*stack.PacketBuffer) {} + +// ParseHeader implements stack.LinkEndpoint. +func (*endpoint) ParseHeader(*stack.PacketBuffer) bool { return true } + +// Close implements stack.LinkEndpoint. +func (*endpoint) Close() {} + +// SetOnCloseAction implements stack.LinkEndpoint. +func (*endpoint) SetOnCloseAction(func()) {} diff --git a/pkg/tcpip/link/loopback/loopback_state_autogen.go b/pkg/tcpip/link/loopback/loopback_state_autogen.go new file mode 100644 index 0000000..8e03a7a --- /dev/null +++ b/pkg/tcpip/link/loopback/loopback_state_autogen.go @@ -0,0 +1,44 @@ +// automatically generated by stateify. + +package loopback + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/link/loopback.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "dispatcher", + "addr", + "mtu", + } +} + +func (e *endpoint) beforeSave() {} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.dispatcher) + stateSinkObject.Save(1, &e.addr) + stateSinkObject.Save(2, &e.mtu) +} + +func (e *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.dispatcher) + stateSourceObject.Load(1, &e.addr) + stateSourceObject.Load(2, &e.mtu) +} + +func init() { + state.Register((*endpoint)(nil)) +} diff --git a/pkg/tcpip/link/muxed/endpoint_mutex.go b/pkg/tcpip/link/muxed/endpoint_mutex.go new file mode 100644 index 0000000..2f39d9a --- /dev/null +++ b/pkg/tcpip/link/muxed/endpoint_mutex.go @@ -0,0 +1,96 @@ +package muxed + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/muxed/injectable.go b/pkg/tcpip/link/muxed/injectable.go new file mode 100644 index 0000000..6f313de --- /dev/null +++ b/pkg/tcpip/link/muxed/injectable.go @@ -0,0 +1,174 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package muxed provides a muxed link endpoints. +package muxed + +import ( + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// InjectableEndpoint is an injectable multi endpoint. The endpoint has +// trivial routing rules that determine which InjectableEndpoint a given packet +// will be written to. Note that HandleLocal works differently for this +// endpoint (see WritePacket). +// +// +stateify savable +type InjectableEndpoint struct { + routes map[tcpip.Address]stack.InjectableLinkEndpoint + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher +} + +// MTU implements stack.LinkEndpoint. +func (m *InjectableEndpoint) MTU() uint32 { + minMTU := ^uint32(0) + for _, endpoint := range m.routes { + if endpointMTU := endpoint.MTU(); endpointMTU < minMTU { + minMTU = endpointMTU + } + } + return minMTU +} + +// SetMTU implements stack.LinkEndpoint. +func (m *InjectableEndpoint) SetMTU(mtu uint32) { + for _, endpoint := range m.routes { + endpoint.SetMTU(mtu) + } +} + +// Capabilities implements stack.LinkEndpoint. +func (m *InjectableEndpoint) Capabilities() stack.LinkEndpointCapabilities { + minCapabilities := stack.LinkEndpointCapabilities(^uint(0)) + for _, endpoint := range m.routes { + minCapabilities &= endpoint.Capabilities() + } + return minCapabilities +} + +// MaxHeaderLength implements stack.LinkEndpoint. +func (m *InjectableEndpoint) MaxHeaderLength() uint16 { + minHeaderLen := ^uint16(0) + for _, endpoint := range m.routes { + if headerLen := endpoint.MaxHeaderLength(); headerLen < minHeaderLen { + minHeaderLen = headerLen + } + } + return minHeaderLen +} + +// LinkAddress implements stack.LinkEndpoint. +func (m *InjectableEndpoint) LinkAddress() tcpip.LinkAddress { + return "" +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (m *InjectableEndpoint) SetLinkAddress(tcpip.LinkAddress) {} + +// Attach implements stack.LinkEndpoint. +func (m *InjectableEndpoint) Attach(dispatcher stack.NetworkDispatcher) { + for _, endpoint := range m.routes { + endpoint.Attach(dispatcher) + } + m.mu.Lock() + m.dispatcher = dispatcher + m.mu.Unlock() +} + +// IsAttached implements stack.LinkEndpoint. +func (m *InjectableEndpoint) IsAttached() bool { + m.mu.RLock() + defer m.mu.RUnlock() + return m.dispatcher != nil +} + +// InjectInbound implements stack.InjectableLinkEndpoint. +func (m *InjectableEndpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + m.mu.RLock() + d := m.dispatcher + m.mu.RUnlock() + d.DeliverNetworkPacket(protocol, pkt) +} + +// WritePackets writes outbound packets to the appropriate +// LinkInjectableEndpoint based on the RemoteAddress. HandleLocal only works if +// pkt.EgressRoute.RemoteAddress has a route registered in this endpoint. +func (m *InjectableEndpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + i := 0 + for _, pkt := range pkts.AsSlice() { + endpoint, ok := m.routes[pkt.EgressRoute.RemoteAddress] + if !ok { + return i, &tcpip.ErrHostUnreachable{} + } + + var tmpPkts stack.PacketBufferList + tmpPkts.PushBack(pkt) + + n, err := endpoint.WritePackets(tmpPkts) + if err != nil { + return i, err + } + + i += n + } + + return i, nil +} + +// InjectOutbound writes outbound packets to the appropriate +// LinkInjectableEndpoint based on the dest address. +func (m *InjectableEndpoint) InjectOutbound(dest tcpip.Address, packet *buffer.View) tcpip.Error { + endpoint, ok := m.routes[dest] + if !ok { + return &tcpip.ErrHostUnreachable{} + } + return endpoint.InjectOutbound(dest, packet) +} + +// Wait implements stack.LinkEndpoint.Wait. +func (m *InjectableEndpoint) Wait() { + for _, ep := range m.routes { + ep.Wait() + } +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (*InjectableEndpoint) ARPHardwareType() header.ARPHardwareType { + panic("unsupported operation") +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (*InjectableEndpoint) AddHeader(*stack.PacketBuffer) {} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (*InjectableEndpoint) ParseHeader(*stack.PacketBuffer) bool { return true } + +// Close implements stack.LinkEndpoint. +func (*InjectableEndpoint) Close() {} + +// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction. +func (*InjectableEndpoint) SetOnCloseAction(func()) {} + +// NewInjectableEndpoint creates a new multi-endpoint injectable endpoint. +func NewInjectableEndpoint(routes map[tcpip.Address]stack.InjectableLinkEndpoint) *InjectableEndpoint { + return &InjectableEndpoint{ + routes: routes, + } +} diff --git a/pkg/tcpip/link/muxed/muxed_state_autogen.go b/pkg/tcpip/link/muxed/muxed_state_autogen.go new file mode 100644 index 0000000..55b7b33 --- /dev/null +++ b/pkg/tcpip/link/muxed/muxed_state_autogen.go @@ -0,0 +1,41 @@ +// automatically generated by stateify. + +package muxed + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (m *InjectableEndpoint) StateTypeName() string { + return "pkg/tcpip/link/muxed.InjectableEndpoint" +} + +func (m *InjectableEndpoint) StateFields() []string { + return []string{ + "routes", + "dispatcher", + } +} + +func (m *InjectableEndpoint) beforeSave() {} + +// +checklocksignore +func (m *InjectableEndpoint) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.routes) + stateSinkObject.Save(1, &m.dispatcher) +} + +func (m *InjectableEndpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (m *InjectableEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.routes) + stateSourceObject.Load(1, &m.dispatcher) +} + +func init() { + state.Register((*InjectableEndpoint)(nil)) +} diff --git a/pkg/tcpip/link/nested/nested.go b/pkg/tcpip/link/nested/nested.go new file mode 100644 index 0000000..093ad4c --- /dev/null +++ b/pkg/tcpip/link/nested/nested.go @@ -0,0 +1,185 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nested provides helpers to implement the pattern of nested +// stack.LinkEndpoints. +package nested + +import ( + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// Endpoint is a wrapper around stack.LinkEndpoint and stack.NetworkDispatcher +// that can be used to implement nesting safely by providing lifecycle +// concurrency guards. +// +// See the tests in this package for example usage. +// +// +stateify savable +type Endpoint struct { + child stack.LinkEndpoint + embedder stack.NetworkDispatcher + + // mu protects dispatcher. + mu sync.RWMutex `state:"nosave"` + dispatcher stack.NetworkDispatcher +} + +var ( + _ stack.GSOEndpoint = (*Endpoint)(nil) + _ stack.LinkEndpoint = (*Endpoint)(nil) + _ stack.NetworkDispatcher = (*Endpoint)(nil) +) + +// Init initializes a nested.Endpoint that uses embedder as the dispatcher for +// child on Attach. +// +// See the tests in this package for example usage. +func (e *Endpoint) Init(child stack.LinkEndpoint, embedder stack.NetworkDispatcher) { + e.child = child + e.embedder = embedder +} + +// DeliverNetworkPacket implements stack.NetworkDispatcher. +func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverNetworkPacket(protocol, pkt) + } +} + +// DeliverLinkPacket implements stack.NetworkDispatcher. +func (e *Endpoint) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverLinkPacket(protocol, pkt) + } +} + +// Attach implements stack.LinkEndpoint. +func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + e.dispatcher = dispatcher + e.mu.Unlock() + // If we're attaching to a valid dispatcher, pass embedder as the dispatcher + // to our child, otherwise detach the child by giving it a nil dispatcher. + var pass stack.NetworkDispatcher + if dispatcher != nil { + pass = e.embedder + } + e.child.Attach(pass) +} + +// IsAttached implements stack.LinkEndpoint. +func (e *Endpoint) IsAttached() bool { + e.mu.RLock() + isAttached := e.dispatcher != nil + e.mu.RUnlock() + return isAttached +} + +// MTU implements stack.LinkEndpoint. +func (e *Endpoint) MTU() uint32 { + return e.child.MTU() +} + +// SetMTU implements stack.LinkEndpoint. +func (e *Endpoint) SetMTU(mtu uint32) { + e.child.SetMTU(mtu) +} + +// Capabilities implements stack.LinkEndpoint. +func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities { + return e.child.Capabilities() +} + +// MaxHeaderLength implements stack.LinkEndpoint. +func (e *Endpoint) MaxHeaderLength() uint16 { + return e.child.MaxHeaderLength() +} + +// LinkAddress implements stack.LinkEndpoint. +func (e *Endpoint) LinkAddress() tcpip.LinkAddress { + return e.child.LinkAddress() +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.child.SetLinkAddress(addr) +} + +// WritePackets implements stack.LinkEndpoint. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + return e.child.WritePackets(pkts) +} + +// Wait implements stack.LinkEndpoint. +func (e *Endpoint) Wait() { + e.child.Wait() +} + +// GSOMaxSize implements stack.GSOEndpoint. +func (e *Endpoint) GSOMaxSize() uint32 { + if e, ok := e.child.(stack.GSOEndpoint); ok { + return e.GSOMaxSize() + } + return 0 +} + +// SupportedGSO implements stack.GSOEndpoint. +func (e *Endpoint) SupportedGSO() stack.SupportedGSO { + if e, ok := e.child.(stack.GSOEndpoint); ok { + return e.SupportedGSO() + } + return stack.GSONotSupported +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType +func (e *Endpoint) ARPHardwareType() header.ARPHardwareType { + return e.child.ARPHardwareType() +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *Endpoint) AddHeader(pkt *stack.PacketBuffer) { + e.child.AddHeader(pkt) +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (e *Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + return e.child.ParseHeader(pkt) +} + +// Close implements stack.LinkEndpoint. +func (e *Endpoint) Close() { + e.child.Close() +} + +// SetOnCloseAction implement stack.LinkEndpoints. +func (e *Endpoint) SetOnCloseAction(action func()) { + e.child.SetOnCloseAction(action) +} + +// Child returns the child endpoint. +func (e *Endpoint) Child() stack.LinkEndpoint { + return e.child +} diff --git a/pkg/tcpip/link/nested/nested_state_autogen.go b/pkg/tcpip/link/nested/nested_state_autogen.go new file mode 100644 index 0000000..0ca3329 --- /dev/null +++ b/pkg/tcpip/link/nested/nested_state_autogen.go @@ -0,0 +1,44 @@ +// automatically generated by stateify. + +package nested + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/nested.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "child", + "embedder", + "dispatcher", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.child) + stateSinkObject.Save(1, &e.embedder) + stateSinkObject.Save(2, &e.dispatcher) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.child) + stateSourceObject.Load(1, &e.embedder) + stateSourceObject.Load(2, &e.dispatcher) +} + +func init() { + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/packetsocket/packetsocket.go b/pkg/tcpip/link/packetsocket/packetsocket.go new file mode 100644 index 0000000..058dc6d --- /dev/null +++ b/pkg/tcpip/link/packetsocket/packetsocket.go @@ -0,0 +1,62 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package packetsocket provides a link endpoint that enables delivery of +// incoming and outgoing packets to any interested packet sockets. +package packetsocket + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/link/nested" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var ( + _ stack.NetworkDispatcher = (*Endpoint)(nil) + _ stack.LinkEndpoint = (*Endpoint)(nil) +) + +// Endpoint is a link endpoint that enables delivery of incoming and outgoing +// packets to any interested packet sockets. +// +// +stateify savable +type Endpoint struct { + nested.Endpoint +} + +// New creates a new packetsocket link endpoint wrapping a lower link endpoint. +// +// On ingress, the lower link endpoint must only deliver packets that have +// a link-layer header set if one is required for the link. +func New(lower stack.LinkEndpoint) stack.LinkEndpoint { + e := &Endpoint{} + e.Endpoint.Init(lower, e) + return e +} + +// DeliverNetworkPacket implements stack.NetworkDispatcher. +func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.Endpoint.DeliverLinkPacket(protocol, pkt) + + e.Endpoint.DeliverNetworkPacket(protocol, pkt) +} + +// WritePackets implements stack.LinkEndpoint. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + for _, pkt := range pkts.AsSlice() { + e.Endpoint.DeliverLinkPacket(pkt.NetworkProtocolNumber, pkt) + } + + return e.Endpoint.WritePackets(pkts) +} diff --git a/pkg/tcpip/link/packetsocket/packetsocket_state_autogen.go b/pkg/tcpip/link/packetsocket/packetsocket_state_autogen.go new file mode 100644 index 0000000..3172064 --- /dev/null +++ b/pkg/tcpip/link/packetsocket/packetsocket_state_autogen.go @@ -0,0 +1,38 @@ +// automatically generated by stateify. + +package packetsocket + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/packetsocket.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "Endpoint", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.Endpoint) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.Endpoint) +} + +func init() { + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/pipe/endpoint_mutex.go b/pkg/tcpip/link/pipe/endpoint_mutex.go new file mode 100644 index 0000000..456122f --- /dev/null +++ b/pkg/tcpip/link/pipe/endpoint_mutex.go @@ -0,0 +1,96 @@ +package pipe + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/pipe/pipe.go b/pkg/tcpip/link/pipe/pipe.go new file mode 100644 index 0000000..fc89ecc --- /dev/null +++ b/pkg/tcpip/link/pipe/pipe.go @@ -0,0 +1,154 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pipe provides the implementation of pipe-like data-link layer +// endpoints. Such endpoints allow packets to be sent between two interfaces. +package pipe + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var _ stack.LinkEndpoint = (*Endpoint)(nil) + +// New returns both ends of a new pipe. +func New(linkAddr1, linkAddr2 tcpip.LinkAddress, mtu uint32) (*Endpoint, *Endpoint) { + ep1 := &Endpoint{ + linkAddr: linkAddr1, + mtu: mtu, + } + ep2 := &Endpoint{ + linkAddr: linkAddr2, + mtu: mtu, + } + ep1.linked = ep2 + ep2.linked = ep1 + return ep1, ep2 +} + +// Endpoint is one end of a pipe. +// +// +stateify savable +type Endpoint struct { + linked *Endpoint + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher + // +checklocks:mu + linkAddr tcpip.LinkAddress + // +checklocks:mu + mtu uint32 +} + +func (e *Endpoint) deliverPackets(pkts stack.PacketBufferList) { + e.linked.mu.RLock() + d := e.linked.dispatcher + e.linked.mu.RUnlock() + if d == nil { + return + } + + for _, pkt := range pkts.AsSlice() { + // Create a fresh packet with pkt's payload but without struct fields + // or headers set so the next link protocol can properly set the link + // header. + newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: pkt.ToBuffer(), + }) + d.DeliverNetworkPacket(pkt.NetworkProtocolNumber, newPkt) + newPkt.DecRef() + } +} + +// WritePackets implements stack.LinkEndpoint. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + n := pkts.Len() + e.deliverPackets(pkts) + return n, nil +} + +// Attach implements stack.LinkEndpoint. +func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + defer e.mu.Unlock() + e.dispatcher = dispatcher +} + +// IsAttached implements stack.LinkEndpoint. +func (e *Endpoint) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.dispatcher != nil +} + +// Wait implements stack.LinkEndpoint. +func (*Endpoint) Wait() {} + +// MTU implements stack.LinkEndpoint. +func (e *Endpoint) MTU() uint32 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mtu +} + +// SetMTU implements stack.LinkEndpoint. +func (e *Endpoint) SetMTU(mtu uint32) { + e.mu.Lock() + defer e.mu.Unlock() + e.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint. +func (*Endpoint) Capabilities() stack.LinkEndpointCapabilities { + return 0 +} + +// MaxHeaderLength implements stack.LinkEndpoint. +func (*Endpoint) MaxHeaderLength() uint16 { + return 0 +} + +// LinkAddress implements stack.LinkEndpoint. +func (e *Endpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.linkAddr +} + +// SetLinkAddress implements stack.LinkEndpoint. +func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.linkAddr = addr +} + +// ARPHardwareType implements stack.LinkEndpoint. +func (*Endpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareNone +} + +// AddHeader implements stack.LinkEndpoint. +func (*Endpoint) AddHeader(*stack.PacketBuffer) {} + +// ParseHeader implements stack.LinkEndpoint. +func (*Endpoint) ParseHeader(*stack.PacketBuffer) bool { return true } + +// Close implements stack.LinkEndpoint. +func (e *Endpoint) Close() {} + +// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction. +func (*Endpoint) SetOnCloseAction(func()) {} diff --git a/pkg/tcpip/link/pipe/pipe_state_autogen.go b/pkg/tcpip/link/pipe/pipe_state_autogen.go new file mode 100644 index 0000000..40f8b77 --- /dev/null +++ b/pkg/tcpip/link/pipe/pipe_state_autogen.go @@ -0,0 +1,47 @@ +// automatically generated by stateify. + +package pipe + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/pipe.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "linked", + "dispatcher", + "linkAddr", + "mtu", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.linked) + stateSinkObject.Save(1, &e.dispatcher) + stateSinkObject.Save(2, &e.linkAddr) + stateSinkObject.Save(3, &e.mtu) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.linked) + stateSourceObject.Load(1, &e.dispatcher) + stateSourceObject.Load(2, &e.linkAddr) + stateSourceObject.Load(3, &e.mtu) +} + +func init() { + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go b/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go new file mode 100644 index 0000000..557341a --- /dev/null +++ b/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go @@ -0,0 +1,64 @@ +package fifo + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type queueDispatcherMutex struct { + mu sync.Mutex +} + +var queueDispatcherprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var queueDispatcherlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type queueDispatcherlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *queueDispatcherMutex) Lock() { + locking.AddGLock(queueDispatcherprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *queueDispatcherMutex) NestedLock(i queueDispatcherlockNameIndex) { + locking.AddGLock(queueDispatcherprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *queueDispatcherMutex) Unlock() { + locking.DelGLock(queueDispatcherprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *queueDispatcherMutex) NestedUnlock(i queueDispatcherlockNameIndex) { + locking.DelGLock(queueDispatcherprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func queueDispatcherinitLockNames() {} + +func init() { + queueDispatcherinitLockNames() + queueDispatcherprefixIndex = locking.NewMutexClass(reflect.TypeOf(queueDispatcherMutex{}), queueDispatcherlockNames) +} diff --git a/pkg/tcpip/link/qdisc/fifo/fifo.go b/pkg/tcpip/link/qdisc/fifo/fifo.go new file mode 100644 index 0000000..fe3e135 --- /dev/null +++ b/pkg/tcpip/link/qdisc/fifo/fifo.go @@ -0,0 +1,158 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fifo provides the implementation of FIFO queuing discipline that +// queues all outbound packets and asynchronously dispatches them to the +// lower link endpoint in the order that they were queued. +package fifo + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/sleep" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var _ stack.QueueingDiscipline = (*discipline)(nil) + +const ( + // BatchSize is the number of packets to write in each syscall. It is 47 + // because when GVisorGSO is in use then a single 65KB TCP segment can get + // split into 46 segments of 1420 bytes and a single 216 byte segment. + BatchSize = 47 + + qDiscClosed = 1 +) + +// discipline represents a QueueingDiscipline which implements a FIFO queue for +// all outgoing packets. discipline can have 1 or more underlying +// queueDispatchers. All outgoing packets are consistently hashed to a single +// underlying queue using the PacketBuffer.Hash if set, otherwise all packets +// are queued to the first queue to avoid reordering in case of missing hash. +// +// +stateify savable +type discipline struct { + wg sync.WaitGroup `state:"nosave"` + dispatchers []queueDispatcher + + closed atomicbitops.Int32 +} + +// queueDispatcher is responsible for dispatching all outbound packets in its +// queue. It will also smartly batch packets when possible and write them +// through the lower LinkWriter. +// +// +stateify savable +type queueDispatcher struct { + lower stack.LinkWriter + + mu queueDispatcherMutex `state:"nosave"` + // +checklocks:mu + queue packetBufferCircularList + + newPacketWaker sleep.Waker `state:"nosave"` + closeWaker sleep.Waker `state:"nosave"` +} + +// New creates a new fifo queuing discipline with the n queues with maximum +// capacity of queueLen. +// +// +checklocksignore: we don't have to hold locks during initialization. +func New(lower stack.LinkWriter, n int, queueLen int) stack.QueueingDiscipline { + d := &discipline{ + dispatchers: make([]queueDispatcher, n), + } + // Create the required dispatchers + for i := range d.dispatchers { + qd := &d.dispatchers[i] + qd.lower = lower + qd.queue.init(queueLen) + + d.wg.Add(1) + go func() { + defer d.wg.Done() + qd.dispatchLoop() + }() + } + return d +} + +func (qd *queueDispatcher) dispatchLoop() { + s := sleep.Sleeper{} + s.AddWaker(&qd.newPacketWaker) + s.AddWaker(&qd.closeWaker) + defer s.Done() + + var batch stack.PacketBufferList + for { + switch w := s.Fetch(true); w { + case &qd.newPacketWaker: + case &qd.closeWaker: + qd.mu.Lock() + for p := qd.queue.removeFront(); p != nil; p = qd.queue.removeFront() { + p.DecRef() + } + qd.queue.decRef() + qd.mu.Unlock() + return + default: + panic("unknown waker") + } + qd.mu.Lock() + for pkt := qd.queue.removeFront(); pkt != nil; pkt = qd.queue.removeFront() { + batch.PushBack(pkt) + if batch.Len() < BatchSize && !qd.queue.isEmpty() { + continue + } + qd.mu.Unlock() + _, _ = qd.lower.WritePackets(batch) + batch.Reset() + qd.mu.Lock() + } + qd.mu.Unlock() + } +} + +// WritePacket implements stack.QueueingDiscipline.WritePacket. +// +// The packet must have the following fields populated: +// - pkt.EgressRoute +// - pkt.GSOOptions +// - pkt.NetworkProtocolNumber +func (d *discipline) WritePacket(pkt *stack.PacketBuffer) tcpip.Error { + if d.closed.Load() == qDiscClosed { + return &tcpip.ErrClosedForSend{} + } + qd := &d.dispatchers[int(pkt.Hash)%len(d.dispatchers)] + qd.mu.Lock() + haveSpace := qd.queue.hasSpace() + if haveSpace { + qd.queue.pushBack(pkt.IncRef()) + } + qd.mu.Unlock() + if !haveSpace { + return &tcpip.ErrNoBufferSpace{} + } + qd.newPacketWaker.Assert() + return nil +} + +func (d *discipline) Close() { + d.closed.Store(qDiscClosed) + for i := range d.dispatchers { + d.dispatchers[i].closeWaker.Assert() + } + d.wg.Wait() +} diff --git a/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go b/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go new file mode 100644 index 0000000..5dccdd1 --- /dev/null +++ b/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go @@ -0,0 +1,102 @@ +// automatically generated by stateify. + +package fifo + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (d *discipline) StateTypeName() string { + return "pkg/tcpip/link/qdisc/fifo.discipline" +} + +func (d *discipline) StateFields() []string { + return []string{ + "dispatchers", + "closed", + } +} + +func (d *discipline) beforeSave() {} + +// +checklocksignore +func (d *discipline) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.dispatchers) + stateSinkObject.Save(1, &d.closed) +} + +func (d *discipline) afterLoad(context.Context) {} + +// +checklocksignore +func (d *discipline) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.dispatchers) + stateSourceObject.Load(1, &d.closed) +} + +func (qd *queueDispatcher) StateTypeName() string { + return "pkg/tcpip/link/qdisc/fifo.queueDispatcher" +} + +func (qd *queueDispatcher) StateFields() []string { + return []string{ + "lower", + "queue", + } +} + +func (qd *queueDispatcher) beforeSave() {} + +// +checklocksignore +func (qd *queueDispatcher) StateSave(stateSinkObject state.Sink) { + qd.beforeSave() + stateSinkObject.Save(0, &qd.lower) + stateSinkObject.Save(1, &qd.queue) +} + +func (qd *queueDispatcher) afterLoad(context.Context) {} + +// +checklocksignore +func (qd *queueDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &qd.lower) + stateSourceObject.Load(1, &qd.queue) +} + +func (pl *packetBufferCircularList) StateTypeName() string { + return "pkg/tcpip/link/qdisc/fifo.packetBufferCircularList" +} + +func (pl *packetBufferCircularList) StateFields() []string { + return []string{ + "pbs", + "head", + "size", + } +} + +func (pl *packetBufferCircularList) beforeSave() {} + +// +checklocksignore +func (pl *packetBufferCircularList) StateSave(stateSinkObject state.Sink) { + pl.beforeSave() + stateSinkObject.Save(0, &pl.pbs) + stateSinkObject.Save(1, &pl.head) + stateSinkObject.Save(2, &pl.size) +} + +func (pl *packetBufferCircularList) afterLoad(context.Context) {} + +// +checklocksignore +func (pl *packetBufferCircularList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &pl.pbs) + stateSourceObject.Load(1, &pl.head) + stateSourceObject.Load(2, &pl.size) +} + +func init() { + state.Register((*discipline)(nil)) + state.Register((*queueDispatcher)(nil)) + state.Register((*packetBufferCircularList)(nil)) +} diff --git a/pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go b/pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go new file mode 100644 index 0000000..27ce701 --- /dev/null +++ b/pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go @@ -0,0 +1,93 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fifo + +import "github.com/sagernet/gvisor/pkg/tcpip/stack" + +// packetBufferCircularList is a slice-backed circular list. All operations are +// O(1) unless otherwise noted. It only allocates once, during the call to +// init(). +// +// Users should call init() before using packetBufferCircularList. +// +// +stateify savable +type packetBufferCircularList struct { + pbs []*stack.PacketBuffer + head int + size int +} + +// init initializes the list with the given size. +func (pl *packetBufferCircularList) init(size int) { + pl.pbs = make([]*stack.PacketBuffer, size) +} + +// length returns the number of elements in the list. +// +//go:nosplit +func (pl *packetBufferCircularList) length() int { + return pl.size +} + +// hasSpace returns whether there is space left in the list. +// +//go:nosplit +func (pl *packetBufferCircularList) hasSpace() bool { + return pl.size < len(pl.pbs) +} + +// isEmpty returns whether the list is empty. +// +//go:nosplit +func (pl *packetBufferCircularList) isEmpty() bool { + return pl.size == 0 +} + +// pushBack inserts the PacketBuffer at the end of the list. +// +// Users must check beforehand that there is space via a call to hasSpace(). +// Failing to do so may clobber existing entries. +// +//go:nosplit +func (pl *packetBufferCircularList) pushBack(pb *stack.PacketBuffer) { + next := (pl.head + pl.size) % len(pl.pbs) + pl.pbs[next] = pb + pl.size++ +} + +// removeFront returns the first element of the list or nil. +// +//go:nosplit +func (pl *packetBufferCircularList) removeFront() *stack.PacketBuffer { + if pl.isEmpty() { + return nil + } + ret := pl.pbs[pl.head] + pl.pbs[pl.head] = nil + pl.head = (pl.head + 1) % len(pl.pbs) + pl.size-- + return ret +} + +// decRef decreases the reference count on each stack.PacketBuffer stored in +// the list. +// +// NOTE: runs in O(n) time. +// +//go:nosplit +func (pl *packetBufferCircularList) decRef() { + for i := 0; i < pl.size; i++ { + pl.pbs[(pl.head+i)%len(pl.pbs)].DecRef() + } +} diff --git a/pkg/tcpip/link/sharedmem/endpoint_mutex.go b/pkg/tcpip/link/sharedmem/endpoint_mutex.go new file mode 100644 index 0000000..e970f75 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/endpoint_mutex.go @@ -0,0 +1,96 @@ +package sharedmem + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/sharedmem/pipe/pipe.go b/pkg/tcpip/link/sharedmem/pipe/pipe.go new file mode 100644 index 0000000..75a8d8f --- /dev/null +++ b/pkg/tcpip/link/sharedmem/pipe/pipe.go @@ -0,0 +1,80 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pipe implements a shared memory ring buffer on which a single reader +// and a single writer can operate (read/write) concurrently. The ring buffer +// allows for data of different sizes to be written, and preserves the boundary +// of the written data. +// +// Example usage is as follows: +// +// wb := t.Push(20) +// // Write data to wb. +// t.Flush() +// +// rb := r.Pull() +// // Do something with data in rb. +// t.Flush() +package pipe + +import ( + "math" +) + +const ( + jump uint64 = math.MaxUint32 + 1 + offsetMask uint64 = math.MaxUint32 + revolutionMask uint64 = ^offsetMask + + sizeOfSlotHeader = 8 // sizeof(uint64) + slotFree uint64 = 1 << 63 + slotSizeMask uint64 = math.MaxUint32 +) + +// payloadToSlotSize calculates the total size of a slot based on its payload +// size. The total size is the header size, plus the payload size, plus padding +// if necessary to make the total size a multiple of sizeOfSlotHeader. +func payloadToSlotSize(payloadSize uint64) uint64 { + s := sizeOfSlotHeader + payloadSize + return (s + sizeOfSlotHeader - 1) &^ (sizeOfSlotHeader - 1) +} + +// slotToPayloadSize calculates the payload size of a slot based on the total +// size of the slot. This is only meant to be used when creating slots that +// don't carry information (e.g., free slots or wrap slots). +func slotToPayloadSize(offset uint64) uint64 { + return offset - sizeOfSlotHeader +} + +// pipe is a basic data structure used by both (transmit & receive) ends of a +// pipe. Indices into this pipe are split into two fields: offset, which counts +// the number of bytes from the beginning of the buffer, and revolution, which +// counts the number of times the index has wrapped around. +// +// +stateify savable +type pipe struct { + buffer []byte +} + +// init initializes the pipe buffer such that its size is a multiple of the size +// of the slot header. +func (p *pipe) init(b []byte) { + p.buffer = b[:len(b)&^(sizeOfSlotHeader-1)] +} + +// data returns a section of the buffer starting at the given index (which may +// include revolution information) and with the given size. +func (p *pipe) data(idx uint64, size uint64) []byte { + return p.buffer[(idx&offsetMask)+sizeOfSlotHeader:][:size] +} diff --git a/pkg/tcpip/link/sharedmem/pipe/pipe_state_autogen.go b/pkg/tcpip/link/sharedmem/pipe/pipe_state_autogen.go new file mode 100644 index 0000000..17ec121 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/pipe/pipe_state_autogen.go @@ -0,0 +1,111 @@ +// automatically generated by stateify. + +package pipe + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *pipe) StateTypeName() string { + return "pkg/tcpip/link/sharedmem/pipe.pipe" +} + +func (p *pipe) StateFields() []string { + return []string{ + "buffer", + } +} + +func (p *pipe) beforeSave() {} + +// +checklocksignore +func (p *pipe) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.buffer) +} + +func (p *pipe) afterLoad(context.Context) {} + +// +checklocksignore +func (p *pipe) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.buffer) +} + +func (r *Rx) StateTypeName() string { + return "pkg/tcpip/link/sharedmem/pipe.Rx" +} + +func (r *Rx) StateFields() []string { + return []string{ + "p", + "tail", + "head", + } +} + +func (r *Rx) beforeSave() {} + +// +checklocksignore +func (r *Rx) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.p) + stateSinkObject.Save(1, &r.tail) + stateSinkObject.Save(2, &r.head) +} + +func (r *Rx) afterLoad(context.Context) {} + +// +checklocksignore +func (r *Rx) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.p) + stateSourceObject.Load(1, &r.tail) + stateSourceObject.Load(2, &r.head) +} + +func (t *Tx) StateTypeName() string { + return "pkg/tcpip/link/sharedmem/pipe.Tx" +} + +func (t *Tx) StateFields() []string { + return []string{ + "p", + "maxPayloadSize", + "head", + "tail", + "next", + "tailHeader", + } +} + +func (t *Tx) beforeSave() {} + +// +checklocksignore +func (t *Tx) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.p) + stateSinkObject.Save(1, &t.maxPayloadSize) + stateSinkObject.Save(2, &t.head) + stateSinkObject.Save(3, &t.tail) + stateSinkObject.Save(4, &t.next) + stateSinkObject.Save(5, &t.tailHeader) +} + +func (t *Tx) afterLoad(context.Context) {} + +// +checklocksignore +func (t *Tx) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.p) + stateSourceObject.Load(1, &t.maxPayloadSize) + stateSourceObject.Load(2, &t.head) + stateSourceObject.Load(3, &t.tail) + stateSourceObject.Load(4, &t.next) + stateSourceObject.Load(5, &t.tailHeader) +} + +func init() { + state.Register((*pipe)(nil)) + state.Register((*Rx)(nil)) + state.Register((*Tx)(nil)) +} diff --git a/pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go b/pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go new file mode 100644 index 0000000..2b3d05c --- /dev/null +++ b/pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go @@ -0,0 +1,36 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipe + +import ( + "unsafe" + + "github.com/sagernet/gvisor/pkg/atomicbitops" +) + +func (p *pipe) write(idx uint64, v uint64) { + ptr := (*uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0])) + *ptr = v +} + +func (p *pipe) writeAtomic(idx uint64, v uint64) { + ptr := (*atomicbitops.Uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0])) + ptr.Store(v) +} + +func (p *pipe) readAtomic(idx uint64) uint64 { + ptr := (*atomicbitops.Uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0])) + return ptr.Load() +} diff --git a/pkg/tcpip/link/sharedmem/pipe/pipe_unsafe_state_autogen.go b/pkg/tcpip/link/sharedmem/pipe/pipe_unsafe_state_autogen.go new file mode 100644 index 0000000..d3b40fe --- /dev/null +++ b/pkg/tcpip/link/sharedmem/pipe/pipe_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package pipe diff --git a/pkg/tcpip/link/sharedmem/pipe/rx.go b/pkg/tcpip/link/sharedmem/pipe/rx.go new file mode 100644 index 0000000..5257010 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/pipe/rx.go @@ -0,0 +1,108 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipe + +// Rx is the receive side of the shared memory ring buffer. +// +// +stateify savable +type Rx struct { + p pipe + + tail uint64 + head uint64 +} + +// Init initializes the receive end of the pipe. In the initial state, the next +// slot to be inspected is the very first one. +func (r *Rx) Init(b []byte) { + r.p.init(b) + r.tail = 0xfffffffe * jump + r.head = r.tail +} + +// Pull reads the next buffer from the pipe, returning nil if there isn't one +// currently available. +// +// The returned slice is available until Flush() is next called. After that, it +// must not be touched. +func (r *Rx) Pull() []byte { + if r.head == r.tail+jump { + // We've already pulled the whole pipe. + return nil + } + + header := r.p.readAtomic(r.head) + if header&slotFree != 0 { + // The next slot is free, we can't pull it yet. + return nil + } + + payloadSize := header & slotSizeMask + newHead := r.head + payloadToSlotSize(payloadSize) + headWrap := (r.head & revolutionMask) | uint64(len(r.p.buffer)) + + // Check if this is a wrapping slot. If that's the case, it carries no + // data, so we just skip it and try again from the first slot. + if int64(newHead-headWrap) >= 0 { + // If newHead passes the tail, the pipe is either damaged or the + // RX view of the pipe has completely wrapped without an + // intervening flush. + if int64(newHead-(r.tail+jump)) > 0 { + return nil + } + // The pipe is damaged if newHead doesn't point to the start of + // the ring. + if newHead&offsetMask != 0 { + return nil + } + + if r.tail == r.head { + // If this is the first pull since the last Flush() + // call, we flush the state so that the sender can use + // this space if it needs to. + r.p.writeAtomic(r.head, slotFree|slotToPayloadSize(newHead-r.head)) + r.tail = newHead + } + + r.head = newHead + return r.Pull() + } + + // Grab the buffer before updating r.head. + b := r.p.data(r.head, payloadSize) + r.head = newHead + return b +} + +// Flush tells the transmitter that all buffers pulled since the last Flush() +// have been used, so the transmitter is free to used their slots for further +// transmission. +func (r *Rx) Flush() { + if r.head == r.tail { + return + } + r.p.writeAtomic(r.tail, slotFree|slotToPayloadSize(r.head-r.tail)) + r.tail = r.head +} + +// Abort unpulls any pulled buffers. +func (r *Rx) Abort() { + r.head = r.tail +} + +// Bytes returns the byte slice on which the pipe operates. +func (r *Rx) Bytes() []byte { + return r.p.buffer +} diff --git a/pkg/tcpip/link/sharedmem/pipe/tx.go b/pkg/tcpip/link/sharedmem/pipe/tx.go new file mode 100644 index 0000000..2a14d5d --- /dev/null +++ b/pkg/tcpip/link/sharedmem/pipe/tx.go @@ -0,0 +1,164 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipe + +// Tx is the transmit side of the shared memory ring buffer. +// +// +stateify savable +type Tx struct { + p pipe + maxPayloadSize uint64 + + head uint64 + tail uint64 + next uint64 + + tailHeader uint64 +} + +// Init initializes the transmit end of the pipe. In the initial state, the next +// slot to be written is the very first one, and the transmitter has the whole +// ring buffer available to it. +func (t *Tx) Init(b []byte) { + t.p.init(b) + // maxPayloadSize excludes the header of the payload, and the header + // of the wrapping message. + t.maxPayloadSize = uint64(len(t.p.buffer)) - 2*sizeOfSlotHeader + t.tail = 0xfffffffe * jump + t.next = t.tail + t.head = t.tail + jump + t.p.write(t.tail, slotFree) +} + +// Capacity determines how many records of the given size can be written to the +// pipe before it fills up. +func (t *Tx) Capacity(recordSize uint64) uint64 { + available := uint64(len(t.p.buffer)) - sizeOfSlotHeader + entryLen := payloadToSlotSize(recordSize) + return available / entryLen +} + +// Push reserves "payloadSize" bytes for transmission in the pipe. The caller +// populates the returned slice with the data to be transferred and enventually +// calls Flush() to make the data visible to the reader, or Abort() to make the +// pipe forget all Push() calls since the last Flush(). +// +// The returned slice is available until Flush() or Abort() is next called. +// After that, it must not be touched. +func (t *Tx) Push(payloadSize uint64) []byte { + // Fail request if we know we will never have enough room. + if payloadSize > t.maxPayloadSize { + return nil + } + + // True if TxPipe currently has a pushed message, i.e., it is not + // Flush()'ed. + messageAhead := t.next != t.tail + totalLen := payloadToSlotSize(payloadSize) + newNext := t.next + totalLen + nextWrap := (t.next & revolutionMask) | uint64(len(t.p.buffer)) + if int64(newNext-nextWrap) >= 0 { + // The new buffer would overflow the pipe, so we push a wrapping + // slot, then try to add the actual slot to the front of the + // pipe. + newNext = (newNext & revolutionMask) + jump + if !t.reclaim(newNext) { + return nil + } + wrappingPayloadSize := slotToPayloadSize(newNext - t.next) + oldNext := t.next + t.next = newNext + if messageAhead { + t.p.write(oldNext, wrappingPayloadSize) + } else { + t.tailHeader = wrappingPayloadSize + t.Flush() + } + return t.Push(payloadSize) + } + + // Check that we have enough room for the buffer. + if !t.reclaim(newNext) { + return nil + } + + if messageAhead { + t.p.write(t.next, payloadSize) + } else { + t.tailHeader = payloadSize + } + + // Grab the buffer before updating t.next. + b := t.p.data(t.next, payloadSize) + t.next = newNext + + return b +} + +// reclaim attempts to advance the head until at least newNext. If the head is +// already at or beyond newNext, nothing happens and true is returned; otherwise +// it tries to reclaim slots that have already been consumed by the receive end +// of the pipe (they will be marked as free) and returns a boolean indicating +// whether it was successful in reclaiming enough slots. +func (t *Tx) reclaim(newNext uint64) bool { + for int64(newNext-t.head) > 0 { + // Can't reclaim if slot is not free. + header := t.p.readAtomic(t.head) + if header&slotFree == 0 { + return false + } + + payloadSize := header & slotSizeMask + newHead := t.head + payloadToSlotSize(payloadSize) + + // Check newHead is within bounds and valid. + if int64(newHead-t.tail) > int64(jump) || newHead&offsetMask >= uint64(len(t.p.buffer)) { + return false + } + + t.head = newHead + } + + return true +} + +// Abort causes all Push() calls since the last Flush() to be forgotten and +// therefore they will not be made visible to the receiver. +func (t *Tx) Abort() { + t.next = t.tail +} + +// Flush causes all buffers pushed since the last Flush() [or Abort(), whichever +// is the most recent] to be made visible to the receiver. +func (t *Tx) Flush() { + if t.next == t.tail { + // Nothing to do if there are no pushed buffers. + return + } + + if t.next != t.head { + // The receiver will spin in t.next, so we must make sure that + // the slotFree bit is set. + t.p.write(t.next, slotFree) + } + + t.p.writeAtomic(t.tail, t.tailHeader) + t.tail = t.next +} + +// Bytes returns the byte slice on which the pipe operates. +func (t *Tx) Bytes() []byte { + return t.p.buffer +} diff --git a/pkg/tcpip/link/sharedmem/queue/queue_state_autogen.go b/pkg/tcpip/link/sharedmem/queue/queue_state_autogen.go new file mode 100644 index 0000000..563d4fb --- /dev/null +++ b/pkg/tcpip/link/sharedmem/queue/queue_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package queue diff --git a/pkg/tcpip/link/sharedmem/queue/rx.go b/pkg/tcpip/link/sharedmem/queue/rx.go new file mode 100644 index 0000000..b4e8954 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/queue/rx.go @@ -0,0 +1,226 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package queue provides the implementation of transmit and receive queues +// based on shared memory ring buffers. +package queue + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe" +) + +const ( + // Offsets within a posted buffer. + postedOffset = 0 + postedSize = 8 + postedRemainingInGroup = 12 + postedUserData = 16 + postedID = 24 + + sizeOfPostedBuffer = 32 + + // Offsets within a received packet header. + consumedPacketSize = 0 + consumedPacketReserved = 4 + + sizeOfConsumedPacketHeader = 8 + + // Offsets within a consumed buffer. + consumedOffset = 0 + consumedSize = 8 + consumedUserData = 12 + consumedID = 20 + + sizeOfConsumedBuffer = 28 + + // The following are the allowed states of the shared data area. + // EventFDUinitialized is the value stored at the start of the shared data + // region when it hasn't been initialized. + EventFDUninitialized = 0 + // EventFDDisabled is the value stored at the start of the shared data region + // when notifications using eventFD has been disabled. + EventFDDisabled = 1 + // EventFDEnabled is the value stored at the start of the shared data region + // when eventFD should be notified as the peer might be blocked waiting on + // notifications. + EventFDEnabled = 2 +) + +// RxBuffer is the descriptor of a receive buffer. +type RxBuffer struct { + Offset uint64 + Size uint32 + ID uint64 + UserData uint64 +} + +// Rx is a receive queue. It is implemented with one tx and one rx pipe: the tx +// pipe is used to "post" buffers, while the rx pipe is used to receive packets +// whose contents have been written to previously posted buffers. +// +// This struct is thread-compatible. +type Rx struct { + tx pipe.Tx + rx pipe.Rx + sharedEventFDState *atomicbitops.Uint32 +} + +// Init initializes the receive queue with the given pipes, and shared state +// pointer -- the latter is used to enable/disable eventfd notifications. +func (r *Rx) Init(tx, rx []byte, sharedEventFDState *atomicbitops.Uint32) { + r.sharedEventFDState = sharedEventFDState + r.tx.Init(tx) + r.rx.Init(rx) +} + +// EnableNotification updates the shared state such that the peer will notify +// the eventfd when there are packets to be dequeued. +func (r *Rx) EnableNotification() { + r.sharedEventFDState.Store(EventFDEnabled) +} + +// DisableNotification updates the shared state such that the peer will not +// notify the eventfd. +func (r *Rx) DisableNotification() { + r.sharedEventFDState.Store(EventFDDisabled) +} + +// PostedBuffersLimit returns the maximum number of buffers that can be posted +// before the tx queue fills up. +func (r *Rx) PostedBuffersLimit() uint64 { + return r.tx.Capacity(sizeOfPostedBuffer) +} + +// PostBuffers makes the given buffers available for receiving data from the +// peer. Once they are posted, the peer is free to write to them and will +// eventually post them back for consumption. +func (r *Rx) PostBuffers(buffers []RxBuffer) bool { + for i := range buffers { + b := r.tx.Push(sizeOfPostedBuffer) + if b == nil { + r.tx.Abort() + return false + } + + pb := &buffers[i] + binary.LittleEndian.PutUint64(b[postedOffset:], pb.Offset) + binary.LittleEndian.PutUint32(b[postedSize:], pb.Size) + binary.LittleEndian.PutUint32(b[postedRemainingInGroup:], 0) + binary.LittleEndian.PutUint64(b[postedUserData:], pb.UserData) + binary.LittleEndian.PutUint64(b[postedID:], pb.ID) + } + + r.tx.Flush() + return true +} + +// Dequeue receives buffers that have been previously posted by PostBuffers() +// and that have been filled by the peer and posted back. +// +// This is similar to append() in that new buffers are appended to "bufs", with +// reallocation only if "bufs" doesn't have enough capacity. +func (r *Rx) Dequeue(bufs []RxBuffer) ([]RxBuffer, uint32) { + for { + outBufs := bufs + // Pull the next descriptor from the rx pipe. + b := r.rx.Pull() + if b == nil { + return bufs, 0 + } + + if len(b) < sizeOfConsumedPacketHeader { + log.Warningf("Ignoring packet header: size (%v) is less than header size (%v)", len(b), sizeOfConsumedPacketHeader) + r.rx.Flush() + continue + } + + totalDataSize := binary.LittleEndian.Uint32(b[consumedPacketSize:]) + + // Calculate the number of buffer descriptors and copy them + // over to the output. + count := (len(b) - sizeOfConsumedPacketHeader) / sizeOfConsumedBuffer + offset := sizeOfConsumedPacketHeader + buffersSize := uint32(0) + for i := count; i > 0; i-- { + s := binary.LittleEndian.Uint32(b[offset+consumedSize:]) + buffersSize += s + if buffersSize < s { + // The buffer size overflows an unsigned 32-bit + // integer, so break out and force it to be + // ignored. + totalDataSize = 1 + buffersSize = 0 + break + } + + outBufs = append(outBufs, RxBuffer{ + Offset: binary.LittleEndian.Uint64(b[offset+consumedOffset:]), + Size: s, + ID: binary.LittleEndian.Uint64(b[offset+consumedID:]), + }) + + offset += sizeOfConsumedBuffer + } + + r.rx.Flush() + + if buffersSize < totalDataSize { + // The descriptor is corrupted, ignore it. + log.Warningf("Ignoring packet: actual data size (%v) less than expected size (%v)", buffersSize, totalDataSize) + continue + } + + return outBufs, totalDataSize + } +} + +// Bytes returns the byte slices on which the queue operates. +func (r *Rx) Bytes() (tx, rx []byte) { + return r.tx.Bytes(), r.rx.Bytes() +} + +// DecodeRxBufferHeader decodes the header of a buffer posted on an rx queue. +func DecodeRxBufferHeader(b []byte) RxBuffer { + return RxBuffer{ + Offset: binary.LittleEndian.Uint64(b[postedOffset:]), + Size: binary.LittleEndian.Uint32(b[postedSize:]), + ID: binary.LittleEndian.Uint64(b[postedID:]), + UserData: binary.LittleEndian.Uint64(b[postedUserData:]), + } +} + +// RxCompletionSize returns the number of bytes needed to encode an rx +// completion containing "count" buffers. +func RxCompletionSize(count int) uint64 { + return sizeOfConsumedPacketHeader + uint64(count)*sizeOfConsumedBuffer +} + +// EncodeRxCompletion encodes an rx completion header. +func EncodeRxCompletion(b []byte, size, reserved uint32) { + binary.LittleEndian.PutUint32(b[consumedPacketSize:], size) + binary.LittleEndian.PutUint32(b[consumedPacketReserved:], reserved) +} + +// EncodeRxCompletionBuffer encodes the i-th rx completion buffer header. +func EncodeRxCompletionBuffer(b []byte, i int, rxb RxBuffer) { + b = b[RxCompletionSize(i):] + binary.LittleEndian.PutUint64(b[consumedOffset:], rxb.Offset) + binary.LittleEndian.PutUint32(b[consumedSize:], rxb.Size) + binary.LittleEndian.PutUint64(b[consumedUserData:], rxb.UserData) + binary.LittleEndian.PutUint64(b[consumedID:], rxb.ID) +} diff --git a/pkg/tcpip/link/sharedmem/queue/tx.go b/pkg/tcpip/link/sharedmem/queue/tx.go new file mode 100644 index 0000000..196c880 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/queue/tx.go @@ -0,0 +1,161 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe" +) + +const ( + // Offsets within a packet header. + packetID = 0 + packetSize = 8 + packetReserved = 12 + + sizeOfPacketHeader = 16 + + // Offsets with a buffer descriptor + bufferOffset = 0 + bufferSize = 8 + + sizeOfBufferDescriptor = 12 +) + +// TxBuffer is the descriptor of a transmit buffer. +type TxBuffer struct { + Next *TxBuffer + Offset uint64 + Size uint32 +} + +// Tx is a transmit queue. It is implemented with one tx and one rx pipe: the +// tx pipe is used to request the transmission of packets, while the rx pipe +// is used to receive which transmissions have completed. +// +// This struct is thread-compatible. +type Tx struct { + tx pipe.Tx + rx pipe.Rx + sharedEventFDState *atomicbitops.Uint32 +} + +// Init initializes the transmit queue with the given pipes. +func (t *Tx) Init(tx, rx []byte, sharedEventFDState *atomicbitops.Uint32) { + t.tx.Init(tx) + t.rx.Init(rx) + t.sharedEventFDState = sharedEventFDState +} + +// NotificationsEnabled returns true if eventFD should be used to notify the +// peer of events (eg. packet transmit etc). +func (t *Tx) NotificationsEnabled() bool { + // Notifications are considered enabled unless explicitly disabled. + return t.sharedEventFDState.Load() != EventFDDisabled +} + +// Enqueue queues the given linked list of buffers for transmission as one +// packet. While it is queued, the caller must not modify them. +func (t *Tx) Enqueue(id uint64, totalDataLen, bufferCount uint32, buffer *TxBuffer) bool { + // Reserve room in the tx pipe. + totalLen := sizeOfPacketHeader + uint64(bufferCount)*sizeOfBufferDescriptor + + b := t.tx.Push(totalLen) + if b == nil { + return false + } + + // Initialize the packet and buffer descriptors. + binary.LittleEndian.PutUint64(b[packetID:], id) + binary.LittleEndian.PutUint32(b[packetSize:], totalDataLen) + binary.LittleEndian.PutUint32(b[packetReserved:], 0) + + offset := sizeOfPacketHeader + for i := bufferCount; i != 0; i-- { + binary.LittleEndian.PutUint64(b[offset+bufferOffset:], buffer.Offset) + binary.LittleEndian.PutUint32(b[offset+bufferSize:], buffer.Size) + offset += sizeOfBufferDescriptor + buffer = buffer.Next + } + + t.tx.Flush() + + return true +} + +// CompletedPacket returns the id of the last completed transmission. The +// returned id, if any, refers to a value passed on a previous call to +// Enqueue(). +func (t *Tx) CompletedPacket() (id uint64, ok bool) { + for { + b := t.rx.Pull() + if b == nil { + return 0, false + } + + if len(b) != 8 { + t.rx.Flush() + log.Warningf("Ignoring completed packet: size (%v) is less than expected (%v)", len(b), 8) + continue + } + + v := binary.LittleEndian.Uint64(b) + + t.rx.Flush() + + return v, true + } +} + +// Bytes returns the byte slices on which the queue operates. +func (t *Tx) Bytes() (tx, rx []byte) { + return t.tx.Bytes(), t.rx.Bytes() +} + +// TxPacketInfo holds information about a packet sent on a tx queue. +type TxPacketInfo struct { + ID uint64 + Size uint32 + Reserved uint32 + BufferCount int +} + +// DecodeTxPacketHeader decodes the header of a packet sent over a tx queue. +func DecodeTxPacketHeader(b []byte) TxPacketInfo { + return TxPacketInfo{ + ID: binary.LittleEndian.Uint64(b[packetID:]), + Size: binary.LittleEndian.Uint32(b[packetSize:]), + Reserved: binary.LittleEndian.Uint32(b[packetReserved:]), + BufferCount: (len(b) - sizeOfPacketHeader) / sizeOfBufferDescriptor, + } +} + +// DecodeTxBufferHeader decodes the header of the i-th buffer of a packet sent +// over a tx queue. +func DecodeTxBufferHeader(b []byte, i int) TxBuffer { + b = b[sizeOfPacketHeader+i*sizeOfBufferDescriptor:] + return TxBuffer{ + Offset: binary.LittleEndian.Uint64(b[bufferOffset:]), + Size: binary.LittleEndian.Uint32(b[bufferSize:]), + } +} + +// EncodeTxCompletion encodes a tx completion header. +func EncodeTxCompletion(b []byte, id uint64) { + binary.LittleEndian.PutUint64(b, id) +} diff --git a/pkg/tcpip/link/sharedmem/queuepair.go b/pkg/tcpip/link/sharedmem/queuepair.go new file mode 100644 index 0000000..64b26e2 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/queuepair.go @@ -0,0 +1,220 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sharedmem + +import ( + "fmt" + "os" + + "github.com/sagernet/gvisor/pkg/eventfd" + "golang.org/x/sys/unix" +) + +const ( + // DefaultQueueDataSize is the size of the shared memory data region that + // holds the scatter/gather buffers. + DefaultQueueDataSize = 1 << 20 // 1MiB + + // DefaultQueuePipeSize is the size of the pipe that holds the packet descriptors. + // + // Assuming each packet data is approximately 1280 bytes (IPv6 Minimum MTU) + // then we can hold approximately 1024*1024/1280 ~ 819 packets in the data + // area. Which means the pipe needs to be big enough to hold 819 + // descriptors. + // + // Each descriptor is approximately 8 (slot descriptor in pipe) + + // 16 (packet descriptor) + 12 (for buffer descriptor) assuming each packet is + // stored in exactly 1 buffer descriptor (see queue/tx.go and pipe/tx.go.) + // + // Which means we need approximately 36*819 ~ 29 KiB to store all packet + // descriptors. We could go with a 32 KiB pipe but to give it some slack in + // how the upper layer may make use of the scatter gather buffers we double + // this to hold enough descriptors. + DefaultQueuePipeSize = 64 << 10 // 64KiB + + // DefaultSharedDataSize is the size of the sharedData region used to + // enable/disable notifications. + DefaultSharedDataSize = 4 << 10 // 4KiB + + // DefaultBufferSize is the size of each individual buffer that the data + // region is broken down into to hold packet data. Should be larger than + // 1500 + 14 (Ethernet header) + 10 (VirtIO header) to fit each packet + // in a single buffer. + DefaultBufferSize = 2048 + + // DefaultTmpDir is the path used to create the memory files if a path + // is not provided. + DefaultTmpDir = "/dev/shm" +) + +// A QueuePair represents a pair of TX/RX queues. +type QueuePair struct { + // txCfg is the QueueConfig to be used for transmit queue. + txCfg QueueConfig + + // rxCfg is the QueueConfig to be used for receive queue. + rxCfg QueueConfig +} + +// QueueOptions allows queue specific configuration to be specified when +// creating a QueuePair. +type QueueOptions struct { + // SharedMemPath is the path to use to create the shared memory backing + // files for the queue. + // + // If unspecified it defaults to "/dev/shm". + SharedMemPath string +} + +// NewQueuePair creates a shared memory QueuePair. +func NewQueuePair(opts QueueOptions) (*QueuePair, error) { + txCfg, err := createQueueFDs(opts.SharedMemPath, queueSizes{ + dataSize: DefaultQueueDataSize, + txPipeSize: DefaultQueuePipeSize, + rxPipeSize: DefaultQueuePipeSize, + sharedDataSize: DefaultSharedDataSize, + }) + if err != nil { + return nil, fmt.Errorf("failed to create tx queue: %s", err) + } + + rxCfg, err := createQueueFDs(opts.SharedMemPath, queueSizes{ + dataSize: DefaultQueueDataSize, + txPipeSize: DefaultQueuePipeSize, + rxPipeSize: DefaultQueuePipeSize, + sharedDataSize: DefaultSharedDataSize, + }) + if err != nil { + closeFDs(txCfg) + return nil, fmt.Errorf("failed to create rx queue: %s", err) + } + + return &QueuePair{ + txCfg: txCfg, + rxCfg: rxCfg, + }, nil +} + +// Close closes underlying tx/rx queue fds. +func (q *QueuePair) Close() { + closeFDs(q.txCfg) + closeFDs(q.rxCfg) +} + +// TXQueueConfig returns the QueueConfig for the receive queue. +func (q *QueuePair) TXQueueConfig() QueueConfig { + return q.txCfg +} + +// RXQueueConfig returns the QueueConfig for the transmit queue. +func (q *QueuePair) RXQueueConfig() QueueConfig { + return q.rxCfg +} + +type queueSizes struct { + dataSize int64 + txPipeSize int64 + rxPipeSize int64 + sharedDataSize int64 +} + +func createQueueFDs(sharedMemPath string, s queueSizes) (QueueConfig, error) { + success := false + var eventFD eventfd.Eventfd + var dataFD, txPipeFD, rxPipeFD, sharedDataFD int + defer func() { + if success { + return + } + closeFDs(QueueConfig{ + EventFD: eventFD, + DataFD: dataFD, + TxPipeFD: txPipeFD, + RxPipeFD: rxPipeFD, + SharedDataFD: sharedDataFD, + }) + }() + eventFD, err := eventfd.Create() + if err != nil { + return QueueConfig{}, fmt.Errorf("eventfd failed: %v", err) + } + dataFD, err = createFile(sharedMemPath, s.dataSize, false) + if err != nil { + return QueueConfig{}, fmt.Errorf("failed to create dataFD: %s", err) + } + txPipeFD, err = createFile(sharedMemPath, s.txPipeSize, true) + if err != nil { + return QueueConfig{}, fmt.Errorf("failed to create txPipeFD: %s", err) + } + rxPipeFD, err = createFile(sharedMemPath, s.rxPipeSize, true) + if err != nil { + return QueueConfig{}, fmt.Errorf("failed to create rxPipeFD: %s", err) + } + sharedDataFD, err = createFile(sharedMemPath, s.sharedDataSize, false) + if err != nil { + return QueueConfig{}, fmt.Errorf("failed to create sharedDataFD: %s", err) + } + success = true + return QueueConfig{ + EventFD: eventFD, + DataFD: dataFD, + TxPipeFD: txPipeFD, + RxPipeFD: rxPipeFD, + SharedDataFD: sharedDataFD, + }, nil +} + +func createFile(sharedMemPath string, size int64, initQueue bool) (fd int, err error) { + tmpDir := DefaultTmpDir + if sharedMemPath != "" { + tmpDir = sharedMemPath + } + f, err := os.CreateTemp(tmpDir, "sharedmem_test") + if err != nil { + return -1, fmt.Errorf("TempFile failed: %v", err) + } + defer f.Close() + unix.Unlink(f.Name()) + + if initQueue { + // Write the "slot-free" flag in the initial queue. + if _, err := f.WriteAt([]byte{0, 0, 0, 0, 0, 0, 0, 0x80}, 0); err != nil { + return -1, fmt.Errorf("WriteAt failed: %v", err) + } + } + + fd, err = unix.Dup(int(f.Fd())) + if err != nil { + return -1, fmt.Errorf("unix.Dup(%d) failed: %v", f.Fd(), err) + } + + if err := unix.Ftruncate(fd, size); err != nil { + unix.Close(fd) + return -1, fmt.Errorf("ftruncate(%d, %d) failed: %v", fd, size, err) + } + + return fd, nil +} + +func closeFDs(c QueueConfig) { + unix.Close(c.DataFD) + c.EventFD.Close() + unix.Close(c.TxPipeFD) + unix.Close(c.RxPipeFD) + unix.Close(c.SharedDataFD) +} diff --git a/pkg/tcpip/link/sharedmem/rx.go b/pkg/tcpip/link/sharedmem/rx.go new file mode 100644 index 0000000..4b93baf --- /dev/null +++ b/pkg/tcpip/link/sharedmem/rx.go @@ -0,0 +1,152 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sharedmem + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/eventfd" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue" + "golang.org/x/sys/unix" +) + +// rx holds all state associated with an rx queue. +type rx struct { + data []byte + sharedData []byte + q queue.Rx + eventFD eventfd.Eventfd +} + +// init initializes all state needed by the rx queue based on the information +// provided. +// +// The caller always retains ownership of all file descriptors passed in. The +// queue implementation will duplicate any that it may need in the future. +func (r *rx) init(mtu uint32, c *QueueConfig) error { + // Map in all buffers. + txPipe, err := getBuffer(c.TxPipeFD) + if err != nil { + return err + } + + rxPipe, err := getBuffer(c.RxPipeFD) + if err != nil { + unix.Munmap(txPipe) + return err + } + + data, err := getBuffer(c.DataFD) + if err != nil { + unix.Munmap(txPipe) + unix.Munmap(rxPipe) + return err + } + + sharedData, err := getBuffer(c.SharedDataFD) + if err != nil { + unix.Munmap(txPipe) + unix.Munmap(rxPipe) + unix.Munmap(data) + return err + } + + // Duplicate the eventFD so that caller can close it but we can still + // use it. + efd, err := c.EventFD.Dup() + if err != nil { + unix.Munmap(txPipe) + unix.Munmap(rxPipe) + unix.Munmap(data) + unix.Munmap(sharedData) + return err + } + + // Initialize state based on buffers. + r.q.Init(txPipe, rxPipe, sharedDataPointer(sharedData)) + r.data = data + r.eventFD = efd + r.sharedData = sharedData + + return nil +} + +// cleanup releases all resources allocated during init() except r.eventFD. It +// must only be called if init() has previously succeeded. +func (r *rx) cleanup() { + a, b := r.q.Bytes() + unix.Munmap(a) + unix.Munmap(b) + + unix.Munmap(r.data) + unix.Munmap(r.sharedData) +} + +// notify writes to the tx.eventFD to indicate to the peer that there is data to +// be read. +func (r *rx) notify() { + r.eventFD.Notify() +} + +// postAndReceive posts the provided buffers (if any), and then tries to read +// from the receive queue. +// +// Capacity permitting, it reuses the posted buffer slice to store the buffers +// that were read as well. +// +// This function will block if there aren't any available packets. +func (r *rx) postAndReceive(b []queue.RxBuffer, stopRequested *atomicbitops.Uint32) ([]queue.RxBuffer, uint32) { + // Post the buffers first. If we cannot post, sleep until we can. We + // never post more than will fit concurrently, so it's safe to wait + // until enough room is available. + if len(b) != 0 && !r.q.PostBuffers(b) { + r.q.EnableNotification() + for !r.q.PostBuffers(b) { + r.eventFD.Wait() + if stopRequested.Load() != 0 { + r.q.DisableNotification() + return nil, 0 + } + } + r.q.DisableNotification() + } + + // Read the next set of descriptors. + b, n := r.q.Dequeue(b[:0]) + if len(b) != 0 { + return b, n + } + + // Data isn't immediately available. Enable eventfd notifications. + r.q.EnableNotification() + for { + b, n = r.q.Dequeue(b) + if len(b) != 0 { + break + } + + // Wait for notification. + r.eventFD.Wait() + if stopRequested.Load() != 0 { + r.q.DisableNotification() + return nil, 0 + } + } + r.q.DisableNotification() + + return b, n +} diff --git a/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go b/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go new file mode 100644 index 0000000..e892e5c --- /dev/null +++ b/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go @@ -0,0 +1,96 @@ +package sharedmem + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type serverEndpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var serverEndpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type serverEndpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *serverEndpointRWMutex) Lock() { + locking.AddGLock(serverEndpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *serverEndpointRWMutex) NestedLock(i serverEndpointlockNameIndex) { + locking.AddGLock(serverEndpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *serverEndpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(serverEndpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *serverEndpointRWMutex) NestedUnlock(i serverEndpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(serverEndpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *serverEndpointRWMutex) RLock() { + locking.AddGLock(serverEndpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *serverEndpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(serverEndpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *serverEndpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *serverEndpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *serverEndpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var serverEndpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func serverEndpointinitLockNames() {} + +func init() { + serverEndpointinitLockNames() + serverEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(serverEndpointRWMutex{}), serverEndpointlockNames) +} diff --git a/pkg/tcpip/link/sharedmem/server_rx.go b/pkg/tcpip/link/sharedmem/server_rx.go new file mode 100644 index 0000000..9519155 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/server_rx.go @@ -0,0 +1,162 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sharedmem + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/cleanup" + "github.com/sagernet/gvisor/pkg/eventfd" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue" + "golang.org/x/sys/unix" +) + +// +stateify savable +type serverRx struct { + // packetPipe represents the receive end of the pipe that carries the packet + // descriptors sent by the client. + packetPipe pipe.Rx + + // completionPipe represents the transmit end of the pipe that will carry + // completion notifications from the server to the client. + completionPipe pipe.Tx + + // data represents the buffer area where the packet payload is held. + data []byte + + // eventFD is used to notify the peer when transmission is completed. + eventFD eventfd.Eventfd + + // sharedData the memory region to use to enable/disable notifications. + sharedData []byte + + // sharedEventFDState is the memory region in sharedData used to enable + // disable notifications on eventFD. + sharedEventFDState *atomicbitops.Uint32 +} + +// init initializes all state needed by the serverTx queue based on the +// information provided. +// +// The caller always retains ownership of all file descriptors passed in. The +// queue implementation will duplicate any that it may need in the future. +func (s *serverRx) init(c *QueueConfig) error { + // Map in all buffers. + packetPipeMem, err := getBuffer(c.TxPipeFD) + if err != nil { + return err + } + cu := cleanup.Make(func() { unix.Munmap(packetPipeMem) }) + defer cu.Clean() + + completionPipeMem, err := getBuffer(c.RxPipeFD) + if err != nil { + return err + } + cu.Add(func() { unix.Munmap(completionPipeMem) }) + + data, err := getBuffer(c.DataFD) + if err != nil { + return err + } + cu.Add(func() { unix.Munmap(data) }) + + sharedData, err := getBuffer(c.SharedDataFD) + if err != nil { + return err + } + cu.Add(func() { unix.Munmap(sharedData) }) + + // Duplicate the eventFD so that caller can close it but we can still + // use it. + efd, err := c.EventFD.Dup() + if err != nil { + return err + } + cu.Add(func() { efd.Close() }) + + s.packetPipe.Init(packetPipeMem) + s.completionPipe.Init(completionPipeMem) + s.data = data + s.eventFD = efd + s.sharedData = sharedData + s.sharedEventFDState = sharedDataPointer(sharedData) + + cu.Release() + return nil +} + +func (s *serverRx) cleanup() { + unix.Munmap(s.packetPipe.Bytes()) + unix.Munmap(s.completionPipe.Bytes()) + unix.Munmap(s.data) + unix.Munmap(s.sharedData) + s.eventFD.Close() +} + +// EnableNotification updates the shared state such that the peer will notify +// the eventfd when there are packets to be dequeued. +func (s *serverRx) EnableNotification() { + s.sharedEventFDState.Store(queue.EventFDEnabled) +} + +// DisableNotification updates the shared state such that the peer will not +// notify the eventfd. +func (s *serverRx) DisableNotification() { + s.sharedEventFDState.Store(queue.EventFDDisabled) +} + +// completionNotificationSize is size in bytes of a completion notification sent +// on the completion queue after a transmitted packet has been handled. +const completionNotificationSize = 8 + +// receive receives a single packet from the packetPipe. +func (s *serverRx) receive() *buffer.View { + desc := s.packetPipe.Pull() + if desc == nil { + return nil + } + + pktInfo := queue.DecodeTxPacketHeader(desc) + contents := buffer.NewView(int(pktInfo.Size)) + toCopy := pktInfo.Size + for i := 0; i < pktInfo.BufferCount; i++ { + txBuf := queue.DecodeTxBufferHeader(desc, i) + if txBuf.Size <= toCopy { + contents.Write(s.data[txBuf.Offset:][:txBuf.Size]) + toCopy -= txBuf.Size + continue + } + contents.Write(s.data[txBuf.Offset:][:toCopy]) + break + } + + // Flush to let peer know that slots queued for transmission have been handled + // and its free to reuse the slots. + s.packetPipe.Flush() + // Encode packet completion. + b := s.completionPipe.Push(completionNotificationSize) + queue.EncodeTxCompletion(b, pktInfo.ID) + s.completionPipe.Flush() + return contents +} + +func (s *serverRx) waitForPackets() { + s.eventFD.Wait() +} diff --git a/pkg/tcpip/link/sharedmem/server_tx.go b/pkg/tcpip/link/sharedmem/server_tx.go new file mode 100644 index 0000000..5137239 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/server_tx.go @@ -0,0 +1,194 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sharedmem + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/cleanup" + "github.com/sagernet/gvisor/pkg/eventfd" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "golang.org/x/sys/unix" +) + +// serverTx represents the server end of the sharedmem queue and is used to send +// packets to the peer in the buffers posted by the peer in the fillPipe. +// +// +stateify savable +type serverTx struct { + // fillPipe represents the receive end of the pipe that carries the RxBuffers + // posted by the peer. + fillPipe pipe.Rx + + // completionPipe represents the transmit end of the pipe that carries the + // descriptors for filled RxBuffers. + completionPipe pipe.Tx + + // data represents the buffer area where the packet payload is held. + data []byte + + // eventFD is used to notify the peer when fill requests are fulfilled. + eventFD eventfd.Eventfd + + // sharedData the memory region to use to enable/disable notifications. + sharedData []byte + + // sharedEventFDState is the memory region in sharedData used to enable/disable + // notifications on eventFD. + sharedEventFDState *atomicbitops.Uint32 +} + +// init initializes all tstate needed by the serverTx queue based on the +// information provided. +// +// The caller always retains ownership of all file descriptors passed in. The +// queue implementation will duplicate any that it may need in the future. +func (s *serverTx) init(c *QueueConfig) error { + // Map in all buffers. + fillPipeMem, err := getBuffer(c.TxPipeFD) + if err != nil { + return err + } + cu := cleanup.Make(func() { unix.Munmap(fillPipeMem) }) + defer cu.Clean() + + completionPipeMem, err := getBuffer(c.RxPipeFD) + if err != nil { + return err + } + cu.Add(func() { unix.Munmap(completionPipeMem) }) + + data, err := getBuffer(c.DataFD) + if err != nil { + return err + } + cu.Add(func() { unix.Munmap(data) }) + + sharedData, err := getBuffer(c.SharedDataFD) + if err != nil { + return err + } + cu.Add(func() { unix.Munmap(sharedData) }) + + // Duplicate the eventFD so that caller can close it but we can still + // use it. + efd, err := c.EventFD.Dup() + if err != nil { + return err + } + cu.Add(func() { efd.Close() }) + + cu.Release() + + s.fillPipe.Init(fillPipeMem) + s.completionPipe.Init(completionPipeMem) + s.data = data + s.eventFD = efd + s.sharedData = sharedData + s.sharedEventFDState = sharedDataPointer(sharedData) + + return nil +} + +func (s *serverTx) cleanup() { + unix.Munmap(s.fillPipe.Bytes()) + unix.Munmap(s.completionPipe.Bytes()) + unix.Munmap(s.data) + unix.Munmap(s.sharedData) + s.eventFD.Close() +} + +// acquireBuffers acquires enough buffers to hold all the data in views or +// returns nil if not enough buffers are currently available. +func (s *serverTx) acquireBuffers(pktBuffer buffer.Buffer, buffers []queue.RxBuffer) (acquiredBuffers []queue.RxBuffer) { + acquiredBuffers = buffers[:0] + wantBytes := int(pktBuffer.Size()) + for wantBytes > 0 { + var b []byte + if b = s.fillPipe.Pull(); b == nil { + s.fillPipe.Abort() + return nil + } + rxBuffer := queue.DecodeRxBufferHeader(b) + acquiredBuffers = append(acquiredBuffers, rxBuffer) + wantBytes -= int(rxBuffer.Size) + } + return acquiredBuffers +} + +// fillPacket copies the data in the provided views into buffers pulled from the +// fillPipe and returns a slice of RxBuffers that contain the copied data as +// well as the total number of bytes copied. +// +// To avoid allocations the filledBuffers are appended to the buffers slice +// which will be grown as required. This method takes ownership of pktBuffer. +func (s *serverTx) fillPacket(pktBuffer buffer.Buffer, buffers []queue.RxBuffer) (filledBuffers []queue.RxBuffer, totalCopied uint32) { + bufs := s.acquireBuffers(pktBuffer, buffers) + if bufs == nil { + pktBuffer.Release() + return nil, 0 + } + br := pktBuffer.AsBufferReader() + defer br.Close() + + for i := 0; br.Len() > 0 && i < len(bufs); i++ { + buf := bufs[i] + copied, err := br.Read(s.data[buf.Offset:][:buf.Size]) + buf.Size = uint32(copied) + // Copy the packet into the posted buffer. + totalCopied += bufs[i].Size + if err != nil { + return bufs, totalCopied + } + } + return bufs, totalCopied +} + +func (s *serverTx) transmit(pkt *stack.PacketBuffer) bool { + buffers := make([]queue.RxBuffer, 8) + buffers, totalCopied := s.fillPacket(pkt.ToBuffer(), buffers) + if totalCopied == 0 { + // drop the packet as not enough buffers were probably available + // to send. + return false + } + b := s.completionPipe.Push(queue.RxCompletionSize(len(buffers))) + if b == nil { + return false + } + queue.EncodeRxCompletion(b, totalCopied, 0 /* reserved */) + for i := 0; i < len(buffers); i++ { + queue.EncodeRxCompletionBuffer(b, i, buffers[i]) + } + s.completionPipe.Flush() + s.fillPipe.Flush() + return true +} + +func (s *serverTx) notificationsEnabled() bool { + // notifications are considered to be enabled unless explicitly disabled. + return s.sharedEventFDState.Load() != queue.EventFDDisabled +} + +func (s *serverTx) notify() { + if s.notificationsEnabled() { + s.eventFD.Notify() + } +} diff --git a/pkg/tcpip/link/sharedmem/sharedmem.go b/pkg/tcpip/link/sharedmem/sharedmem.go new file mode 100644 index 0000000..506ac44 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/sharedmem.go @@ -0,0 +1,559 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package sharedmem provides the implementation of data-link layer endpoints +// backed by shared memory. +// +// Shared memory endpoints can be used in the networking stack by calling New() +// to create a new endpoint, and then passing it as an argument to +// Stack.CreateNIC(). +package sharedmem + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/eventfd" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// QueueConfig holds all the file descriptors needed to describe a tx or rx +// queue over shared memory. It is used when creating new shared memory +// endpoints to describe tx and rx queues. +// +// +stateify savable +type QueueConfig struct { + // DataFD is a file descriptor for the file that contains the data to + // be transmitted via this queue. Descriptors contain offsets within + // this file. + DataFD int + + // EventFD is a file descriptor for the event that is signaled when + // data is becomes available in this queue. + EventFD eventfd.Eventfd + + // TxPipeFD is a file descriptor for the tx pipe associated with the + // queue. + TxPipeFD int + + // RxPipeFD is a file descriptor for the rx pipe associated with the + // queue. + RxPipeFD int + + // SharedDataFD is a file descriptor for the file that contains shared + // state between the two ends of the queue. This data specifies, for + // example, whether EventFD signaling is enabled or disabled. + SharedDataFD int +} + +// FDs returns the FD's in the QueueConfig as a slice of ints. This must +// be used in conjunction with QueueConfigFromFDs to ensure the order +// of FDs matches when reconstructing the config when serialized or sent +// as part of control messages. +func (q *QueueConfig) FDs() []int { + return []int{q.DataFD, q.EventFD.FD(), q.TxPipeFD, q.RxPipeFD, q.SharedDataFD} +} + +// QueueConfigFromFDs constructs a QueueConfig out of a slice of ints where each +// entry represents an file descriptor. The order of FDs in the slice must be in +// the order specified below for the config to be valid. QueueConfig.FDs() +// should be used when the config needs to be serialized or sent as part of a +// control message to ensure the correct order. +func QueueConfigFromFDs(fds []int) (QueueConfig, error) { + if len(fds) != 5 { + return QueueConfig{}, fmt.Errorf("insufficient number of fds: len(fds): %d, want: 5", len(fds)) + } + return QueueConfig{ + DataFD: fds[0], + EventFD: eventfd.Wrap(fds[1]), + TxPipeFD: fds[2], + RxPipeFD: fds[3], + SharedDataFD: fds[4], + }, nil +} + +// Options specify the details about the sharedmem endpoint to be created. +// +// +stateify savable +type Options struct { + // MTU is the mtu to use for this endpoint. + MTU uint32 + + // BufferSize is the size of each scatter/gather buffer that will hold packet + // data. + // + // NOTE: This directly determines number of packets that can be held in + // the ring buffer at any time. This does not have to be sized to the MTU as + // the shared memory queue design allows usage of more than one buffer to be + // used to make up a given packet. + BufferSize uint32 + + // LinkAddress is the link address for this endpoint (required). + LinkAddress tcpip.LinkAddress + + // TX is the transmit queue configuration for this shared memory endpoint. + TX QueueConfig + + // RX is the receive queue configuration for this shared memory endpoint. + RX QueueConfig + + // PeerFD is the fd for the connected peer which can be used to detect + // peer disconnects. + PeerFD int + + // OnClosed is a function that is called when the endpoint is being closed + // (probably due to peer going away) + OnClosed func(err tcpip.Error) + + // TXChecksumOffload if true, indicates that this endpoints capability + // set should include CapabilityTXChecksumOffload. + TXChecksumOffload bool + + // RXChecksumOffload if true, indicates that this endpoints capability + // set should include CapabilityRXChecksumOffload. + RXChecksumOffload bool + + // VirtioNetHeaderRequired if true, indicates that all outbound packets should have + // a virtio header and inbound packets should have a virtio header as well. + VirtioNetHeaderRequired bool + + // GSOMaxSize is the maximum GSO packet size. It is zero if GSO is + // disabled. Note that only gVisor GSO is supported, not host GSO. + GSOMaxSize uint32 +} + +var ( + _ stack.LinkEndpoint = (*endpoint)(nil) + _ stack.GSOEndpoint = (*endpoint)(nil) +) + +// +stateify savable +type endpoint struct { + // bufferSize is the size of each individual buffer. + // bufferSize is immutable. + bufferSize uint32 + + // peerFD is an fd to the peer that can be used to detect when the + // peer is gone. + // peerFD is immutable. + peerFD int + + // caps holds the endpoint capabilities. + caps stack.LinkEndpointCapabilities + + // hdrSize is the size of the link layer header if any. + // hdrSize is immutable. + hdrSize uint32 + + // gSOMaxSize is the maximum GSO packet size. It is zero if GSO is + // disabled. Note that only gVisor GSO is supported, not host GSO. + // gsoMaxSize is immutable. + gsoMaxSize uint32 + + // virtioNetHeaderRequired if true indicates that a virtio header is expected + // in all inbound/outbound packets. + virtioNetHeaderRequired bool + + // rx is the receive queue. + rx rx + + // stopRequested determines whether the worker goroutines should stop. + stopRequested atomicbitops.Uint32 + + // Wait group used to indicate that all workers have stopped. + completed sync.WaitGroup + + // onClosed is a function to be called when the FD's peer (if any) closes + // its end of the communication pipe. + // TODO(b/341946753): Restore when netstack is savable. + onClosed func(tcpip.Error) `state:"nosave"` + + // mu protects the following fields. + mu endpointRWMutex `state:"nosave"` + + // tx is the transmit queue. + // +checklocks:mu + tx tx + + // workerStarted specifies whether the worker goroutine was started. + // +checklocks:mu + workerStarted bool + + // addr is the local address of this endpoint. + // + // +checklocks:mu + addr tcpip.LinkAddress + // mtu (maximum transmission unit) is the maximum size of a packet. + // +checklocks:mu + mtu uint32 +} + +// New creates a new shared-memory-based endpoint. Buffers will be broken up +// into buffers of "bufferSize" bytes. +// +// In order to release all resources held by the returned endpoint, Close() +// must be called followed by Wait(). +func New(opts Options) (stack.LinkEndpoint, error) { + e := &endpoint{ + mtu: opts.MTU, + bufferSize: opts.BufferSize, + addr: opts.LinkAddress, + peerFD: opts.PeerFD, + onClosed: opts.OnClosed, + virtioNetHeaderRequired: opts.VirtioNetHeaderRequired, + gsoMaxSize: opts.GSOMaxSize, + } + + if err := e.tx.init(opts.BufferSize, &opts.TX); err != nil { + return nil, err + } + + if err := e.rx.init(opts.BufferSize, &opts.RX); err != nil { + e.tx.cleanup() + return nil, err + } + + e.caps = stack.LinkEndpointCapabilities(0) + if opts.RXChecksumOffload { + e.caps |= stack.CapabilityRXChecksumOffload + } + + if opts.TXChecksumOffload { + e.caps |= stack.CapabilityTXChecksumOffload + } + + if opts.LinkAddress != "" { + e.hdrSize = header.EthernetMinimumSize + e.caps |= stack.CapabilityResolutionRequired + } + + if opts.VirtioNetHeaderRequired { + e.hdrSize += header.VirtioNetHeaderSize + } + + return e, nil +} + +// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction. +func (e *endpoint) SetOnCloseAction(func()) {} + +// Close frees most resources associated with the endpoint. Wait() must be +// called after Close() in order to free the rest. +func (e *endpoint) Close() { + // Tell dispatch goroutine to stop, then write to the eventfd so that + // it wakes up in case it's sleeping. + if e.stopRequested.Swap(1) == 1 { + // It is already closed. + return + } + e.rx.eventFD.Notify() + + // Cleanup the queues inline if the worker hasn't started yet; we also + // know it won't start from now on because stopRequested is set to 1. + e.mu.Lock() + defer e.mu.Unlock() + workerPresent := e.workerStarted + + if !workerPresent { + e.tx.cleanup() + e.rx.cleanup() + } +} + +// Wait implements stack.LinkEndpoint.Wait. It waits until all workers have +// stopped after a Close() call. +func (e *endpoint) Wait() { + e.completed.Wait() + e.rx.eventFD.Close() +} + +// Attach implements stack.LinkEndpoint.Attach. It launches the goroutine that +// reads packets from the rx queue. +func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) { + if dispatcher == nil { + e.Close() + return + } + e.mu.Lock() + if !e.workerStarted && e.stopRequested.Load() == 0 { + e.workerStarted = true + e.completed.Add(1) + + // Spin up a goroutine to monitor for peer shutdown. + if e.peerFD >= 0 { + e.completed.Add(1) + go func() { + defer e.completed.Done() + b := make([]byte, 1) + // When sharedmem endpoint is in use the peerFD is never used for any data + // transfer and this Read should only return if the peer is shutting down. + _, errno := rawfile.BlockingRead(e.peerFD, b) + if e.onClosed != nil { + if errno == 0 { + e.onClosed(nil) + } else { + e.onClosed(tcpip.TranslateErrno(errno)) + } + } + }() + } + + // Link endpoints are not savable. When transportation endpoints + // are saved, they stop sending outgoing packets and all + // incoming packets are rejected. + go e.dispatchLoop(dispatcher) // S/R-SAFE: see above. + } + e.mu.Unlock() +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *endpoint) IsAttached() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.workerStarted +} + +// MTU implements stack.LinkEndpoint.MTU. +func (e *endpoint) MTU() uint32 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mtu +} + +func (e *endpoint) SetMTU(mtu uint32) { + e.mu.Lock() + defer e.mu.Unlock() + e.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (e *endpoint) Capabilities() stack.LinkEndpointCapabilities { + return e.caps +} + +// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It returns the +// ethernet frame header size. +func (e *endpoint) MaxHeaderLength() uint16 { + return uint16(e.hdrSize) +} + +// LinkAddress implements stack.LinkEndpoint.LinkAddress. It returns the local +// link address. +func (e *endpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.addr = addr +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *endpoint) AddHeader(pkt *stack.PacketBuffer) { + e.mu.RLock() + defer e.mu.RUnlock() + // Add ethernet header if needed. + if len(e.addr) == 0 { + return + } + + eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize)) + eth.Encode(&header.EthernetFields{ + SrcAddr: pkt.EgressRoute.LocalLinkAddress, + DstAddr: pkt.EgressRoute.RemoteLinkAddress, + Type: pkt.NetworkProtocolNumber, + }) +} + +func (e *endpoint) parseHeader(pkt *stack.PacketBuffer) bool { + _, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize) + return ok +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (e *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + e.mu.RLock() + defer e.mu.RUnlock() + // Add ethernet header if needed. + if len(e.addr) == 0 { + return true + } + + return e.parseHeader(pkt) +} + +func (e *endpoint) AddVirtioNetHeader(pkt *stack.PacketBuffer) { + virtio := header.VirtioNetHeader(pkt.VirtioNetHeader().Push(header.VirtioNetHeaderSize)) + virtio.Encode(&header.VirtioNetHeaderFields{}) +} + +// +checklocks:e.mu +func (e *endpoint) writePacketLocked(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error { + if e.virtioNetHeaderRequired { + e.AddVirtioNetHeader(pkt) + } + + // Transmit the packet. + b := pkt.ToBuffer() + defer b.Release() + ok := e.tx.transmit(b) + if !ok { + return &tcpip.ErrWouldBlock{} + } + + return nil +} + +// WritePackets implements stack.LinkEndpoint.WritePackets. +func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + n := 0 + var err tcpip.Error + e.mu.Lock() + defer e.mu.Unlock() + for _, pkt := range pkts.AsSlice() { + if err = e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil { + break + } + n++ + } + // WritePackets never returns an error if it successfully transmitted at least + // one packet. + if err != nil && n == 0 { + return 0, err + } + e.tx.notify() + return n, nil +} + +// dispatchLoop reads packets from the rx queue in a loop and dispatches them +// to the network stack. +func (e *endpoint) dispatchLoop(d stack.NetworkDispatcher) { + // Post initial set of buffers. + limit := e.rx.q.PostedBuffersLimit() + if l := uint64(len(e.rx.data)) / uint64(e.bufferSize); limit > l { + limit = l + } + for i := uint64(0); i < limit; i++ { + b := queue.RxBuffer{ + Offset: i * uint64(e.bufferSize), + Size: e.bufferSize, + ID: i, + } + if !e.rx.q.PostBuffers([]queue.RxBuffer{b}) { + log.Warningf("Unable to post %v-th buffer", i) + } + } + + // Read in a loop until a stop is requested. + var rxb []queue.RxBuffer + for e.stopRequested.Load() == 0 { + var n uint32 + rxb, n = e.rx.postAndReceive(rxb, &e.stopRequested) + + // Copy data from the shared area to its own buffer, then + // prepare to repost the buffer. + v := buffer.NewView(int(n)) + v.Grow(int(n)) + offset := uint32(0) + for i := range rxb { + v.WriteAt(e.rx.data[rxb[i].Offset:][:rxb[i].Size], int(offset)) + offset += rxb[i].Size + + rxb[i].Size = e.bufferSize + } + + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithView(v), + }) + + if e.virtioNetHeaderRequired { + _, ok := pkt.VirtioNetHeader().Consume(header.VirtioNetHeaderSize) + if !ok { + pkt.DecRef() + continue + } + } + + var proto tcpip.NetworkProtocolNumber + e.mu.RLock() + addrLen := len(e.addr) + e.mu.RUnlock() + if addrLen != 0 { + if !e.parseHeader(pkt) { + pkt.DecRef() + continue + } + proto = header.Ethernet(pkt.LinkHeader().Slice()).Type() + } else { + // We don't get any indication of what the packet is, so try to guess + // if it's an IPv4 or IPv6 packet. + // IP version information is at the first octet, so pulling up 1 byte. + h, ok := pkt.Data().PullUp(1) + if !ok { + pkt.DecRef() + continue + } + switch header.IPVersion(h) { + case header.IPv4Version: + proto = header.IPv4ProtocolNumber + case header.IPv6Version: + proto = header.IPv6ProtocolNumber + default: + pkt.DecRef() + continue + } + } + // Send packet up the stack. + d.DeliverNetworkPacket(proto, pkt) + pkt.DecRef() + } + + e.mu.Lock() + defer e.mu.Unlock() + + // Clean state. + e.tx.cleanup() + e.rx.cleanup() + + e.completed.Done() +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType +func (*endpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareEther +} + +// GSOMaxSize implements stack.GSOEndpoint. +func (e *endpoint) GSOMaxSize() uint32 { + return e.gsoMaxSize +} + +// SupportsGSO implements stack.GSOEndpoint. +func (e *endpoint) SupportedGSO() stack.SupportedGSO { + return stack.GVisorGSOSupported +} diff --git a/pkg/tcpip/link/sharedmem/sharedmem_server.go b/pkg/tcpip/link/sharedmem/sharedmem_server.go new file mode 100644 index 0000000..e9c981f --- /dev/null +++ b/pkg/tcpip/link/sharedmem/sharedmem_server.go @@ -0,0 +1,399 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sharedmem + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// +stateify savable +type serverEndpoint struct { + // bufferSize is the size of each individual buffer. + // bufferSize is immutable. + bufferSize uint32 + + // rx is the receive queue. + rx serverRx + + // stopRequested determines whether the worker goroutines should stop. + stopRequested atomicbitops.Uint32 + + // Wait group used to indicate that all workers have stopped. + completed sync.WaitGroup `state:"nosave"` + + // peerFD is an fd to the peer that can be used to detect when the peer is + // gone. + // peerFD is immutable. + peerFD int + + // caps holds the endpoint capabilities. + caps stack.LinkEndpointCapabilities + + // hdrSize is the size of the link layer header if any. + // hdrSize is immutable. + hdrSize uint32 + + // virtioNetHeaderRequired if true indicates that a virtio header is expected + // in all inbound/outbound packets. + virtioNetHeaderRequired bool + + // onClosed is a function to be called when the FD's peer (if any) closes its + // end of the communication pipe. + onClosed func(tcpip.Error) `state:"nosave"` + + // mu protects the following fields. + mu serverEndpointRWMutex `state:"nosave"` + + // tx is the transmit queue. + // +checklocks:mu + tx serverTx + + // workerStarted specifies whether the worker goroutine was started. + // +checklocks:mu + workerStarted bool + + // addr is the local address of this endpoint. + // + // +checklocks:mu + addr tcpip.LinkAddress + // mtu (maximum transmission unit) is the maximum size of a packet. + // +checklocks:mu + mtu uint32 +} + +// NewServerEndpoint creates a new shared-memory-based endpoint. Buffers will be +// broken up into buffers of "bufferSize" bytes. +func NewServerEndpoint(opts Options) (stack.LinkEndpoint, error) { + e := &serverEndpoint{ + mtu: opts.MTU, + bufferSize: opts.BufferSize, + addr: opts.LinkAddress, + peerFD: opts.PeerFD, + onClosed: opts.OnClosed, + } + + if err := e.tx.init(&opts.RX); err != nil { + return nil, err + } + + if err := e.rx.init(&opts.TX); err != nil { + e.tx.cleanup() + return nil, err + } + + e.caps = stack.LinkEndpointCapabilities(0) + if opts.RXChecksumOffload { + e.caps |= stack.CapabilityRXChecksumOffload + } + + if opts.TXChecksumOffload { + e.caps |= stack.CapabilityTXChecksumOffload + } + + if opts.LinkAddress != "" { + e.hdrSize = header.EthernetMinimumSize + e.caps |= stack.CapabilityResolutionRequired + } + + return e, nil +} + +// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction. +func (*serverEndpoint) SetOnCloseAction(func()) {} + +// Close frees all resources associated with the endpoint. +func (e *serverEndpoint) Close() { + // Tell dispatch goroutine to stop, then write to the eventfd so that it wakes + // up in case it's sleeping. + e.stopRequested.Store(1) + e.rx.eventFD.Notify() + + // Cleanup the queues inline if the worker hasn't started yet; we also know it + // won't start from now on because stopRequested is set to 1. + e.mu.Lock() + defer e.mu.Unlock() + workerPresent := e.workerStarted + + if !workerPresent { + e.tx.cleanup() + e.rx.cleanup() + } +} + +// Wait implements stack.LinkEndpoint.Wait. It waits until all workers have +// stopped after a Close() call. +func (e *serverEndpoint) Wait() { + e.completed.Wait() +} + +// Attach implements stack.LinkEndpoint.Attach. It launches the goroutine that +// reads packets from the rx queue. +func (e *serverEndpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + if !e.workerStarted && e.stopRequested.Load() == 0 { + e.workerStarted = true + e.completed.Add(1) + if e.peerFD >= 0 { + e.completed.Add(1) + // Spin up a goroutine to monitor for peer shutdown. + go func() { + b := make([]byte, 1) + // When sharedmem endpoint is in use the peerFD is never used for any + // data transfer and this Read should only return if the peer is + // shutting down. + _, errno := rawfile.BlockingRead(e.peerFD, b) + if e.onClosed != nil { + if errno == 0 { + e.onClosed(nil) + } else { + e.onClosed(tcpip.TranslateErrno(errno)) + } + } + e.completed.Done() + }() + } + // Link endpoints are not savable. When transportation endpoints are saved, + // they stop sending outgoing packets and all incoming packets are rejected. + go e.dispatchLoop(dispatcher) // S/R-SAFE: see above. + } + e.mu.Unlock() +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *serverEndpoint) IsAttached() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.workerStarted +} + +// MTU implements stack.LinkEndpoint.MTU. +func (e *serverEndpoint) MTU() uint32 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mtu +} + +func (e *serverEndpoint) SetMTU(mtu uint32) { + e.mu.Lock() + defer e.mu.Unlock() + e.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (e *serverEndpoint) Capabilities() stack.LinkEndpointCapabilities { + return e.caps +} + +// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It returns the +// ethernet frame header size. +func (e *serverEndpoint) MaxHeaderLength() uint16 { + return uint16(e.hdrSize) +} + +// LinkAddress implements stack.LinkEndpoint.LinkAddress. It returns the local +// link address. +func (e *serverEndpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *serverEndpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.addr = addr +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *serverEndpoint) AddHeader(pkt *stack.PacketBuffer) { + e.mu.RLock() + defer e.mu.RUnlock() + // Add ethernet header if needed. + if len(e.addr) == 0 { + return + } + + eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize)) + eth.Encode(&header.EthernetFields{ + SrcAddr: pkt.EgressRoute.LocalLinkAddress, + DstAddr: pkt.EgressRoute.RemoteLinkAddress, + Type: pkt.NetworkProtocolNumber, + }) +} + +func (e *serverEndpoint) parseHeader(pkt *stack.PacketBuffer) bool { + _, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize) + return ok +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (e *serverEndpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + e.mu.RLock() + defer e.mu.RUnlock() + // Add ethernet header if needed. + if len(e.addr) == 0 { + return true + } + + return e.parseHeader(pkt) +} + +func (e *serverEndpoint) AddVirtioNetHeader(pkt *stack.PacketBuffer) { + virtio := header.VirtioNetHeader(pkt.VirtioNetHeader().Push(header.VirtioNetHeaderSize)) + virtio.Encode(&header.VirtioNetHeaderFields{}) +} + +// +checklocks:e.mu +func (e *serverEndpoint) writePacketLocked(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error { + if e.virtioNetHeaderRequired { + e.AddVirtioNetHeader(pkt) + } + + ok := e.tx.transmit(pkt) + if !ok { + return &tcpip.ErrWouldBlock{} + } + + return nil +} + +// WritePacket writes outbound packets to the file descriptor. If it is not +// currently writable, the packet is dropped. +// WritePacket implements stack.LinkEndpoint.WritePacket. +func (e *serverEndpoint) WritePacket(_ stack.RouteInfo, _ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error { + // Transmit the packet. + e.mu.Lock() + defer e.mu.Unlock() + if err := e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil { + return err + } + e.tx.notify() + return nil +} + +// WritePackets implements stack.LinkEndpoint.WritePackets. +func (e *serverEndpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + n := 0 + var err tcpip.Error + e.mu.Lock() + defer e.mu.Unlock() + for _, pkt := range pkts.AsSlice() { + if err = e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil { + break + } + n++ + } + // WritePackets never returns an error if it successfully transmitted at least + // one packet. + if err != nil && n == 0 { + return 0, err + } + e.tx.notify() + return n, nil +} + +// dispatchLoop reads packets from the rx queue in a loop and dispatches them +// to the network stack. +func (e *serverEndpoint) dispatchLoop(d stack.NetworkDispatcher) { + for e.stopRequested.Load() == 0 { + b := e.rx.receive() + if b == nil { + e.rx.EnableNotification() + // Now pull again to make sure we didn't receive any packets + // while notifications were not enabled. + for { + b = e.rx.receive() + if b != nil { + // Disable notifications as we only need to be notified when we are going + // to block on eventFD. This should prevent the peer from needlessly + // writing to eventFD when this end is already awake and processing + // packets. + e.rx.DisableNotification() + break + } + e.rx.waitForPackets() + } + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithView(b), + }) + if e.virtioNetHeaderRequired { + _, ok := pkt.VirtioNetHeader().Consume(header.VirtioNetHeaderSize) + if !ok { + pkt.DecRef() + continue + } + } + var proto tcpip.NetworkProtocolNumber + e.mu.RLock() + addrLen := len(e.addr) + e.mu.RUnlock() + if addrLen != 0 { + if !e.parseHeader(pkt) { + pkt.DecRef() + continue + } + proto = header.Ethernet(pkt.LinkHeader().Slice()).Type() + } else { + // We don't get any indication of what the packet is, so try to guess + // if it's an IPv4 or IPv6 packet. + // IP version information is at the first octet, so pulling up 1 byte. + h, ok := pkt.Data().PullUp(1) + if !ok { + pkt.DecRef() + continue + } + switch header.IPVersion(h) { + case header.IPv4Version: + proto = header.IPv4ProtocolNumber + case header.IPv6Version: + proto = header.IPv6ProtocolNumber + default: + pkt.DecRef() + continue + } + } + // Send packet up the stack. + d.DeliverNetworkPacket(proto, pkt) + pkt.DecRef() + } + + e.mu.Lock() + defer e.mu.Unlock() + + // Clean state. + e.tx.cleanup() + e.rx.cleanup() + + e.completed.Done() +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType +func (e *serverEndpoint) ARPHardwareType() header.ARPHardwareType { + if e.hdrSize > 0 { + return header.ARPHardwareEther + } + return header.ARPHardwareNone +} diff --git a/pkg/tcpip/link/sharedmem/sharedmem_state_autogen.go b/pkg/tcpip/link/sharedmem/sharedmem_state_autogen.go new file mode 100644 index 0000000..cb8ac66 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/sharedmem_state_autogen.go @@ -0,0 +1,309 @@ +// automatically generated by stateify. + +//go:build linux && linux && linux && linux && linux && linux +// +build linux,linux,linux,linux,linux,linux + +package sharedmem + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (s *serverRx) StateTypeName() string { + return "pkg/tcpip/link/sharedmem.serverRx" +} + +func (s *serverRx) StateFields() []string { + return []string{ + "packetPipe", + "completionPipe", + "data", + "eventFD", + "sharedData", + "sharedEventFDState", + } +} + +func (s *serverRx) beforeSave() {} + +// +checklocksignore +func (s *serverRx) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.packetPipe) + stateSinkObject.Save(1, &s.completionPipe) + stateSinkObject.Save(2, &s.data) + stateSinkObject.Save(3, &s.eventFD) + stateSinkObject.Save(4, &s.sharedData) + stateSinkObject.Save(5, &s.sharedEventFDState) +} + +func (s *serverRx) afterLoad(context.Context) {} + +// +checklocksignore +func (s *serverRx) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.packetPipe) + stateSourceObject.Load(1, &s.completionPipe) + stateSourceObject.Load(2, &s.data) + stateSourceObject.Load(3, &s.eventFD) + stateSourceObject.Load(4, &s.sharedData) + stateSourceObject.Load(5, &s.sharedEventFDState) +} + +func (s *serverTx) StateTypeName() string { + return "pkg/tcpip/link/sharedmem.serverTx" +} + +func (s *serverTx) StateFields() []string { + return []string{ + "fillPipe", + "completionPipe", + "data", + "eventFD", + "sharedData", + "sharedEventFDState", + } +} + +func (s *serverTx) beforeSave() {} + +// +checklocksignore +func (s *serverTx) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.fillPipe) + stateSinkObject.Save(1, &s.completionPipe) + stateSinkObject.Save(2, &s.data) + stateSinkObject.Save(3, &s.eventFD) + stateSinkObject.Save(4, &s.sharedData) + stateSinkObject.Save(5, &s.sharedEventFDState) +} + +func (s *serverTx) afterLoad(context.Context) {} + +// +checklocksignore +func (s *serverTx) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.fillPipe) + stateSourceObject.Load(1, &s.completionPipe) + stateSourceObject.Load(2, &s.data) + stateSourceObject.Load(3, &s.eventFD) + stateSourceObject.Load(4, &s.sharedData) + stateSourceObject.Load(5, &s.sharedEventFDState) +} + +func (q *QueueConfig) StateTypeName() string { + return "pkg/tcpip/link/sharedmem.QueueConfig" +} + +func (q *QueueConfig) StateFields() []string { + return []string{ + "DataFD", + "EventFD", + "TxPipeFD", + "RxPipeFD", + "SharedDataFD", + } +} + +func (q *QueueConfig) beforeSave() {} + +// +checklocksignore +func (q *QueueConfig) StateSave(stateSinkObject state.Sink) { + q.beforeSave() + stateSinkObject.Save(0, &q.DataFD) + stateSinkObject.Save(1, &q.EventFD) + stateSinkObject.Save(2, &q.TxPipeFD) + stateSinkObject.Save(3, &q.RxPipeFD) + stateSinkObject.Save(4, &q.SharedDataFD) +} + +func (q *QueueConfig) afterLoad(context.Context) {} + +// +checklocksignore +func (q *QueueConfig) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &q.DataFD) + stateSourceObject.Load(1, &q.EventFD) + stateSourceObject.Load(2, &q.TxPipeFD) + stateSourceObject.Load(3, &q.RxPipeFD) + stateSourceObject.Load(4, &q.SharedDataFD) +} + +func (o *Options) StateTypeName() string { + return "pkg/tcpip/link/sharedmem.Options" +} + +func (o *Options) StateFields() []string { + return []string{ + "MTU", + "BufferSize", + "LinkAddress", + "TX", + "RX", + "PeerFD", + "OnClosed", + "TXChecksumOffload", + "RXChecksumOffload", + "VirtioNetHeaderRequired", + "GSOMaxSize", + } +} + +func (o *Options) beforeSave() {} + +// +checklocksignore +func (o *Options) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.MTU) + stateSinkObject.Save(1, &o.BufferSize) + stateSinkObject.Save(2, &o.LinkAddress) + stateSinkObject.Save(3, &o.TX) + stateSinkObject.Save(4, &o.RX) + stateSinkObject.Save(5, &o.PeerFD) + stateSinkObject.Save(6, &o.OnClosed) + stateSinkObject.Save(7, &o.TXChecksumOffload) + stateSinkObject.Save(8, &o.RXChecksumOffload) + stateSinkObject.Save(9, &o.VirtioNetHeaderRequired) + stateSinkObject.Save(10, &o.GSOMaxSize) +} + +func (o *Options) afterLoad(context.Context) {} + +// +checklocksignore +func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.MTU) + stateSourceObject.Load(1, &o.BufferSize) + stateSourceObject.Load(2, &o.LinkAddress) + stateSourceObject.Load(3, &o.TX) + stateSourceObject.Load(4, &o.RX) + stateSourceObject.Load(5, &o.PeerFD) + stateSourceObject.Load(6, &o.OnClosed) + stateSourceObject.Load(7, &o.TXChecksumOffload) + stateSourceObject.Load(8, &o.RXChecksumOffload) + stateSourceObject.Load(9, &o.VirtioNetHeaderRequired) + stateSourceObject.Load(10, &o.GSOMaxSize) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/link/sharedmem.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "bufferSize", + "peerFD", + "caps", + "hdrSize", + "gsoMaxSize", + "virtioNetHeaderRequired", + "rx", + "stopRequested", + "completed", + "tx", + "workerStarted", + "addr", + "mtu", + } +} + +func (e *endpoint) beforeSave() {} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.bufferSize) + stateSinkObject.Save(1, &e.peerFD) + stateSinkObject.Save(2, &e.caps) + stateSinkObject.Save(3, &e.hdrSize) + stateSinkObject.Save(4, &e.gsoMaxSize) + stateSinkObject.Save(5, &e.virtioNetHeaderRequired) + stateSinkObject.Save(6, &e.rx) + stateSinkObject.Save(7, &e.stopRequested) + stateSinkObject.Save(8, &e.completed) + stateSinkObject.Save(9, &e.tx) + stateSinkObject.Save(10, &e.workerStarted) + stateSinkObject.Save(11, &e.addr) + stateSinkObject.Save(12, &e.mtu) +} + +func (e *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.bufferSize) + stateSourceObject.Load(1, &e.peerFD) + stateSourceObject.Load(2, &e.caps) + stateSourceObject.Load(3, &e.hdrSize) + stateSourceObject.Load(4, &e.gsoMaxSize) + stateSourceObject.Load(5, &e.virtioNetHeaderRequired) + stateSourceObject.Load(6, &e.rx) + stateSourceObject.Load(7, &e.stopRequested) + stateSourceObject.Load(8, &e.completed) + stateSourceObject.Load(9, &e.tx) + stateSourceObject.Load(10, &e.workerStarted) + stateSourceObject.Load(11, &e.addr) + stateSourceObject.Load(12, &e.mtu) +} + +func (e *serverEndpoint) StateTypeName() string { + return "pkg/tcpip/link/sharedmem.serverEndpoint" +} + +func (e *serverEndpoint) StateFields() []string { + return []string{ + "bufferSize", + "rx", + "stopRequested", + "peerFD", + "caps", + "hdrSize", + "virtioNetHeaderRequired", + "tx", + "workerStarted", + "addr", + "mtu", + } +} + +func (e *serverEndpoint) beforeSave() {} + +// +checklocksignore +func (e *serverEndpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.bufferSize) + stateSinkObject.Save(1, &e.rx) + stateSinkObject.Save(2, &e.stopRequested) + stateSinkObject.Save(3, &e.peerFD) + stateSinkObject.Save(4, &e.caps) + stateSinkObject.Save(5, &e.hdrSize) + stateSinkObject.Save(6, &e.virtioNetHeaderRequired) + stateSinkObject.Save(7, &e.tx) + stateSinkObject.Save(8, &e.workerStarted) + stateSinkObject.Save(9, &e.addr) + stateSinkObject.Save(10, &e.mtu) +} + +func (e *serverEndpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *serverEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.bufferSize) + stateSourceObject.Load(1, &e.rx) + stateSourceObject.Load(2, &e.stopRequested) + stateSourceObject.Load(3, &e.peerFD) + stateSourceObject.Load(4, &e.caps) + stateSourceObject.Load(5, &e.hdrSize) + stateSourceObject.Load(6, &e.virtioNetHeaderRequired) + stateSourceObject.Load(7, &e.tx) + stateSourceObject.Load(8, &e.workerStarted) + stateSourceObject.Load(9, &e.addr) + stateSourceObject.Load(10, &e.mtu) +} + +func init() { + state.Register((*serverRx)(nil)) + state.Register((*serverTx)(nil)) + state.Register((*QueueConfig)(nil)) + state.Register((*Options)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*serverEndpoint)(nil)) +} diff --git a/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go b/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go new file mode 100644 index 0000000..3abfdae --- /dev/null +++ b/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go @@ -0,0 +1,59 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sharedmem + +import ( + "fmt" + "reflect" + "unsafe" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/memutil" + "golang.org/x/sys/unix" +) + +// sharedDataPointer converts the shared data slice into a pointer so that it +// can be used in atomic operations. +func sharedDataPointer(sharedData []byte) *atomicbitops.Uint32 { + return (*atomicbitops.Uint32)(unsafe.Pointer(&sharedData[0:4][0])) +} + +// getBuffer returns a memory region mapped to the full contents of the given +// file descriptor. +func getBuffer(fd int) ([]byte, error) { + var s unix.Stat_t + if err := unix.Fstat(fd, &s); err != nil { + return nil, err + } + + // Check that size doesn't overflow an int. + if s.Size > int64(^uint(0)>>1) { + return nil, unix.EDOM + } + + addr, err := memutil.MapFile(0 /* addr */, uintptr(s.Size), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_FILE, uintptr(fd), 0 /*offset*/) + if err != nil { + return nil, fmt.Errorf("failed to map memory for buffer fd: %d, error: %s", fd, err) + } + + // Use unsafe to convert addr into a []byte. + var b []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + hdr.Data = addr + hdr.Len = int(s.Size) + hdr.Cap = int(s.Size) + + return b, nil +} diff --git a/pkg/tcpip/link/sharedmem/sharedmem_unsafe_state_autogen.go b/pkg/tcpip/link/sharedmem/sharedmem_unsafe_state_autogen.go new file mode 100644 index 0000000..ac3a665 --- /dev/null +++ b/pkg/tcpip/link/sharedmem/sharedmem_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package sharedmem diff --git a/pkg/tcpip/link/sharedmem/tx.go b/pkg/tcpip/link/sharedmem/tx.go new file mode 100644 index 0000000..7c568bf --- /dev/null +++ b/pkg/tcpip/link/sharedmem/tx.go @@ -0,0 +1,279 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sharedmem + +import ( + "math" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/eventfd" + "github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue" + "golang.org/x/sys/unix" +) + +const ( + nilID = math.MaxUint64 +) + +// tx holds all state associated with a tx queue. +type tx struct { + data []byte + q queue.Tx + ids idManager + bufs bufferManager + eventFD eventfd.Eventfd + sharedData []byte + sharedDataFD int +} + +// init initializes all state needed by the tx queue based on the information +// provided. +// +// The caller always retains ownership of all file descriptors passed in. The +// queue implementation will duplicate any that it may need in the future. +func (t *tx) init(bufferSize uint32, c *QueueConfig) error { + // Map in all buffers. + txPipe, err := getBuffer(c.TxPipeFD) + if err != nil { + return err + } + + rxPipe, err := getBuffer(c.RxPipeFD) + if err != nil { + unix.Munmap(txPipe) + return err + } + + data, err := getBuffer(c.DataFD) + if err != nil { + unix.Munmap(txPipe) + unix.Munmap(rxPipe) + return err + } + + sharedData, err := getBuffer(c.SharedDataFD) + if err != nil { + unix.Munmap(txPipe) + unix.Munmap(rxPipe) + unix.Munmap(data) + } + + // Initialize state based on buffers. + t.q.Init(txPipe, rxPipe, sharedDataPointer(sharedData)) + t.ids.init() + t.bufs.init(0, len(data), int(bufferSize)) + t.data = data + t.eventFD = c.EventFD + t.sharedDataFD = c.SharedDataFD + t.sharedData = sharedData + + return nil +} + +// cleanup releases all resources allocated during init(). It must only be +// called if init() has previously succeeded. +func (t *tx) cleanup() { + a, b := t.q.Bytes() + unix.Munmap(a) + unix.Munmap(b) + unix.Munmap(t.data) +} + +// transmit sends a packet made of bufs. Returns a boolean that specifies +// whether the packet was successfully transmitted. +func (t *tx) transmit(transmitBuf buffer.Buffer) bool { + // Pull completions from the tx queue and add their buffers back to the + // pool so that we can reuse them. + for { + id, ok := t.q.CompletedPacket() + if !ok { + break + } + + if buf := t.ids.remove(id); buf != nil { + t.bufs.free(buf) + } + } + + bSize := t.bufs.entrySize + total := uint32(transmitBuf.Size()) + bufCount := (total + bSize - 1) / bSize + + // Allocate enough buffers to hold all the data. + var buf *queue.TxBuffer + for i := bufCount; i != 0; i-- { + b := t.bufs.alloc() + if b == nil { + // Failed to get all buffers. Return to the pool + // whatever we had managed to get. + if buf != nil { + t.bufs.free(buf) + } + return false + } + b.Next = buf + buf = b + } + + // Copy data into allocated buffers. + nBuf := buf + var dBuf []byte + transmitBuf.Apply(func(v *buffer.View) { + for v.Size() > 0 { + if len(dBuf) == 0 { + dBuf = t.data[nBuf.Offset:][:nBuf.Size] + nBuf = nBuf.Next + } + n := copy(dBuf, v.AsSlice()) + v.TrimFront(n) + dBuf = dBuf[n:] + } + }) + + // Get an id for this packet and send it out. + id := t.ids.add(buf) + if !t.q.Enqueue(id, total, bufCount, buf) { + t.ids.remove(id) + t.bufs.free(buf) + return false + } + + return true +} + +// notify writes to the tx.eventFD to indicate to the peer that there is data to +// be read. +func (t *tx) notify() { + if t.q.NotificationsEnabled() { + t.eventFD.Notify() + } +} + +// idDescriptor is used by idManager to either point to a tx buffer (in case +// the ID is assigned) or to the next free element (if the id is not assigned). +type idDescriptor struct { + buf *queue.TxBuffer + nextFree uint64 +} + +// idManager is a manager of tx buffer identifiers. It assigns unique IDs to +// tx buffers that are added to it; the IDs can only be reused after they have +// been removed. +// +// The ID assignments are stored so that the tx buffers can be retrieved from +// the IDs previously assigned to them. +type idManager struct { + // ids is a slice containing all tx buffers. The ID is the index into + // this slice. + ids []idDescriptor + + // freeList a list of free IDs. + freeList uint64 +} + +// init initializes the id manager. +func (m *idManager) init() { + m.freeList = nilID +} + +// add assigns an ID to the given tx buffer. +func (m *idManager) add(b *queue.TxBuffer) uint64 { + if i := m.freeList; i != nilID { + // There is an id available in the free list, just use it. + m.ids[i].buf = b + m.freeList = m.ids[i].nextFree + return i + } + + // We need to expand the id descriptor. + m.ids = append(m.ids, idDescriptor{buf: b}) + return uint64(len(m.ids) - 1) +} + +// remove retrieves the tx buffer associated with the given ID, and removes the +// ID from the assigned table so that it can be reused in the future. +func (m *idManager) remove(i uint64) *queue.TxBuffer { + if i >= uint64(len(m.ids)) { + return nil + } + + desc := &m.ids[i] + b := desc.buf + if b == nil { + // The provided id is not currently assigned. + return nil + } + + desc.buf = nil + desc.nextFree = m.freeList + m.freeList = i + + return b +} + +// bufferManager manages a buffer region broken up into smaller, equally sized +// buffers. Smaller buffers can be allocated and freed. +type bufferManager struct { + freeList *queue.TxBuffer + curOffset uint64 + limit uint64 + entrySize uint32 +} + +// init initializes the buffer manager. +func (b *bufferManager) init(initialOffset, size, entrySize int) { + b.freeList = nil + b.curOffset = uint64(initialOffset) + b.limit = uint64(initialOffset + size/entrySize*entrySize) + b.entrySize = uint32(entrySize) +} + +// alloc allocates a buffer from the manager, if one is available. +func (b *bufferManager) alloc() *queue.TxBuffer { + if b.freeList != nil { + // There is a descriptor ready for reuse in the free list. + d := b.freeList + b.freeList = d.Next + d.Next = nil + return d + } + + if b.curOffset < b.limit { + // There is room available in the never-used range, so create + // a new descriptor for it. + d := &queue.TxBuffer{ + Offset: b.curOffset, + Size: b.entrySize, + } + b.curOffset += uint64(b.entrySize) + return d + } + + return nil +} + +// free returns all buffers in the list to the buffer manager so that they can +// be reused. +func (b *bufferManager) free(d *queue.TxBuffer) { + // Find the last buffer in the list. + last := d + for last.Next != nil { + last = last.Next + } + + // Push list onto free list. + last.Next = b.freeList + b.freeList = d +} diff --git a/pkg/tcpip/link/sniffer/pcap.go b/pkg/tcpip/link/sniffer/pcap.go new file mode 100644 index 0000000..fc037e5 --- /dev/null +++ b/pkg/tcpip/link/sniffer/pcap.go @@ -0,0 +1,85 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sniffer + +import ( + "encoding" + "encoding/binary" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +type pcapHeader struct { + // MagicNumber is the file magic number. + MagicNumber uint32 + + // VersionMajor is the major version number. + VersionMajor uint16 + + // VersionMinor is the minor version number. + VersionMinor uint16 + + // Thiszone is the GMT to local correction. + Thiszone int32 + + // Sigfigs is the accuracy of timestamps. + Sigfigs uint32 + + // Snaplen is the max length of captured packets, in octets. + Snaplen uint32 + + // Network is the data link type. + Network uint32 +} + +var _ encoding.BinaryMarshaler = (*pcapPacket)(nil) + +type pcapPacket struct { + timestamp time.Time + packet *stack.PacketBuffer + maxCaptureLen int +} + +func (p *pcapPacket) MarshalBinary() ([]byte, error) { + pkt := trimmedClone(p.packet) + defer pkt.DecRef() + packetSize := pkt.Size() + captureLen := p.maxCaptureLen + if packetSize < captureLen { + captureLen = packetSize + } + b := make([]byte, 16+captureLen) + binary.LittleEndian.PutUint32(b[0:4], uint32(p.timestamp.Unix())) + binary.LittleEndian.PutUint32(b[4:8], uint32(p.timestamp.Nanosecond()/1000)) + binary.LittleEndian.PutUint32(b[8:12], uint32(captureLen)) + binary.LittleEndian.PutUint32(b[12:16], uint32(packetSize)) + w := tcpip.SliceWriter(b[16:]) + for _, v := range pkt.AsSlices() { + if captureLen == 0 { + break + } + if len(v) > captureLen { + v = v[:captureLen] + } + n, err := w.Write(v) + if err != nil { + panic(err) + } + captureLen -= n + } + return b, nil +} diff --git a/pkg/tcpip/link/sniffer/sniffer.go b/pkg/tcpip/link/sniffer/sniffer.go new file mode 100644 index 0000000..3ad3c9c --- /dev/null +++ b/pkg/tcpip/link/sniffer/sniffer.go @@ -0,0 +1,399 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sniffer provides the implementation of data-link layer endpoints that +// wrap another endpoint and logs inbound and outbound packets. +// +// Sniffer endpoints can be used in the networking stack by calling New(eID) to +// create a new endpoint, where eID is the ID of the endpoint being wrapped, +// and then passing it as an argument to Stack.CreateNIC(). +package sniffer + +import ( + "encoding/binary" + "fmt" + "io" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" + "github.com/sagernet/gvisor/pkg/tcpip/link/nested" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// LogPackets is a flag used to enable or disable packet logging via the log +// package. Valid values are 0 or 1. +var LogPackets atomicbitops.Uint32 = atomicbitops.FromUint32(1) + +// Endpoint is used to sniff and log network traffic. +// +// +stateify savable +type Endpoint struct { + nested.Endpoint + writer io.Writer + maxPCAPLen uint32 + logPrefix string +} + +var ( + _ stack.GSOEndpoint = (*Endpoint)(nil) + _ stack.LinkEndpoint = (*Endpoint)(nil) + _ stack.NetworkDispatcher = (*Endpoint)(nil) +) + +// A Direction indicates whether the packing is being sent or received. +type Direction int + +const ( + // DirectionSend indicates a sent packet. + DirectionSend = iota + // DirectionRecv indicates a received packet. + DirectionRecv +) + +func (dr Direction) String() string { + switch dr { + case DirectionSend: + return "send" + case DirectionRecv: + return "recv" + default: + panic(fmt.Sprintf("invalid Direction %d", dr)) + } +} + +// New creates a new sniffer link-layer endpoint. It wraps around another +// endpoint and logs packets and they traverse the endpoint. +func New(lower stack.LinkEndpoint) *Endpoint { + return NewWithPrefix(lower, "") +} + +// NewWithPrefix creates a new sniffer link-layer endpoint. It wraps around +// another endpoint and logs packets prefixed with logPrefix as they traverse +// the endpoint. +// +// logPrefix is prepended to the log line without any separators. +// E.g. logPrefix = "NIC:en0/" will produce log lines like +// "NIC:en0/send udp [...]". +func NewWithPrefix(lower stack.LinkEndpoint, logPrefix string) *Endpoint { + sniffer := &Endpoint{logPrefix: logPrefix} + sniffer.Endpoint.Init(lower, sniffer) + return sniffer +} + +func zoneOffset() (int32, error) { + date := time.Date(0, 0, 0, 0, 0, 0, 0, time.Local) + _, offset := date.Zone() + return int32(offset), nil +} + +func writePCAPHeader(w io.Writer, maxLen uint32) error { + offset, err := zoneOffset() + if err != nil { + return err + } + return binary.Write(w, binary.LittleEndian, pcapHeader{ + // From https://wiki.wireshark.org/Development/LibpcapFileFormat + MagicNumber: 0xa1b2c3d4, + + VersionMajor: 2, + VersionMinor: 4, + Thiszone: offset, + Sigfigs: 0, + Snaplen: maxLen, + Network: 101, // LINKTYPE_RAW + }) +} + +// NewWithWriter creates a new sniffer link-layer endpoint. It wraps around +// another endpoint and logs packets as they traverse the endpoint. +// +// Each packet is written to writer in the pcap format in a single Write call +// without synchronization. A sniffer created with this function will not emit +// packets using the standard log package. +// +// snapLen is the maximum amount of a packet to be saved. Packets with a length +// less than or equal to snapLen will be saved in their entirety. Longer +// packets will be truncated to snapLen. +func NewWithWriter(lower stack.LinkEndpoint, writer io.Writer, snapLen uint32) (*Endpoint, error) { + if err := writePCAPHeader(writer, snapLen); err != nil { + return nil, err + } + sniffer := &Endpoint{ + writer: writer, + maxPCAPLen: snapLen, + } + sniffer.Endpoint.Init(lower, sniffer) + return sniffer, nil +} + +// DeliverNetworkPacket implements the stack.NetworkDispatcher interface. It is +// called by the link-layer endpoint being wrapped when a packet arrives, and +// logs the packet before forwarding to the actual dispatcher. +func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.DumpPacket(DirectionRecv, protocol, pkt, nil) + e.Endpoint.DeliverNetworkPacket(protocol, pkt) +} + +// DumpPacket logs a packet, depending on configuration, to stderr and/or a +// pcap file. ts is an optional timestamp for the packet. +func (e *Endpoint) DumpPacket(dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer, ts *time.Time) { + if LogPackets.Load() == 1 { + LogPacket(e.logPrefix, dir, protocol, pkt) + } + if e.writer != nil { + packet := pcapPacket{ + packet: pkt, + maxCaptureLen: int(e.maxPCAPLen), + } + if ts == nil { + packet.timestamp = time.Now() + } else { + packet.timestamp = *ts + } + b, err := packet.MarshalBinary() + if err != nil { + panic(err) + } + if _, err := e.writer.Write(b); err != nil { + panic(err) + } + } +} + +// WritePackets implements the stack.LinkEndpoint interface. It is called by +// higher-level protocols to write packets; it just logs the packet and +// forwards the request to the lower endpoint. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + for _, pkt := range pkts.AsSlice() { + e.DumpPacket(DirectionSend, pkt.NetworkProtocolNumber, pkt, nil) + } + return e.Endpoint.WritePackets(pkts) +} + +// LogPacket logs a packet to stdout. +func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + // Figure out the network layer info. + var transProto uint8 + var src tcpip.Address + var dst tcpip.Address + var size uint16 + var id uint32 + var fragmentOffset uint16 + var moreFragments bool + + clone := trimmedClone(pkt) + defer clone.DecRef() + switch protocol { + case header.IPv4ProtocolNumber: + if ok := parse.IPv4(clone); !ok { + return + } + + ipv4 := header.IPv4(clone.NetworkHeader().Slice()) + fragmentOffset = ipv4.FragmentOffset() + moreFragments = ipv4.Flags()&header.IPv4FlagMoreFragments == header.IPv4FlagMoreFragments + src = ipv4.SourceAddress() + dst = ipv4.DestinationAddress() + transProto = ipv4.Protocol() + size = ipv4.TotalLength() - uint16(ipv4.HeaderLength()) + id = uint32(ipv4.ID()) + + case header.IPv6ProtocolNumber: + proto, fragID, fragOffset, fragMore, ok := parse.IPv6(clone) + if !ok { + return + } + + ipv6 := header.IPv6(clone.NetworkHeader().Slice()) + src = ipv6.SourceAddress() + dst = ipv6.DestinationAddress() + transProto = uint8(proto) + size = ipv6.PayloadLength() + id = fragID + moreFragments = fragMore + fragmentOffset = fragOffset + + case header.ARPProtocolNumber: + if !parse.ARP(clone) { + return + } + + arp := header.ARP(clone.NetworkHeader().Slice()) + log.Infof( + "%s%s arp %s (%s) -> %s (%s) valid:%t", + prefix, + dir, + tcpip.AddrFromSlice(arp.ProtocolAddressSender()), tcpip.LinkAddress(arp.HardwareAddressSender()), + tcpip.AddrFromSlice(arp.ProtocolAddressTarget()), tcpip.LinkAddress(arp.HardwareAddressTarget()), + arp.IsValid(), + ) + return + default: + log.Infof("%s%s unknown network protocol: %d", prefix, dir, protocol) + return + } + + // Figure out the transport layer info. + transName := "unknown" + srcPort := uint16(0) + dstPort := uint16(0) + details := "" + switch tcpip.TransportProtocolNumber(transProto) { + case header.ICMPv4ProtocolNumber: + transName = "icmp" + hdr, ok := clone.Data().PullUp(header.ICMPv4MinimumSize) + if !ok { + break + } + icmp := header.ICMPv4(hdr) + icmpType := "unknown" + if fragmentOffset == 0 { + switch icmp.Type() { + case header.ICMPv4EchoReply: + icmpType = "echo reply" + case header.ICMPv4DstUnreachable: + icmpType = "destination unreachable" + case header.ICMPv4SrcQuench: + icmpType = "source quench" + case header.ICMPv4Redirect: + icmpType = "redirect" + case header.ICMPv4Echo: + icmpType = "echo" + case header.ICMPv4TimeExceeded: + icmpType = "time exceeded" + case header.ICMPv4ParamProblem: + icmpType = "param problem" + case header.ICMPv4Timestamp: + icmpType = "timestamp" + case header.ICMPv4TimestampReply: + icmpType = "timestamp reply" + case header.ICMPv4InfoRequest: + icmpType = "info request" + case header.ICMPv4InfoReply: + icmpType = "info reply" + } + } + log.Infof("%s%s %s %s -> %s %s len:%d id:%04x code:%d", prefix, dir, transName, src, dst, icmpType, size, id, icmp.Code()) + return + + case header.ICMPv6ProtocolNumber: + transName = "icmp" + hdr, ok := clone.Data().PullUp(header.ICMPv6MinimumSize) + if !ok { + break + } + icmp := header.ICMPv6(hdr) + icmpType := "unknown" + switch icmp.Type() { + case header.ICMPv6DstUnreachable: + icmpType = "destination unreachable" + case header.ICMPv6PacketTooBig: + icmpType = "packet too big" + case header.ICMPv6TimeExceeded: + icmpType = "time exceeded" + case header.ICMPv6ParamProblem: + icmpType = "param problem" + case header.ICMPv6EchoRequest: + icmpType = "echo request" + case header.ICMPv6EchoReply: + icmpType = "echo reply" + case header.ICMPv6RouterSolicit: + icmpType = "router solicit" + case header.ICMPv6RouterAdvert: + icmpType = "router advert" + case header.ICMPv6NeighborSolicit: + icmpType = "neighbor solicit" + case header.ICMPv6NeighborAdvert: + icmpType = "neighbor advert" + case header.ICMPv6RedirectMsg: + icmpType = "redirect message" + } + log.Infof("%s%s %s %s -> %s %s len:%d id:%04x code:%d", prefix, dir, transName, src, dst, icmpType, size, id, icmp.Code()) + return + + case header.UDPProtocolNumber: + transName = "udp" + if ok := parse.UDP(clone); !ok { + break + } + + udp := header.UDP(clone.TransportHeader().Slice()) + if fragmentOffset == 0 { + srcPort = udp.SourcePort() + dstPort = udp.DestinationPort() + details = fmt.Sprintf("xsum: 0x%x", udp.Checksum()) + size -= header.UDPMinimumSize + } + + case header.TCPProtocolNumber: + transName = "tcp" + if ok := parse.TCP(clone); !ok { + break + } + + tcp := header.TCP(clone.TransportHeader().Slice()) + if fragmentOffset == 0 { + offset := int(tcp.DataOffset()) + if offset < header.TCPMinimumSize { + details += fmt.Sprintf("invalid packet: tcp data offset too small %d", offset) + break + } + if size := clone.Data().Size() + len(tcp); offset > size && !moreFragments { + details += fmt.Sprintf("invalid packet: tcp data offset %d larger than tcp packet length %d", offset, size) + break + } + + srcPort = tcp.SourcePort() + dstPort = tcp.DestinationPort() + size -= uint16(offset) + + // Initialize the TCP flags. + flags := tcp.Flags() + details = fmt.Sprintf("flags:%s seqnum:%d ack:%d win:%d xsum:0x%x", flags, tcp.SequenceNumber(), tcp.AckNumber(), tcp.WindowSize(), tcp.Checksum()) + if flags&header.TCPFlagSyn != 0 { + details += fmt.Sprintf(" options:%+v", header.ParseSynOptions(tcp.Options(), flags&header.TCPFlagAck != 0)) + } else { + details += fmt.Sprintf(" options:%+v", tcp.ParsedOptions()) + } + } + + default: + log.Infof("%s%s %s -> %s unknown transport protocol: %d", prefix, dir, src, dst, transProto) + return + } + + if pkt.GSOOptions.Type != stack.GSONone { + details += fmt.Sprintf(" gso:%#v", pkt.GSOOptions) + } + + log.Infof("%s%s %s %s:%d -> %s:%d len:%d id:0x%04x %s", prefix, dir, transName, src, srcPort, dst, dstPort, size, id, details) +} + +// trimmedClone clones the packet buffer to not modify the original. It trims +// anything before the network header. +func trimmedClone(pkt *stack.PacketBuffer) *stack.PacketBuffer { + // We don't clone the original packet buffer so that the new packet buffer + // does not have any of its headers set. + // + // We trim the link headers from the cloned buffer as the sniffer doesn't + // handle link headers. + buf := pkt.ToBuffer() + buf.TrimFront(int64(len(pkt.VirtioNetHeader().Slice()))) + buf.TrimFront(int64(len(pkt.LinkHeader().Slice()))) + return stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf}) +} diff --git a/pkg/tcpip/link/sniffer/sniffer_state_autogen.go b/pkg/tcpip/link/sniffer/sniffer_state_autogen.go new file mode 100644 index 0000000..b6e8015 --- /dev/null +++ b/pkg/tcpip/link/sniffer/sniffer_state_autogen.go @@ -0,0 +1,47 @@ +// automatically generated by stateify. + +package sniffer + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/sniffer.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "Endpoint", + "writer", + "maxPCAPLen", + "logPrefix", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.Endpoint) + stateSinkObject.Save(1, &e.writer) + stateSinkObject.Save(2, &e.maxPCAPLen) + stateSinkObject.Save(3, &e.logPrefix) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.Endpoint) + stateSourceObject.Load(1, &e.writer) + stateSourceObject.Load(2, &e.maxPCAPLen) + stateSourceObject.Load(3, &e.logPrefix) +} + +func init() { + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/stopfd/stopfd.go b/pkg/tcpip/link/stopfd/stopfd.go new file mode 100644 index 0000000..b761aad --- /dev/null +++ b/pkg/tcpip/link/stopfd/stopfd.go @@ -0,0 +1,54 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package stopfd provides an type that can be used to signal the stop of a dispatcher. +package stopfd + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// StopFD is an eventfd used to signal the stop of a dispatcher. +// +// +stateify savable +type StopFD struct { + EFD int +} + +// New returns a new, initialized StopFD. +func New() (StopFD, error) { + efd, err := unix.Eventfd(0, unix.EFD_NONBLOCK) + if err != nil { + return StopFD{EFD: -1}, fmt.Errorf("failed to create eventfd: %w", err) + } + return StopFD{EFD: efd}, nil +} + +// Stop writes to the eventfd and notifies the dispatcher to stop. It does not +// block. +func (sf *StopFD) Stop() { + increment := []byte{1, 0, 0, 0, 0, 0, 0, 0} + if n, err := unix.Write(sf.EFD, increment); n != len(increment) || err != nil { + // There are two possible errors documented in eventfd(2) for writing: + // 1. We are writing 8 bytes and not 0xffffffffffffff, thus no EINVAL. + // 2. stop is only supposed to be called once, it can't reach the limit, + // thus no EAGAIN. + panic(fmt.Sprintf("write(EFD) = (%d, %s), want (%d, nil)", n, err, len(increment))) + } +} diff --git a/pkg/tcpip/link/stopfd/stopfd_state_autogen.go b/pkg/tcpip/link/stopfd/stopfd_state_autogen.go new file mode 100644 index 0000000..c075f5f --- /dev/null +++ b/pkg/tcpip/link/stopfd/stopfd_state_autogen.go @@ -0,0 +1,41 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package stopfd + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (sf *StopFD) StateTypeName() string { + return "pkg/tcpip/link/stopfd.StopFD" +} + +func (sf *StopFD) StateFields() []string { + return []string{ + "EFD", + } +} + +func (sf *StopFD) beforeSave() {} + +// +checklocksignore +func (sf *StopFD) StateSave(stateSinkObject state.Sink) { + sf.beforeSave() + stateSinkObject.Save(0, &sf.EFD) +} + +func (sf *StopFD) afterLoad(context.Context) {} + +// +checklocksignore +func (sf *StopFD) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &sf.EFD) +} + +func init() { + state.Register((*StopFD)(nil)) +} diff --git a/pkg/tcpip/link/tun/device.go b/pkg/tcpip/link/tun/device.go new file mode 100644 index 0000000..f94b6e6 --- /dev/null +++ b/pkg/tcpip/link/tun/device.go @@ -0,0 +1,447 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tun + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/context" + "github.com/sagernet/gvisor/pkg/errors/linuxerr" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/link/channel" + "github.com/sagernet/gvisor/pkg/tcpip/link/packetsocket" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +const ( + // drivers/net/tun.c:tun_net_init() + defaultDevMtu = 1500 + + // Queue length for outbound packet, arriving at fd side for read. Overflow + // causes packet drops. gVisor implementation-specific. + defaultDevOutQueueLen = 1024 +) + +var zeroMAC [6]byte + +// Device is an opened /dev/net/tun device. +// +// +stateify savable +type Device struct { + waiter.Queue + + mu deviceRWMutex `state:"nosave"` + endpoint *tunEndpoint + notifyHandle *channel.NotificationHandle + flags Flags +} + +// Flags set properties of a Device +// +// +stateify savable +type Flags struct { + TUN bool + TAP bool + NoPacketInfo bool + Exclusive bool +} + +// beforeSave is invoked by stateify. +func (d *Device) beforeSave() { + d.mu.Lock() + defer d.mu.Unlock() + // TODO(b/110961832): Restore the device to stack. At this moment, the stack + // is not savable. + if d.endpoint != nil { + panic("/dev/net/tun does not support save/restore when a device is associated with it.") + } +} + +func (d *Device) SetPersistent(v bool) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.endpoint == nil { + return linuxerr.EBADFD + } + + d.endpoint.setPersistent(v) + + return nil +} + +// Release implements fs.FileOperations.Release. +func (d *Device) Release(ctx context.Context) { + d.mu.Lock() + defer d.mu.Unlock() + + // Decrease refcount if there is an endpoint associated with this file. + if d.endpoint != nil { + d.endpoint.Drain() + d.endpoint.RemoveNotify(d.notifyHandle) + d.endpoint.DecRef(ctx) + d.endpoint = nil + } +} + +// SetIff services TUNSETIFF ioctl(2) request. +func (d *Device) SetIff(ctx context.Context, s *stack.Stack, name string, flags Flags) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.endpoint != nil { + return linuxerr.EINVAL + } + + // Input validation. + if (flags.TAP && flags.TUN) || (!flags.TAP && !flags.TUN) { + return linuxerr.EINVAL + } + + prefix := "tun" + if flags.TAP { + prefix = "tap" + } + + linkCaps := stack.CapabilityNone + if flags.TAP { + linkCaps |= stack.CapabilityResolutionRequired + } + + endpoint, err := attachOrCreateNIC(ctx, s, name, prefix, linkCaps, flags) + if err != nil { + return err + } + + d.endpoint = endpoint + d.notifyHandle = d.endpoint.AddNotify(d) + d.flags = flags + return nil +} + +func attachOrCreateNIC(ctx context.Context, s *stack.Stack, name, prefix string, linkCaps stack.LinkEndpointCapabilities, flags Flags) (*tunEndpoint, error) { + for { + // 1. Try to attach to an existing NIC. + if name != "" && !flags.Exclusive { + if linkEP := s.GetLinkEndpointByName(name); linkEP != nil { + packetEndpoint, ok := linkEP.(*packetsocket.Endpoint) + if !ok { + // Not a NIC created by tun device. + return nil, linuxerr.EOPNOTSUPP + } + endpoint, ok := packetEndpoint.Child().(*tunEndpoint) + if !ok { + // Not a NIC created by tun device. + return nil, linuxerr.EOPNOTSUPP + } + if !endpoint.TryIncRef() { + // Race detected: NIC got deleted in between. + continue + } + return endpoint, nil + } + } + + // 2. Creating a new NIC. + id := s.NextNICID() + endpoint := &tunEndpoint{ + Endpoint: channel.New(defaultDevOutQueueLen, defaultDevMtu, ""), + stack: s, + nicID: id, + name: name, + isTap: prefix == "tap", + } + endpoint.InitRefs() + endpoint.Endpoint.LinkEPCapabilities = linkCaps + if endpoint.name == "" { + endpoint.name = fmt.Sprintf("%s%d", prefix, id) + } + err := s.CreateNICWithOptions(endpoint.nicID, packetsocket.New(endpoint), stack.NICOptions{ + Name: endpoint.name, + }) + switch err.(type) { + case nil: + return endpoint, nil + case *tcpip.ErrDuplicateNICID: + endpoint.DecRef(ctx) + if !flags.Exclusive { + // Race detected: A NIC has been created in between. + continue + } + return nil, linuxerr.EEXIST + default: + endpoint.DecRef(ctx) + return nil, linuxerr.EINVAL + } + } +} + +// MTU returns the tun endpoint MTU (maximum transmission unit). +func (d *Device) MTU() (uint32, error) { + d.mu.RLock() + endpoint := d.endpoint + d.mu.RUnlock() + if endpoint == nil { + return 0, linuxerr.EBADFD + } + if !endpoint.IsAttached() { + return 0, linuxerr.EIO + } + return endpoint.MTU(), nil +} + +// Write inject one inbound packet to the network interface. +func (d *Device) Write(data *buffer.View) (int64, error) { + d.mu.RLock() + endpoint := d.endpoint + d.mu.RUnlock() + if endpoint == nil { + return 0, linuxerr.EBADFD + } + if !endpoint.IsAttached() { + return 0, linuxerr.EIO + } + + dataLen := int64(data.Size()) + + // Packet information. + var pktInfoHdr PacketInfoHeader + if !d.flags.NoPacketInfo { + if dataLen < PacketInfoHeaderSize { + // Ignore bad packet. + return dataLen, nil + } + pktInfoHdrView := data.Clone() + defer pktInfoHdrView.Release() + pktInfoHdrView.CapLength(PacketInfoHeaderSize) + pktInfoHdr = PacketInfoHeader(pktInfoHdrView.AsSlice()) + data.TrimFront(PacketInfoHeaderSize) + } + + // Ethernet header (TAP only). + var ethHdr header.Ethernet + if d.flags.TAP { + if data.Size() < header.EthernetMinimumSize { + // Ignore bad packet. + return dataLen, nil + } + ethHdrView := data.Clone() + defer ethHdrView.Release() + ethHdrView.CapLength(header.EthernetMinimumSize) + ethHdr = header.Ethernet(ethHdrView.AsSlice()) + data.TrimFront(header.EthernetMinimumSize) + } + + // Try to determine network protocol number, default zero. + var protocol tcpip.NetworkProtocolNumber + switch { + case pktInfoHdr != nil: + protocol = pktInfoHdr.Protocol() + case ethHdr != nil: + protocol = ethHdr.Type() + case d.flags.TUN: + // TUN interface with IFF_NO_PI enabled, thus + // we need to determine protocol from version field + version := data.AsSlice()[0] >> 4 + if version == 4 { + protocol = header.IPv4ProtocolNumber + } else if version == 6 { + protocol = header.IPv6ProtocolNumber + } + } + + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: len(ethHdr), + Payload: buffer.MakeWithView(data.Clone()), + }) + defer pkt.DecRef() + copy(pkt.LinkHeader().Push(len(ethHdr)), ethHdr) + endpoint.InjectInbound(protocol, pkt) + return dataLen, nil +} + +// Read reads one outgoing packet from the network interface. +func (d *Device) Read() (*buffer.View, error) { + d.mu.RLock() + endpoint := d.endpoint + d.mu.RUnlock() + if endpoint == nil { + return nil, linuxerr.EBADFD + } + + pkt := endpoint.Read() + if pkt == nil { + return nil, linuxerr.ErrWouldBlock + } + v := d.encodePkt(pkt) + pkt.DecRef() + return v, nil +} + +// encodePkt encodes packet for fd side. +func (d *Device) encodePkt(pkt *stack.PacketBuffer) *buffer.View { + var view *buffer.View + + // Packet information. + if !d.flags.NoPacketInfo { + view = buffer.NewView(PacketInfoHeaderSize + pkt.Size()) + view.Grow(PacketInfoHeaderSize) + hdr := PacketInfoHeader(view.AsSlice()) + hdr.Encode(&PacketInfoFields{ + Protocol: pkt.NetworkProtocolNumber, + }) + pktView := pkt.ToView() + view.Write(pktView.AsSlice()) + pktView.Release() + } else { + view = pkt.ToView() + } + + return view +} + +// Name returns the name of the attached network interface. Empty string if +// unattached. +func (d *Device) Name() string { + d.mu.RLock() + defer d.mu.RUnlock() + if d.endpoint != nil { + return d.endpoint.name + } + return "" +} + +// Flags returns the flags set for d. Zero value if unset. +func (d *Device) Flags() Flags { + d.mu.RLock() + defer d.mu.RUnlock() + return d.flags +} + +// Readiness implements watier.Waitable.Readiness. +func (d *Device) Readiness(mask waiter.EventMask) waiter.EventMask { + if mask&waiter.ReadableEvents != 0 { + d.mu.RLock() + endpoint := d.endpoint + d.mu.RUnlock() + if endpoint != nil && endpoint.NumQueued() == 0 { + mask &= ^waiter.ReadableEvents + } + } + return mask & (waiter.ReadableEvents | waiter.WritableEvents) +} + +// WriteNotify implements channel.Notification.WriteNotify. +func (d *Device) WriteNotify() { + d.Notify(waiter.ReadableEvents) +} + +// tunEndpoint is the link endpoint for the NIC created by the tun device. +// +// It is ref-counted as multiple opening files can attach to the same NIC. +// The last owner is responsible for deleting the NIC. +// +// +stateify savable +type tunEndpoint struct { + tunEndpointRefs + *channel.Endpoint + + stack *stack.Stack + nicID tcpip.NICID + name string + isTap bool + persistent atomicbitops.Bool + closed atomicbitops.Bool + + mu endpointMutex `state:"nosave"` + onCloseAction func() `state:"nosave"` +} + +func (e *tunEndpoint) setPersistent(v bool) { + old := e.persistent.Swap(v) + if old == v { + return + } + if v { + e.IncRef() + } else { + e.DecRef(context.Background()) + } +} + +func (e *tunEndpoint) Close() { + if e.closed.Swap(true) { + return + } + + if e.persistent.Load() { + e.DecRef(context.Background()) + } + e.mu.Lock() + action := e.onCloseAction + e.onCloseAction = nil + e.mu.Unlock() + if action != nil { + action() + } + e.Endpoint.Close() +} + +// SetOnCloseAction implements stack.LinkEndpoint. +func (e *tunEndpoint) SetOnCloseAction(action func()) { + e.mu.Lock() + defer e.mu.Unlock() + e.onCloseAction = action +} + +// DecRef decrements refcount of e, removing NIC if it reaches 0. +func (e *tunEndpoint) DecRef(ctx context.Context) { + e.tunEndpointRefs.DecRef(func() { + e.Close() + }) +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (e *tunEndpoint) ARPHardwareType() header.ARPHardwareType { + if e.isTap { + return header.ARPHardwareEther + } + return header.ARPHardwareNone +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *tunEndpoint) AddHeader(pkt *stack.PacketBuffer) { + if !e.isTap { + return + } + eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize)) + eth.Encode(&header.EthernetFields{ + SrcAddr: pkt.EgressRoute.LocalLinkAddress, + DstAddr: pkt.EgressRoute.RemoteLinkAddress, + Type: pkt.NetworkProtocolNumber, + }) +} + +// MaxHeaderLength returns the maximum size of the link layer header. +func (e *tunEndpoint) MaxHeaderLength() uint16 { + if e.isTap { + return header.EthernetMinimumSize + } + return 0 +} diff --git a/pkg/tcpip/link/tun/device_mutex.go b/pkg/tcpip/link/tun/device_mutex.go new file mode 100644 index 0000000..8381889 --- /dev/null +++ b/pkg/tcpip/link/tun/device_mutex.go @@ -0,0 +1,96 @@ +package tun + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type deviceRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var devicelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type devicelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *deviceRWMutex) Lock() { + locking.AddGLock(deviceprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *deviceRWMutex) NestedLock(i devicelockNameIndex) { + locking.AddGLock(deviceprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *deviceRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(deviceprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *deviceRWMutex) NestedUnlock(i devicelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(deviceprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *deviceRWMutex) RLock() { + locking.AddGLock(deviceprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *deviceRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(deviceprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *deviceRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *deviceRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *deviceRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var deviceprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func deviceinitLockNames() {} + +func init() { + deviceinitLockNames() + deviceprefixIndex = locking.NewMutexClass(reflect.TypeOf(deviceRWMutex{}), devicelockNames) +} diff --git a/pkg/tcpip/link/tun/endpoint_mutex.go b/pkg/tcpip/link/tun/endpoint_mutex.go new file mode 100644 index 0000000..a57d0e1 --- /dev/null +++ b/pkg/tcpip/link/tun/endpoint_mutex.go @@ -0,0 +1,64 @@ +package tun + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type endpointMutex struct { + mu sync.Mutex +} + +var endpointprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointMutex) Unlock() { + locking.DelGLock(endpointprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointMutex) NestedUnlock(i endpointlockNameIndex) { + locking.DelGLock(endpointprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/tun/protocol.go b/pkg/tcpip/link/tun/protocol.go new file mode 100644 index 0000000..8fac9ef --- /dev/null +++ b/pkg/tcpip/link/tun/protocol.go @@ -0,0 +1,56 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tun + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // PacketInfoHeaderSize is the size of the packet information header. + PacketInfoHeaderSize = 4 + + offsetFlags = 0 + offsetProtocol = 2 +) + +// PacketInfoFields contains fields sent through the wire if IFF_NO_PI flag is +// not set. +type PacketInfoFields struct { + Flags uint16 + Protocol tcpip.NetworkProtocolNumber +} + +// PacketInfoHeader is the wire representation of the packet information sent if +// IFF_NO_PI flag is not set. +type PacketInfoHeader []byte + +// Encode encodes f into h. +func (h PacketInfoHeader) Encode(f *PacketInfoFields) { + binary.BigEndian.PutUint16(h[offsetFlags:][:2], f.Flags) + binary.BigEndian.PutUint16(h[offsetProtocol:][:2], uint16(f.Protocol)) +} + +// Flags returns the flag field in h. +func (h PacketInfoHeader) Flags() uint16 { + return binary.BigEndian.Uint16(h[offsetFlags:]) +} + +// Protocol returns the protocol field in h. +func (h PacketInfoHeader) Protocol() tcpip.NetworkProtocolNumber { + return tcpip.NetworkProtocolNumber(binary.BigEndian.Uint16(h[offsetProtocol:])) +} diff --git a/pkg/tcpip/link/tun/tun_endpoint_refs.go b/pkg/tcpip/link/tun/tun_endpoint_refs.go new file mode 100644 index 0000000..e0c817d --- /dev/null +++ b/pkg/tcpip/link/tun/tun_endpoint_refs.go @@ -0,0 +1,141 @@ +package tun + +import ( + "context" + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/refs" +) + +// enableLogging indicates whether reference-related events should be logged (with +// stack traces). This is false by default and should only be set to true for +// debugging purposes, as it can generate an extremely large amount of output +// and drastically degrade performance. +const tunEndpointenableLogging = false + +// obj is used to customize logging. Note that we use a pointer to T so that +// we do not copy the entire object when passed as a format parameter. +var tunEndpointobj *tunEndpoint + +// Refs implements refs.RefCounter. It keeps a reference count using atomic +// operations and calls the destructor when the count reaches zero. +// +// NOTE: Do not introduce additional fields to the Refs struct. It is used by +// many filesystem objects, and we want to keep it as small as possible (i.e., +// the same size as using an int64 directly) to avoid taking up extra cache +// space. In general, this template should not be extended at the cost of +// performance. If it does not offer enough flexibility for a particular object +// (example: b/187877947), we should implement the RefCounter/CheckedObject +// interfaces manually. +// +// +stateify savable +type tunEndpointRefs struct { + // refCount is composed of two fields: + // + // [32-bit speculative references]:[32-bit real references] + // + // Speculative references are used for TryIncRef, to avoid a CompareAndSwap + // loop. See IncRef, DecRef and TryIncRef for details of how these fields are + // used. + refCount atomicbitops.Int64 +} + +// InitRefs initializes r with one reference and, if enabled, activates leak +// checking. +func (r *tunEndpointRefs) InitRefs() { + r.refCount.RacyStore(1) + refs.Register(r) +} + +// RefType implements refs.CheckedObject.RefType. +func (r *tunEndpointRefs) RefType() string { + return fmt.Sprintf("%T", tunEndpointobj)[1:] +} + +// LeakMessage implements refs.CheckedObject.LeakMessage. +func (r *tunEndpointRefs) LeakMessage() string { + return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs()) +} + +// LogRefs implements refs.CheckedObject.LogRefs. +func (r *tunEndpointRefs) LogRefs() bool { + return tunEndpointenableLogging +} + +// ReadRefs returns the current number of references. The returned count is +// inherently racy and is unsafe to use without external synchronization. +func (r *tunEndpointRefs) ReadRefs() int64 { + return r.refCount.Load() +} + +// IncRef implements refs.RefCounter.IncRef. +// +//go:nosplit +func (r *tunEndpointRefs) IncRef() { + v := r.refCount.Add(1) + if tunEndpointenableLogging { + refs.LogIncRef(r, v) + } + if v <= 1 { + panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType())) + } +} + +// TryIncRef implements refs.TryRefCounter.TryIncRef. +// +// To do this safely without a loop, a speculative reference is first acquired +// on the object. This allows multiple concurrent TryIncRef calls to distinguish +// other TryIncRef calls from genuine references held. +// +//go:nosplit +func (r *tunEndpointRefs) TryIncRef() bool { + const speculativeRef = 1 << 32 + if v := r.refCount.Add(speculativeRef); int32(v) == 0 { + + r.refCount.Add(-speculativeRef) + return false + } + + v := r.refCount.Add(-speculativeRef + 1) + if tunEndpointenableLogging { + refs.LogTryIncRef(r, v) + } + return true +} + +// DecRef implements refs.RefCounter.DecRef. +// +// Note that speculative references are counted here. Since they were added +// prior to real references reaching zero, they will successfully convert to +// real references. In other words, we see speculative references only in the +// following case: +// +// A: TryIncRef [speculative increase => sees non-negative references] +// B: DecRef [real decrease] +// A: TryIncRef [transform speculative to real] +// +//go:nosplit +func (r *tunEndpointRefs) DecRef(destroy func()) { + v := r.refCount.Add(-1) + if tunEndpointenableLogging { + refs.LogDecRef(r, v) + } + switch { + case v < 0: + panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType())) + + case v == 0: + refs.Unregister(r) + + if destroy != nil { + destroy() + } + } +} + +func (r *tunEndpointRefs) afterLoad(context.Context) { + if r.ReadRefs() > 0 { + refs.Register(r) + } +} diff --git a/pkg/tcpip/link/tun/tun_state_autogen.go b/pkg/tcpip/link/tun/tun_state_autogen.go new file mode 100644 index 0000000..0567115 --- /dev/null +++ b/pkg/tcpip/link/tun/tun_state_autogen.go @@ -0,0 +1,152 @@ +// automatically generated by stateify. + +package tun + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (d *Device) StateTypeName() string { + return "pkg/tcpip/link/tun.Device" +} + +func (d *Device) StateFields() []string { + return []string{ + "Queue", + "endpoint", + "notifyHandle", + "flags", + } +} + +// +checklocksignore +func (d *Device) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.Queue) + stateSinkObject.Save(1, &d.endpoint) + stateSinkObject.Save(2, &d.notifyHandle) + stateSinkObject.Save(3, &d.flags) +} + +func (d *Device) afterLoad(context.Context) {} + +// +checklocksignore +func (d *Device) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.Queue) + stateSourceObject.Load(1, &d.endpoint) + stateSourceObject.Load(2, &d.notifyHandle) + stateSourceObject.Load(3, &d.flags) +} + +func (f *Flags) StateTypeName() string { + return "pkg/tcpip/link/tun.Flags" +} + +func (f *Flags) StateFields() []string { + return []string{ + "TUN", + "TAP", + "NoPacketInfo", + "Exclusive", + } +} + +func (f *Flags) beforeSave() {} + +// +checklocksignore +func (f *Flags) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.TUN) + stateSinkObject.Save(1, &f.TAP) + stateSinkObject.Save(2, &f.NoPacketInfo) + stateSinkObject.Save(3, &f.Exclusive) +} + +func (f *Flags) afterLoad(context.Context) {} + +// +checklocksignore +func (f *Flags) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.TUN) + stateSourceObject.Load(1, &f.TAP) + stateSourceObject.Load(2, &f.NoPacketInfo) + stateSourceObject.Load(3, &f.Exclusive) +} + +func (e *tunEndpoint) StateTypeName() string { + return "pkg/tcpip/link/tun.tunEndpoint" +} + +func (e *tunEndpoint) StateFields() []string { + return []string{ + "tunEndpointRefs", + "Endpoint", + "stack", + "nicID", + "name", + "isTap", + "persistent", + "closed", + } +} + +func (e *tunEndpoint) beforeSave() {} + +// +checklocksignore +func (e *tunEndpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.tunEndpointRefs) + stateSinkObject.Save(1, &e.Endpoint) + stateSinkObject.Save(2, &e.stack) + stateSinkObject.Save(3, &e.nicID) + stateSinkObject.Save(4, &e.name) + stateSinkObject.Save(5, &e.isTap) + stateSinkObject.Save(6, &e.persistent) + stateSinkObject.Save(7, &e.closed) +} + +func (e *tunEndpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *tunEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.tunEndpointRefs) + stateSourceObject.Load(1, &e.Endpoint) + stateSourceObject.Load(2, &e.stack) + stateSourceObject.Load(3, &e.nicID) + stateSourceObject.Load(4, &e.name) + stateSourceObject.Load(5, &e.isTap) + stateSourceObject.Load(6, &e.persistent) + stateSourceObject.Load(7, &e.closed) +} + +func (r *tunEndpointRefs) StateTypeName() string { + return "pkg/tcpip/link/tun.tunEndpointRefs" +} + +func (r *tunEndpointRefs) StateFields() []string { + return []string{ + "refCount", + } +} + +func (r *tunEndpointRefs) beforeSave() {} + +// +checklocksignore +func (r *tunEndpointRefs) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.refCount) +} + +// +checklocksignore +func (r *tunEndpointRefs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.refCount) + stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) }) +} + +func init() { + state.Register((*Device)(nil)) + state.Register((*Flags)(nil)) + state.Register((*tunEndpoint)(nil)) + state.Register((*tunEndpointRefs)(nil)) +} diff --git a/pkg/tcpip/link/tun/tun_unsafe.go b/pkg/tcpip/link/tun/tun_unsafe.go new file mode 100644 index 0000000..db4338e --- /dev/null +++ b/pkg/tcpip/link/tun/tun_unsafe.go @@ -0,0 +1,65 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package tun contains methods to open TAP and TUN devices. +package tun + +import ( + "unsafe" + + "golang.org/x/sys/unix" +) + +// Open opens the specified TUN device, sets it to non-blocking mode, and +// returns its file descriptor. +func Open(name string) (int, error) { + return open(name, unix.IFF_TUN|unix.IFF_NO_PI) +} + +// OpenTAP opens the specified TAP device, sets it to non-blocking mode, and +// returns its file descriptor. +func OpenTAP(name string) (int, error) { + return open(name, unix.IFF_TAP|unix.IFF_NO_PI) +} + +func open(name string, flags uint16) (int, error) { + fd, err := unix.Open("/dev/net/tun", unix.O_RDWR, 0) + if err != nil { + return -1, err + } + + var ifr struct { + name [16]byte + flags uint16 + _ [22]byte + } + + copy(ifr.name[:], name) + ifr.flags = flags + _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), unix.TUNSETIFF, uintptr(unsafe.Pointer(&ifr))) + if errno != 0 { + unix.Close(fd) + return -1, errno + } + + if err = unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return -1, err + } + + return fd, nil +} diff --git a/pkg/tcpip/link/tun/tun_unsafe_state_autogen.go b/pkg/tcpip/link/tun/tun_unsafe_state_autogen.go new file mode 100644 index 0000000..8d82ad3 --- /dev/null +++ b/pkg/tcpip/link/tun/tun_unsafe_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package tun diff --git a/pkg/tcpip/link/veth/endpoint_mutex.go b/pkg/tcpip/link/veth/endpoint_mutex.go new file mode 100644 index 0000000..d5dbcd4 --- /dev/null +++ b/pkg/tcpip/link/veth/endpoint_mutex.go @@ -0,0 +1,96 @@ +package veth + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/veth/veth.go b/pkg/tcpip/link/veth/veth.go new file mode 100644 index 0000000..cf6a6ce --- /dev/null +++ b/pkg/tcpip/link/veth/veth.go @@ -0,0 +1,258 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package veth provides the implementation of virtual ethernet device pair. +package veth + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// DefaultBacklogSize is the default size of a veth device's buffer. +const DefaultBacklogSize = 1000 + +var ( + _ stack.LinkEndpoint = (*Endpoint)(nil) + _ stack.GSOEndpoint = (*Endpoint)(nil) +) + +// +stateify savable +type veth struct { + mu vethRWMutex `state:"nosave"` + closed bool + backlogQueue chan vethPacket `state:"nosave"` + mtu uint32 + endpoints [2]Endpoint +} + +func (v *veth) close() { + v.mu.Lock() + closed := v.closed + v.closed = true + v.mu.Unlock() + if closed { + return + } + + for i := range v.endpoints { + e := &v.endpoints[i] + e.mu.Lock() + action := e.onCloseAction + e.onCloseAction = nil + e.mu.Unlock() + if action != nil { + action() + } + } + close(v.backlogQueue) +} + +// +stateify savable +type vethPacket struct { + e *Endpoint + protocol tcpip.NetworkProtocolNumber + pkt *stack.PacketBuffer +} + +// Endpoint is link layer endpoint that redirects packets to a pair veth endpoint. +// +// +stateify savable +type Endpoint struct { + peer *Endpoint + + veth *veth + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher + // linkAddr is the local address of this endpoint. + // + // +checklocks:mu + linkAddr tcpip.LinkAddress + // +checklocks:mu + onCloseAction func() `state:"nosave"` +} + +// NewPair creates a new veth pair. +func NewPair(mtu, backlogQueueSize uint32) (*Endpoint, *Endpoint) { + veth := veth{ + backlogQueue: make(chan vethPacket, backlogQueueSize), + mtu: mtu, + endpoints: [2]Endpoint{ + { + linkAddr: tcpip.GetRandMacAddr(), + }, + { + linkAddr: tcpip.GetRandMacAddr(), + }, + }, + } + a := &veth.endpoints[0] + b := &veth.endpoints[1] + a.peer = b + b.peer = a + a.veth = &veth + b.veth = &veth + go func() { + for t := range veth.backlogQueue { + t.e.InjectInbound(t.protocol, t.pkt) + t.pkt.DecRef() + } + }() + return a, b +} + +// Close closes e. Further packet injections will return an error, and all pending +// packets are discarded. Close may be called concurrently with WritePackets. +func (e *Endpoint) Close() { + e.veth.close() +} + +// InjectInbound injects an inbound packet. If the endpoint is not attached, the +// packet is not delivered. +func (e *Endpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverNetworkPacket(protocol, pkt) + } +} + +// Attach saves the stack network-layer dispatcher for use later when packets +// are injected. +func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + defer e.mu.Unlock() + e.dispatcher = dispatcher +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *Endpoint) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.dispatcher != nil +} + +// MTU implements stack.LinkEndpoint.MTU. +func (e *Endpoint) MTU() uint32 { + e.veth.mu.RLock() + defer e.veth.mu.RUnlock() + return e.veth.mtu +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. +func (e *Endpoint) SetMTU(mtu uint32) { + e.veth.mu.Lock() + defer e.veth.mu.Unlock() + e.veth.mtu = mtu +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities { + // TODO(b/352384218): Enable CapabilityTXChecksumOffload. + return stack.CapabilityRXChecksumOffload | stack.CapabilitySaveRestore +} + +// GSOMaxSize implements stack.GSOEndpoint. +func (*Endpoint) GSOMaxSize() uint32 { + return stack.GVisorGSOMaxSize +} + +// SupportedGSO implements stack.GSOEndpoint. +func (e *Endpoint) SupportedGSO() stack.SupportedGSO { + return stack.GVisorGSOSupported +} + +// MaxHeaderLength returns the maximum size of the link layer header. Given it +// doesn't have a header, it just returns 0. +func (*Endpoint) MaxHeaderLength() uint16 { + return 0 +} + +// LinkAddress returns the link address of this endpoint. +func (e *Endpoint) LinkAddress() tcpip.LinkAddress { + e.mu.RLock() + defer e.mu.RUnlock() + return e.linkAddr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.linkAddr = addr +} + +// WritePackets stores outbound packets into the channel. +// Multiple concurrent calls are permitted. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + e.veth.mu.RLock() + defer e.veth.mu.RUnlock() + + if e.veth.closed { + return 0, nil + } + + n := 0 + for _, pkt := range pkts.AsSlice() { + // In order to properly loop back to the inbound side we must create a + // fresh packet that only contains the underlying payload with no headers + // or struct fields set. We must deep clone the payload to avoid + // two goroutines writing to the same buffer. + // + // TODO(b/240580913): Remove this once IP headers use reference counted + // views instead of raw byte slices. + payload := pkt.ToBuffer() + newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: payload.DeepClone(), + }) + payload.Release() + select { + case (e.veth.backlogQueue) <- vethPacket{ + e: e.peer, + protocol: pkt.NetworkProtocolNumber, + pkt: newPkt, + }: + n++ + default: + newPkt.DecRef() + return n, &tcpip.ErrNoBufferSpace{} + } + } + return n, nil +} + +// Wait implements stack.LinkEndpoint.Wait. +func (*Endpoint) Wait() {} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (*Endpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareNone +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *Endpoint) AddHeader(pkt *stack.PacketBuffer) {} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (e *Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { return true } + +// SetOnCloseAction implements stack.LinkEndpoint. +func (e *Endpoint) SetOnCloseAction(action func()) { + e.mu.Lock() + defer e.mu.Unlock() + e.onCloseAction = action +} diff --git a/pkg/tcpip/link/veth/veth_mutex.go b/pkg/tcpip/link/veth/veth_mutex.go new file mode 100644 index 0000000..2f420df --- /dev/null +++ b/pkg/tcpip/link/veth/veth_mutex.go @@ -0,0 +1,96 @@ +package veth + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type vethRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var vethlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type vethlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *vethRWMutex) Lock() { + locking.AddGLock(vethprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *vethRWMutex) NestedLock(i vethlockNameIndex) { + locking.AddGLock(vethprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *vethRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(vethprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *vethRWMutex) NestedUnlock(i vethlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(vethprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *vethRWMutex) RLock() { + locking.AddGLock(vethprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *vethRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(vethprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *vethRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *vethRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *vethRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var vethprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func vethinitLockNames() {} + +func init() { + vethinitLockNames() + vethprefixIndex = locking.NewMutexClass(reflect.TypeOf(vethRWMutex{}), vethlockNames) +} diff --git a/pkg/tcpip/link/veth/veth_state_autogen.go b/pkg/tcpip/link/veth/veth_state_autogen.go new file mode 100644 index 0000000..c41bbb7 --- /dev/null +++ b/pkg/tcpip/link/veth/veth_state_autogen.go @@ -0,0 +1,111 @@ +// automatically generated by stateify. + +package veth + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (v *veth) StateTypeName() string { + return "pkg/tcpip/link/veth.veth" +} + +func (v *veth) StateFields() []string { + return []string{ + "closed", + "mtu", + "endpoints", + } +} + +func (v *veth) beforeSave() {} + +// +checklocksignore +func (v *veth) StateSave(stateSinkObject state.Sink) { + v.beforeSave() + stateSinkObject.Save(0, &v.closed) + stateSinkObject.Save(1, &v.mtu) + stateSinkObject.Save(2, &v.endpoints) +} + +func (v *veth) afterLoad(context.Context) {} + +// +checklocksignore +func (v *veth) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &v.closed) + stateSourceObject.Load(1, &v.mtu) + stateSourceObject.Load(2, &v.endpoints) +} + +func (v *vethPacket) StateTypeName() string { + return "pkg/tcpip/link/veth.vethPacket" +} + +func (v *vethPacket) StateFields() []string { + return []string{ + "e", + "protocol", + "pkt", + } +} + +func (v *vethPacket) beforeSave() {} + +// +checklocksignore +func (v *vethPacket) StateSave(stateSinkObject state.Sink) { + v.beforeSave() + stateSinkObject.Save(0, &v.e) + stateSinkObject.Save(1, &v.protocol) + stateSinkObject.Save(2, &v.pkt) +} + +func (v *vethPacket) afterLoad(context.Context) {} + +// +checklocksignore +func (v *vethPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &v.e) + stateSourceObject.Load(1, &v.protocol) + stateSourceObject.Load(2, &v.pkt) +} + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/veth.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "peer", + "veth", + "dispatcher", + "linkAddr", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.peer) + stateSinkObject.Save(1, &e.veth) + stateSinkObject.Save(2, &e.dispatcher) + stateSinkObject.Save(3, &e.linkAddr) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.peer) + stateSourceObject.Load(1, &e.veth) + stateSourceObject.Load(2, &e.dispatcher) + stateSourceObject.Load(3, &e.linkAddr) +} + +func init() { + state.Register((*veth)(nil)) + state.Register((*vethPacket)(nil)) + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/waitable/endpoint_mutex.go b/pkg/tcpip/link/waitable/endpoint_mutex.go new file mode 100644 index 0000000..b671008 --- /dev/null +++ b/pkg/tcpip/link/waitable/endpoint_mutex.go @@ -0,0 +1,96 @@ +package waitable + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/waitable/waitable.go b/pkg/tcpip/link/waitable/waitable.go new file mode 100644 index 0000000..eb86b48 --- /dev/null +++ b/pkg/tcpip/link/waitable/waitable.go @@ -0,0 +1,196 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package waitable provides the implementation of data-link layer endpoints +// that wrap other endpoints, and can wait for inflight calls to WritePacket or +// DeliverNetworkPacket to finish (and new ones to be prevented). +// +// Waitable endpoints can be used in the networking stack by calling New(eID) to +// create a new endpoint, where eID is the ID of the endpoint being wrapped, +// and then passing it as an argument to Stack.CreateNIC(). +package waitable + +import ( + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var ( + _ stack.NetworkDispatcher = (*Endpoint)(nil) + _ stack.LinkEndpoint = (*Endpoint)(nil) +) + +// Endpoint is a waitable link-layer endpoint. +// +// +stateify savable +type Endpoint struct { + dispatchGate sync.Gate + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + dispatcher stack.NetworkDispatcher + + writeGate sync.Gate + lower stack.LinkEndpoint +} + +// New creates a new waitable link-layer endpoint. It wraps around another +// endpoint and allows the caller to block new write/dispatch calls and wait for +// the inflight ones to finish before returning. +func New(lower stack.LinkEndpoint) *Endpoint { + return &Endpoint{ + lower: lower, + } +} + +// DeliverNetworkPacket implements stack.NetworkDispatcher.DeliverNetworkPacket. +// It is called by the link-layer endpoint being wrapped when a packet arrives, +// and only forwards to the actual dispatcher if Wait or WaitDispatch haven't +// been called. +func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + if !e.dispatchGate.Enter() { + return + } + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverNetworkPacket(protocol, pkt) + } + e.dispatchGate.Leave() +} + +// DeliverLinkPacket implements stack.NetworkDispatcher. +func (e *Endpoint) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + if !e.dispatchGate.Enter() { + return + } + e.mu.RLock() + d := e.dispatcher + e.mu.RUnlock() + if d != nil { + d.DeliverLinkPacket(protocol, pkt) + } + e.dispatchGate.Leave() +} + +// Attach implements stack.LinkEndpoint.Attach. It saves the dispatcher and +// registers with the lower endpoint as its dispatcher so that "e" is called +// for inbound packets. +func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) { + e.mu.Lock() + e.dispatcher = dispatcher + e.mu.Unlock() + e.lower.Attach(e) +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (e *Endpoint) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.dispatcher != nil +} + +// MTU implements stack.LinkEndpoint.MTU. It just forwards the request to the +// lower endpoint. +func (e *Endpoint) MTU() uint32 { + return e.lower.MTU() +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. It just forwards the request to +// the lower endpoint. +func (e *Endpoint) SetMTU(mtu uint32) { + e.lower.SetMTU(mtu) +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. It just forwards the +// request to the lower endpoint. +func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities { + return e.lower.Capabilities() +} + +// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It just +// forwards the request to the lower endpoint. +func (e *Endpoint) MaxHeaderLength() uint16 { + return e.lower.MaxHeaderLength() +} + +// LinkAddress implements stack.LinkEndpoint.LinkAddress. It just forwards the +// request to the lower endpoint. +func (e *Endpoint) LinkAddress() tcpip.LinkAddress { + return e.lower.LinkAddress() +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. It forwards the +// request to the lower endpoint. +func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + e.mu.Lock() + defer e.mu.Unlock() + e.lower.SetLinkAddress(addr) +} + +// WritePackets implements stack.LinkEndpoint.WritePackets. It is called by +// higher-level protocols to write packets. It only forwards packets to the +// lower endpoint if Wait or WaitWrite haven't been called. +func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + if !e.writeGate.Enter() { + return pkts.Len(), nil + } + + n, err := e.lower.WritePackets(pkts) + e.writeGate.Leave() + return n, err +} + +// WaitWrite prevents new calls to WritePacket from reaching the lower endpoint, +// and waits for inflight ones to finish before returning. +func (e *Endpoint) WaitWrite() { + e.writeGate.Close() +} + +// WaitDispatch prevents new calls to DeliverNetworkPacket from reaching the +// actual dispatcher, and waits for inflight ones to finish before returning. +func (e *Endpoint) WaitDispatch() { + e.dispatchGate.Close() +} + +// Wait implements stack.LinkEndpoint.Wait. +func (e *Endpoint) Wait() {} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (e *Endpoint) ARPHardwareType() header.ARPHardwareType { + return e.lower.ARPHardwareType() +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (e *Endpoint) AddHeader(pkt *stack.PacketBuffer) { + e.lower.AddHeader(pkt) +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (e *Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + return e.lower.ParseHeader(pkt) +} + +// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction. +func (e *Endpoint) SetOnCloseAction(action func()) { + e.lower.SetOnCloseAction(action) +} + +// Close implements stack.LinkEndpoint. +func (e *Endpoint) Close() { + e.lower.Close() +} diff --git a/pkg/tcpip/link/waitable/waitable_state_autogen.go b/pkg/tcpip/link/waitable/waitable_state_autogen.go new file mode 100644 index 0000000..2525a4c --- /dev/null +++ b/pkg/tcpip/link/waitable/waitable_state_autogen.go @@ -0,0 +1,47 @@ +// automatically generated by stateify. + +package waitable + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/link/waitable.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "dispatchGate", + "dispatcher", + "writeGate", + "lower", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.dispatchGate) + stateSinkObject.Save(1, &e.dispatcher) + stateSinkObject.Save(2, &e.writeGate) + stateSinkObject.Save(3, &e.lower) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.dispatchGate) + stateSourceObject.Load(1, &e.dispatcher) + stateSourceObject.Load(2, &e.writeGate) + stateSourceObject.Load(3, &e.lower) +} + +func init() { + state.Register((*Endpoint)(nil)) +} diff --git a/pkg/tcpip/link/xdp/endpoint.go b/pkg/tcpip/link/xdp/endpoint.go new file mode 100644 index 0000000..6990dce --- /dev/null +++ b/pkg/tcpip/link/xdp/endpoint.go @@ -0,0 +1,421 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package xdp provides link layer endpoints backed by AF_XDP sockets. +package xdp + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/rawfile" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/link/qdisc/fifo" + "github.com/sagernet/gvisor/pkg/tcpip/link/stopfd" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/xdp" + "golang.org/x/sys/unix" +) + +// TODO(b/240191988): Turn off GSO, GRO, and LRO. Limit veth MTU to 1500. + +// MTU is sized to ensure packets fit inside a 2048 byte XDP frame. +const MTU = 1500 + +var _ stack.LinkEndpoint = (*endpoint)(nil) + +// +stateify savable +type endpoint struct { + // fd is the underlying AF_XDP socket. + fd int + + // caps holds the endpoint capabilities. + caps stack.LinkEndpointCapabilities + + // closed is a function to be called when the FD's peer (if any) closes + // its end of the communication pipe. + // TODO(b/341946753): Restore when netstack is savable. + closed func(tcpip.Error) `state:"nosave"` + + mu endpointRWMutex `state:"nosave"` + // +checkloks:mu + networkDispatcher stack.NetworkDispatcher + + // wg keeps track of running goroutines. + wg sync.WaitGroup `state:"nosave"` + + // control is used to control the AF_XDP socket. + control *xdp.ControlBlock + + // stopFD is used to stop the dispatch loop. + stopFD stopfd.StopFD + + // addr is the address of the endpoint. + // + // +checklocks:mu + addr tcpip.LinkAddress +} + +// Options specify the details about the fd-based endpoint to be created. +type Options struct { + // FD is used to read/write packets. + FD int + + // ClosedFunc is a function to be called when an endpoint's peer (if + // any) closes its end of the communication pipe. + ClosedFunc func(tcpip.Error) + + // Address is the link address for this endpoint. + Address tcpip.LinkAddress + + // SaveRestore if true, indicates that this NIC capability set should + // include CapabilitySaveRestore + SaveRestore bool + + // DisconnectOk if true, indicates that this NIC capability set should + // include CapabilityDisconnectOk. + DisconnectOk bool + + // TXChecksumOffload if true, indicates that this endpoints capability + // set should include CapabilityTXChecksumOffload. + TXChecksumOffload bool + + // RXChecksumOffload if true, indicates that this endpoints capability + // set should include CapabilityRXChecksumOffload. + RXChecksumOffload bool + + // InterfaceIndex is the interface index of the underlying device. + InterfaceIndex int + + // Bind is true when we're responsible for binding the AF_XDP socket to + // a device. When false, another process is expected to bind for us. + Bind bool + + // GRO enables generic receive offload. + GRO bool +} + +// New creates a new endpoint from an AF_XDP socket. +func New(opts *Options) (stack.LinkEndpoint, error) { + caps := stack.CapabilityResolutionRequired + if opts.RXChecksumOffload { + caps |= stack.CapabilityRXChecksumOffload + } + + if opts.TXChecksumOffload { + caps |= stack.CapabilityTXChecksumOffload + } + + if opts.SaveRestore { + caps |= stack.CapabilitySaveRestore + } + + if opts.DisconnectOk { + caps |= stack.CapabilityDisconnectOk + } + + if err := unix.SetNonblock(opts.FD, true); err != nil { + return nil, fmt.Errorf("unix.SetNonblock(%v) failed: %v", opts.FD, err) + } + + ep := &endpoint{ + fd: opts.FD, + caps: caps, + closed: opts.ClosedFunc, + addr: opts.Address, + } + + stopFD, err := stopfd.New() + if err != nil { + return nil, err + } + ep.stopFD = stopFD + + // Use a 2MB UMEM to match the PACKET_MMAP dispatcher. There will be + // 1024 UMEM frames, and each queue will have 512 descriptors. Having + // fewer descriptors than frames prevents RX and TX from starving each + // other. + // TODO(b/240191988): Consider different numbers of descriptors for + // different queues. + const ( + frameSize = 2048 + umemSize = 1 << 21 + nFrames = umemSize / frameSize + ) + xdpOpts := xdp.Opts{ + NFrames: nFrames, + FrameSize: frameSize, + NDescriptors: nFrames / 2, + Bind: opts.Bind, + } + ep.control, err = xdp.NewFromSocket(opts.FD, uint32(opts.InterfaceIndex), 0 /* queueID */, xdpOpts) + if err != nil { + return nil, fmt.Errorf("failed to create AF_XDP dispatcher: %v", err) + } + + ep.control.UMEM.Lock() + defer ep.control.UMEM.Unlock() + + ep.control.Fill.FillAll(&ep.control.UMEM) + + return ep, nil +} + +// Attach launches the goroutine that reads packets from the file descriptor and +// dispatches them via the provided dispatcher. If one is already attached, +// then nothing happens. +// +// Attach implements stack.LinkEndpoint.Attach. +func (ep *endpoint) Attach(networkDispatcher stack.NetworkDispatcher) { + ep.mu.Lock() + defer ep.mu.Unlock() + // nil means the NIC is being removed. + if networkDispatcher == nil && ep.IsAttached() { + ep.stopFD.Stop() + ep.Wait() + ep.networkDispatcher = nil + return + } + if networkDispatcher != nil && ep.networkDispatcher == nil { + ep.networkDispatcher = networkDispatcher + // Link endpoints are not savable. When transportation endpoints are + // saved, they stop sending outgoing packets and all incoming packets + // are rejected. + ep.wg.Add(1) + go func() { // S/R-SAFE: See above. + defer ep.wg.Done() + for { + cont, err := ep.dispatch() + if err != nil || !cont { + if ep.closed != nil { + ep.closed(err) + } + return + } + } + }() + } +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (ep *endpoint) IsAttached() bool { + ep.mu.RLock() + defer ep.mu.RUnlock() + return ep.networkDispatcher != nil +} + +// MTU implements stack.LinkEndpoint.MTU. It returns the value initialized +// during construction. +func (ep *endpoint) MTU() uint32 { + return MTU +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. It has no impact. +func (*endpoint) SetMTU(uint32) {} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (ep *endpoint) Capabilities() stack.LinkEndpointCapabilities { + return ep.caps +} + +// MaxHeaderLength returns the maximum size of the link-layer header. +func (ep *endpoint) MaxHeaderLength() uint16 { + return uint16(header.EthernetMinimumSize) +} + +// LinkAddress returns the link address of this endpoint. +func (ep *endpoint) LinkAddress() tcpip.LinkAddress { + ep.mu.RLock() + defer ep.mu.RUnlock() + return ep.addr +} + +// SetLinkAddress implemens stack.LinkEndpoint.SetLinkAddress +func (ep *endpoint) SetLinkAddress(addr tcpip.LinkAddress) { + ep.mu.Lock() + defer ep.mu.Unlock() + ep.addr = addr +} + +// Wait implements stack.LinkEndpoint.Wait. It waits for the endpoint to stop +// reading from its FD. +func (ep *endpoint) Wait() { + ep.wg.Wait() +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (ep *endpoint) AddHeader(pkt *stack.PacketBuffer) { + // Add ethernet header if needed. + eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize)) + eth.Encode(&header.EthernetFields{ + SrcAddr: pkt.EgressRoute.LocalLinkAddress, + DstAddr: pkt.EgressRoute.RemoteLinkAddress, + Type: pkt.NetworkProtocolNumber, + }) +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (ep *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { + _, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize) + return ok +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (ep *endpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareEther +} + +// WritePackets writes outbound packets to the underlying file descriptors. If +// one is not currently writable, the packet is dropped. +// +// Each packet in pkts should have the following fields populated: +// - pkt.EgressRoute +// - pkt.NetworkProtocolNumber +// +// The following should not be populated, as GSO is not supported with XDP. +// - pkt.GSOOptions +func (ep *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { + // We expect to be called via fifo, which imposes a limit of + // fifo.BatchSize. + var preallocatedBatch [fifo.BatchSize]unix.XDPDesc + batch := preallocatedBatch[:0] + + ep.control.UMEM.Lock() + + ep.control.Completion.FreeAll(&ep.control.UMEM) + + // Reserve TX queue descriptors and umem buffers + nReserved, index := ep.control.TX.Reserve(&ep.control.UMEM, uint32(pkts.Len())) + if nReserved == 0 { + ep.control.UMEM.Unlock() + return 0, &tcpip.ErrNoBufferSpace{} + } + + // Allocate UMEM space. In order to release the UMEM lock as soon as + // possible we allocate up-front. + for _, pkt := range pkts.AsSlice() { + batch = append(batch, unix.XDPDesc{ + Addr: ep.control.UMEM.AllocFrame(), + Len: uint32(pkt.Size()), + }) + } + + for i, pkt := range pkts.AsSlice() { + // Copy packets into UMEM frame. + frame := ep.control.UMEM.Get(batch[i]) + offset := 0 + var view *buffer.View + views, pktOffset := pkt.AsViewList() + for view = views.Front(); view != nil && pktOffset >= view.Size(); view = view.Next() { + pktOffset -= view.Size() + } + offset += copy(frame[offset:], view.AsSlice()[pktOffset:]) + for view = view.Next(); view != nil; view = view.Next() { + offset += copy(frame[offset:], view.AsSlice()) + } + ep.control.TX.Set(index+uint32(i), batch[i]) + } + + // Notify the kernel that there're packets to write. + ep.control.TX.Notify() + + // TODO(b/240191988): Explore more fine-grained locking. We shouldn't + // need to hold the UMEM lock for the whole duration of packet copying. + ep.control.UMEM.Unlock() + + return pkts.Len(), nil +} + +func (ep *endpoint) dispatch() (bool, tcpip.Error) { + var views []*buffer.View + + for { + stopped, errno := rawfile.BlockingPollUntilStopped(ep.stopFD.EFD, ep.fd, unix.POLLIN|unix.POLLERR) + if errno != 0 { + if errno == unix.EINTR { + continue + } + return !stopped, tcpip.TranslateErrno(errno) + } + if stopped { + return true, nil + } + + // Avoid the cost of the poll syscall if possible by peeking + // until there are no packets left. + for { + // We can receive multiple packets at once. + nReceived, rxIndex := ep.control.RX.Peek() + + if nReceived == 0 { + break + } + + // Reuse views to avoid allocating. + views = views[:0] + + // Populate views quickly so that we can release frames + // back to the kernel. + ep.control.UMEM.Lock() + for i := uint32(0); i < nReceived; i++ { + // Copy packet bytes into a view and free up the + // buffer. + descriptor := ep.control.RX.Get(rxIndex + i) + data := ep.control.UMEM.Get(descriptor) + view := buffer.NewView(len(data)) + view.Write(data) + views = append(views, view) + ep.control.UMEM.FreeFrame(descriptor.Addr) + } + ep.control.Fill.FillAll(&ep.control.UMEM) + ep.control.UMEM.Unlock() + + // Process each packet. + ep.mu.RLock() + d := ep.networkDispatcher + ep.mu.RUnlock() + for i := uint32(0); i < nReceived; i++ { + view := views[i] + data := view.AsSlice() + + netProto := header.Ethernet(data).Type() + + // Wrap the packet in a PacketBuffer and send it up the stack. + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithView(view), + }) + // AF_XDP packets always have a link header. + if !ep.ParseHeader(pkt) { + panic("ParseHeader(_) must succeed") + } + d.DeliverNetworkPacket(netProto, pkt) + pkt.DecRef() + } + // Tell the kernel that we're done with these + // descriptors in the RX queue. + ep.control.RX.Release(nReceived) + } + } +} + +// Close implements stack.LinkEndpoint. +func (*endpoint) Close() {} + +// SetOnCloseAction implements stack.LinkEndpoint. +func (*endpoint) SetOnCloseAction(func()) {} diff --git a/pkg/tcpip/link/xdp/endpoint_mutex.go b/pkg/tcpip/link/xdp/endpoint_mutex.go new file mode 100644 index 0000000..c527707 --- /dev/null +++ b/pkg/tcpip/link/xdp/endpoint_mutex.go @@ -0,0 +1,96 @@ +package xdp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/link/xdp/xdp_state_autogen.go b/pkg/tcpip/link/xdp/xdp_state_autogen.go new file mode 100644 index 0000000..64c8228 --- /dev/null +++ b/pkg/tcpip/link/xdp/xdp_state_autogen.go @@ -0,0 +1,56 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package xdp + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (ep *endpoint) StateTypeName() string { + return "pkg/tcpip/link/xdp.endpoint" +} + +func (ep *endpoint) StateFields() []string { + return []string{ + "fd", + "caps", + "networkDispatcher", + "control", + "stopFD", + "addr", + } +} + +func (ep *endpoint) beforeSave() {} + +// +checklocksignore +func (ep *endpoint) StateSave(stateSinkObject state.Sink) { + ep.beforeSave() + stateSinkObject.Save(0, &ep.fd) + stateSinkObject.Save(1, &ep.caps) + stateSinkObject.Save(2, &ep.networkDispatcher) + stateSinkObject.Save(3, &ep.control) + stateSinkObject.Save(4, &ep.stopFD) + stateSinkObject.Save(5, &ep.addr) +} + +func (ep *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (ep *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ep.fd) + stateSourceObject.Load(1, &ep.caps) + stateSourceObject.Load(2, &ep.networkDispatcher) + stateSourceObject.Load(3, &ep.control) + stateSourceObject.Load(4, &ep.stopFD) + stateSourceObject.Load(5, &ep.addr) +} + +func init() { + state.Register((*endpoint)(nil)) +} diff --git a/pkg/tcpip/network/arp/arp.go b/pkg/tcpip/network/arp/arp.go new file mode 100644 index 0000000..13b7479 --- /dev/null +++ b/pkg/tcpip/network/arp/arp.go @@ -0,0 +1,416 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package arp implements the ARP network protocol. It is used to resolve +// IPv4 addresses into link-local MAC addresses, and advertises IPv4 +// addresses of its stack with the local network. +package arp + +import ( + "fmt" + "reflect" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // ProtocolNumber is the ARP protocol number. + ProtocolNumber = header.ARPProtocolNumber +) + +var ( + _ stack.DuplicateAddressDetector = (*endpoint)(nil) + _ stack.LinkAddressResolver = (*endpoint)(nil) + _ ip.DADProtocol = (*endpoint)(nil) +) + +// ARP endpoints need to implement stack.NetworkEndpoint because the stack +// considers the layer above the link-layer a network layer; the only +// facility provided by the stack to deliver packets to a layer above +// the link-layer is via stack.NetworkEndpoint.HandlePacket. +var _ stack.NetworkEndpoint = (*endpoint)(nil) + +// +stateify savable +type endpoint struct { + protocol *protocol + + // enabled is set to 1 when the NIC is enabled and 0 when it is disabled. + enabled atomicbitops.Uint32 + + nic stack.NetworkInterface + stats sharedStats + + // mu protects annotated fields below. + mu sync.Mutex `state:"nosave"` + + // +checklocks:mu + dad ip.DAD +} + +// CheckDuplicateAddress implements stack.DuplicateAddressDetector. +func (e *endpoint) CheckDuplicateAddress(addr tcpip.Address, h stack.DADCompletionHandler) stack.DADCheckAddressDisposition { + e.mu.Lock() + defer e.mu.Unlock() + return e.dad.CheckDuplicateAddressLocked(addr, h) +} + +// SetDADConfigurations implements stack.DuplicateAddressDetector. +func (e *endpoint) SetDADConfigurations(c stack.DADConfigurations) { + e.mu.Lock() + defer e.mu.Unlock() + e.dad.SetConfigsLocked(c) +} + +// DuplicateAddressProtocol implements stack.DuplicateAddressDetector. +func (*endpoint) DuplicateAddressProtocol() tcpip.NetworkProtocolNumber { + return header.IPv4ProtocolNumber +} + +// SendDADMessage implements ip.DADProtocol. +func (e *endpoint) SendDADMessage(addr tcpip.Address, _ []byte) tcpip.Error { + return e.sendARPRequest(header.IPv4Any, addr, header.EthernetBroadcastAddress) +} + +func (e *endpoint) Enable() tcpip.Error { + if !e.nic.Enabled() { + return &tcpip.ErrNotPermitted{} + } + + e.setEnabled(true) + return nil +} + +func (e *endpoint) Enabled() bool { + return e.nic.Enabled() && e.isEnabled() +} + +// isEnabled returns true if the endpoint is enabled, regardless of the +// enabled status of the NIC. +func (e *endpoint) isEnabled() bool { + return e.enabled.Load() == 1 +} + +// setEnabled sets the enabled status for the endpoint. +func (e *endpoint) setEnabled(v bool) { + if v { + e.enabled.Store(1) + } else { + e.enabled.Store(0) + } +} + +func (e *endpoint) Disable() { + e.setEnabled(false) +} + +// DefaultTTL is unused for ARP. It implements stack.NetworkEndpoint. +func (*endpoint) DefaultTTL() uint8 { + return 0 +} + +func (e *endpoint) MTU() uint32 { + lmtu := e.nic.MTU() + return lmtu - uint32(e.MaxHeaderLength()) +} + +func (e *endpoint) MaxHeaderLength() uint16 { + return e.nic.MaxHeaderLength() + header.ARPSize +} + +func (*endpoint) Close() {} + +func (*endpoint) WritePacket(*stack.Route, stack.NetworkHeaderParams, *stack.PacketBuffer) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// NetworkProtocolNumber implements stack.NetworkEndpoint.NetworkProtocolNumber. +func (*endpoint) NetworkProtocolNumber() tcpip.NetworkProtocolNumber { + return ProtocolNumber +} + +func (*endpoint) WriteHeaderIncludedPacket(*stack.Route, *stack.PacketBuffer) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { + stats := e.stats.arp + stats.packetsReceived.Increment() + + if !e.isEnabled() { + stats.disabledPacketsReceived.Increment() + return + } + + if _, _, ok := e.protocol.Parse(pkt); !ok { + stats.malformedPacketsReceived.Increment() + return + } + + h := header.ARP(pkt.NetworkHeader().Slice()) + if !h.IsValid() { + stats.malformedPacketsReceived.Increment() + return + } + + switch h.Op() { + case header.ARPRequest: + stats.requestsReceived.Increment() + localAddr := tcpip.AddrFrom4Slice(h.ProtocolAddressTarget()) + + if !e.nic.CheckLocalAddress(header.IPv4ProtocolNumber, localAddr) { + stats.requestsReceivedUnknownTargetAddress.Increment() + return // we have no useful answer, ignore the request + } + + remoteAddr := tcpip.AddrFrom4Slice(h.ProtocolAddressSender()) + remoteLinkAddr := tcpip.LinkAddress(h.HardwareAddressSender()) + + switch err := e.nic.HandleNeighborProbe(header.IPv4ProtocolNumber, remoteAddr, remoteLinkAddr); err.(type) { + case nil: + case *tcpip.ErrNotSupported: + // The stack may support ARP but the NIC may not need link resolution. + default: + panic(fmt.Sprintf("unexpected error when informing NIC of neighbor probe message: %s", err)) + } + + respPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(e.nic.MaxHeaderLength()) + header.ARPSize, + }) + defer respPkt.DecRef() + packet := header.ARP(respPkt.NetworkHeader().Push(header.ARPSize)) + respPkt.NetworkProtocolNumber = ProtocolNumber + packet.SetIPv4OverEthernet() + packet.SetOp(header.ARPReply) + // TODO(gvisor.dev/issue/4582): check copied length once TAP devices have a + // link address. + _ = copy(packet.HardwareAddressSender(), e.nic.LinkAddress()) + if n := copy(packet.ProtocolAddressSender(), h.ProtocolAddressTarget()); n != header.IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, header.IPv4AddressSize)) + } + origSender := h.HardwareAddressSender() + if n := copy(packet.HardwareAddressTarget(), origSender); n != header.EthernetAddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, header.EthernetAddressSize)) + } + if n := copy(packet.ProtocolAddressTarget(), h.ProtocolAddressSender()); n != header.IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, header.IPv4AddressSize)) + } + + // As per RFC 826, under Packet Reception: + // Swap hardware and protocol fields, putting the local hardware and + // protocol addresses in the sender fields. + // + // Send the packet to the (new) target hardware address on the same + // hardware on which the request was received. + if err := e.nic.WritePacketToRemote(tcpip.LinkAddress(origSender), respPkt); err != nil { + stats.outgoingRepliesDropped.Increment() + } else { + stats.outgoingRepliesSent.Increment() + } + + case header.ARPReply: + stats.repliesReceived.Increment() + addr := tcpip.AddrFrom4Slice(h.ProtocolAddressSender()) + linkAddr := tcpip.LinkAddress(h.HardwareAddressSender()) + + e.mu.Lock() + e.dad.StopLocked(addr, &stack.DADDupAddrDetected{HolderLinkAddress: linkAddr}) + e.mu.Unlock() + + switch err := e.nic.HandleNeighborConfirmation(header.IPv4ProtocolNumber, addr, linkAddr, stack.ReachabilityConfirmationFlags{ + // Only unicast ARP replies are considered solicited. Broadcast replies + // are gratuitous ARP replies and should not move neighbor entries to the + // reachable state. + Solicited: pkt.PktType == tcpip.PacketHost, + // If a different link address is received than the one cached, the entry + // should always go to Stale. + Override: false, + // ARP does not distinguish between router and non-router hosts. + IsRouter: false, + }); err.(type) { + case nil: + case *tcpip.ErrNotSupported: + // The stack may support ARP but the NIC may not need link resolution. + default: + panic(fmt.Sprintf("unexpected error when informing NIC of neighbor confirmation message: %s", err)) + } + } +} + +// Stats implements stack.NetworkEndpoint. +func (e *endpoint) Stats() stack.NetworkEndpointStats { + return &e.stats.localStats +} + +var _ stack.NetworkProtocol = (*protocol)(nil) + +// +stateify savable +type protocol struct { + stack *stack.Stack + options Options +} + +func (p *protocol) Number() tcpip.NetworkProtocolNumber { return ProtocolNumber } +func (p *protocol) MinimumPacketSize() int { return header.ARPSize } + +func (*protocol) ParseAddresses([]byte) (src, dst tcpip.Address) { + return tcpip.Address{}, tcpip.Address{} +} + +func (p *protocol) NewEndpoint(nic stack.NetworkInterface, _ stack.TransportDispatcher) stack.NetworkEndpoint { + e := &endpoint{ + protocol: p, + nic: nic, + } + + e.mu.Lock() + e.dad.Init(&e.mu, p.options.DADConfigs, ip.DADOptions{ + Clock: p.stack.Clock(), + SecureRNG: p.stack.SecureRNG().Reader, + // ARP does not support sending nonce values. + NonceSize: 0, + Protocol: e, + NICID: nic.ID(), + }) + e.mu.Unlock() + + tcpip.InitStatCounters(reflect.ValueOf(&e.stats.localStats).Elem()) + + stackStats := p.stack.Stats() + e.stats.arp.init(&e.stats.localStats.ARP, &stackStats.ARP) + + return e +} + +// LinkAddressProtocol implements stack.LinkAddressResolver.LinkAddressProtocol. +func (*endpoint) LinkAddressProtocol() tcpip.NetworkProtocolNumber { + return header.IPv4ProtocolNumber +} + +// LinkAddressRequest implements stack.LinkAddressResolver.LinkAddressRequest. +func (e *endpoint) LinkAddressRequest(targetAddr, localAddr tcpip.Address, remoteLinkAddr tcpip.LinkAddress) tcpip.Error { + stats := e.stats.arp + + if len(remoteLinkAddr) == 0 { + remoteLinkAddr = header.EthernetBroadcastAddress + } + + if localAddr.BitLen() == 0 { + addr, err := e.nic.PrimaryAddress(header.IPv4ProtocolNumber) + if err != nil { + return err + } + + if addr.Address.BitLen() == 0 { + stats.outgoingRequestInterfaceHasNoLocalAddressErrors.Increment() + return &tcpip.ErrNetworkUnreachable{} + } + + localAddr = addr.Address + } else if !e.nic.CheckLocalAddress(header.IPv4ProtocolNumber, localAddr) { + stats.outgoingRequestBadLocalAddressErrors.Increment() + return &tcpip.ErrBadLocalAddress{} + } + + return e.sendARPRequest(localAddr, targetAddr, remoteLinkAddr) +} + +func (e *endpoint) sendARPRequest(localAddr, targetAddr tcpip.Address, remoteLinkAddr tcpip.LinkAddress) tcpip.Error { + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(e.MaxHeaderLength()), + }) + defer pkt.DecRef() + h := header.ARP(pkt.NetworkHeader().Push(header.ARPSize)) + pkt.NetworkProtocolNumber = ProtocolNumber + h.SetIPv4OverEthernet() + h.SetOp(header.ARPRequest) + // TODO(gvisor.dev/issue/4582): check copied length once TAP devices have a + // link address. + _ = copy(h.HardwareAddressSender(), e.nic.LinkAddress()) + if n := copy(h.ProtocolAddressSender(), localAddr.AsSlice()); n != header.IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, header.IPv4AddressSize)) + } + if n := copy(h.ProtocolAddressTarget(), targetAddr.AsSlice()); n != header.IPv4AddressSize { + panic(fmt.Sprintf("copied %d bytes, expected %d bytes", n, header.IPv4AddressSize)) + } + + stats := e.stats.arp + if err := e.nic.WritePacketToRemote(remoteLinkAddr, pkt); err != nil { + stats.outgoingRequestsDropped.Increment() + return err + } + stats.outgoingRequestsSent.Increment() + return nil +} + +// ResolveStaticAddress implements stack.LinkAddressResolver.ResolveStaticAddress. +func (*endpoint) ResolveStaticAddress(addr tcpip.Address) (tcpip.LinkAddress, bool) { + if addr == header.IPv4Broadcast { + return header.EthernetBroadcastAddress, true + } + if header.IsV4MulticastAddress(addr) { + return header.EthernetAddressFromMulticastIPv4Address(addr), true + } + return tcpip.LinkAddress([]byte(nil)), false +} + +// SetOption implements stack.NetworkProtocol.SetOption. +func (*protocol) SetOption(tcpip.SettableNetworkProtocolOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// Option implements stack.NetworkProtocol.Option. +func (*protocol) Option(tcpip.GettableNetworkProtocolOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// Close implements stack.TransportProtocol.Close. +func (*protocol) Close() {} + +// Wait implements stack.TransportProtocol.Wait. +func (*protocol) Wait() {} + +// Parse implements stack.NetworkProtocol.Parse. +func (*protocol) Parse(pkt *stack.PacketBuffer) (proto tcpip.TransportProtocolNumber, hasTransportHdr bool, ok bool) { + return 0, false, parse.ARP(pkt) +} + +// Options holds options to configure a protocol. +// +// +stateify savable +type Options struct { + // DADConfigs is the default DAD configurations used by ARP endpoints. + DADConfigs stack.DADConfigurations +} + +// NewProtocolWithOptions returns an ARP network protocol factory that +// will return an ARP network protocol with the provided options. +func NewProtocolWithOptions(opts Options) stack.NetworkProtocolFactory { + return func(s *stack.Stack) stack.NetworkProtocol { + return &protocol{ + stack: s, + options: opts, + } + } +} + +// NewProtocol returns an ARP network protocol. +func NewProtocol(s *stack.Stack) stack.NetworkProtocol { + return NewProtocolWithOptions(Options{})(s) +} diff --git a/pkg/tcpip/network/arp/arp_state_autogen.go b/pkg/tcpip/network/arp/arp_state_autogen.go new file mode 100644 index 0000000..d36bc1a --- /dev/null +++ b/pkg/tcpip/network/arp/arp_state_autogen.go @@ -0,0 +1,219 @@ +// automatically generated by stateify. + +package arp + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/network/arp.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "protocol", + "enabled", + "nic", + "stats", + "dad", + } +} + +func (e *endpoint) beforeSave() {} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.protocol) + stateSinkObject.Save(1, &e.enabled) + stateSinkObject.Save(2, &e.nic) + stateSinkObject.Save(3, &e.stats) + stateSinkObject.Save(4, &e.dad) +} + +func (e *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.protocol) + stateSourceObject.Load(1, &e.enabled) + stateSourceObject.Load(2, &e.nic) + stateSourceObject.Load(3, &e.stats) + stateSourceObject.Load(4, &e.dad) +} + +func (p *protocol) StateTypeName() string { + return "pkg/tcpip/network/arp.protocol" +} + +func (p *protocol) StateFields() []string { + return []string{ + "stack", + "options", + } +} + +func (p *protocol) beforeSave() {} + +// +checklocksignore +func (p *protocol) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.stack) + stateSinkObject.Save(1, &p.options) +} + +func (p *protocol) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.stack) + stateSourceObject.Load(1, &p.options) +} + +func (o *Options) StateTypeName() string { + return "pkg/tcpip/network/arp.Options" +} + +func (o *Options) StateFields() []string { + return []string{ + "DADConfigs", + } +} + +func (o *Options) beforeSave() {} + +// +checklocksignore +func (o *Options) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.DADConfigs) +} + +func (o *Options) afterLoad(context.Context) {} + +// +checklocksignore +func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.DADConfigs) +} + +func (s *Stats) StateTypeName() string { + return "pkg/tcpip/network/arp.Stats" +} + +func (s *Stats) StateFields() []string { + return []string{ + "ARP", + } +} + +func (s *Stats) beforeSave() {} + +// +checklocksignore +func (s *Stats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.ARP) +} + +func (s *Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.ARP) +} + +func (s *sharedStats) StateTypeName() string { + return "pkg/tcpip/network/arp.sharedStats" +} + +func (s *sharedStats) StateFields() []string { + return []string{ + "localStats", + "arp", + } +} + +func (s *sharedStats) beforeSave() {} + +// +checklocksignore +func (s *sharedStats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.localStats) + stateSinkObject.Save(1, &s.arp) +} + +func (s *sharedStats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *sharedStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.localStats) + stateSourceObject.Load(1, &s.arp) +} + +func (m *multiCounterARPStats) StateTypeName() string { + return "pkg/tcpip/network/arp.multiCounterARPStats" +} + +func (m *multiCounterARPStats) StateFields() []string { + return []string{ + "packetsReceived", + "disabledPacketsReceived", + "malformedPacketsReceived", + "requestsReceived", + "requestsReceivedUnknownTargetAddress", + "outgoingRequestInterfaceHasNoLocalAddressErrors", + "outgoingRequestBadLocalAddressErrors", + "outgoingRequestsDropped", + "outgoingRequestsSent", + "repliesReceived", + "outgoingRepliesDropped", + "outgoingRepliesSent", + } +} + +func (m *multiCounterARPStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterARPStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.packetsReceived) + stateSinkObject.Save(1, &m.disabledPacketsReceived) + stateSinkObject.Save(2, &m.malformedPacketsReceived) + stateSinkObject.Save(3, &m.requestsReceived) + stateSinkObject.Save(4, &m.requestsReceivedUnknownTargetAddress) + stateSinkObject.Save(5, &m.outgoingRequestInterfaceHasNoLocalAddressErrors) + stateSinkObject.Save(6, &m.outgoingRequestBadLocalAddressErrors) + stateSinkObject.Save(7, &m.outgoingRequestsDropped) + stateSinkObject.Save(8, &m.outgoingRequestsSent) + stateSinkObject.Save(9, &m.repliesReceived) + stateSinkObject.Save(10, &m.outgoingRepliesDropped) + stateSinkObject.Save(11, &m.outgoingRepliesSent) +} + +func (m *multiCounterARPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterARPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.packetsReceived) + stateSourceObject.Load(1, &m.disabledPacketsReceived) + stateSourceObject.Load(2, &m.malformedPacketsReceived) + stateSourceObject.Load(3, &m.requestsReceived) + stateSourceObject.Load(4, &m.requestsReceivedUnknownTargetAddress) + stateSourceObject.Load(5, &m.outgoingRequestInterfaceHasNoLocalAddressErrors) + stateSourceObject.Load(6, &m.outgoingRequestBadLocalAddressErrors) + stateSourceObject.Load(7, &m.outgoingRequestsDropped) + stateSourceObject.Load(8, &m.outgoingRequestsSent) + stateSourceObject.Load(9, &m.repliesReceived) + stateSourceObject.Load(10, &m.outgoingRepliesDropped) + stateSourceObject.Load(11, &m.outgoingRepliesSent) +} + +func init() { + state.Register((*endpoint)(nil)) + state.Register((*protocol)(nil)) + state.Register((*Options)(nil)) + state.Register((*Stats)(nil)) + state.Register((*sharedStats)(nil)) + state.Register((*multiCounterARPStats)(nil)) +} diff --git a/pkg/tcpip/network/arp/stats.go b/pkg/tcpip/network/arp/stats.go new file mode 100644 index 0000000..674fefa --- /dev/null +++ b/pkg/tcpip/network/arp/stats.go @@ -0,0 +1,74 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arp + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var _ stack.NetworkEndpointStats = (*Stats)(nil) + +// Stats holds statistics related to ARP. +// +// +stateify savable +type Stats struct { + // ARP holds ARP statistics. + ARP tcpip.ARPStats +} + +// IsNetworkEndpointStats implements stack.NetworkEndpointStats. +func (*Stats) IsNetworkEndpointStats() {} + +// +stateify savable +type sharedStats struct { + localStats Stats + arp multiCounterARPStats +} + +// LINT.IfChange(multiCounterARPStats) + +// +stateify savable +type multiCounterARPStats struct { + packetsReceived tcpip.MultiCounterStat + disabledPacketsReceived tcpip.MultiCounterStat + malformedPacketsReceived tcpip.MultiCounterStat + requestsReceived tcpip.MultiCounterStat + requestsReceivedUnknownTargetAddress tcpip.MultiCounterStat + outgoingRequestInterfaceHasNoLocalAddressErrors tcpip.MultiCounterStat + outgoingRequestBadLocalAddressErrors tcpip.MultiCounterStat + outgoingRequestsDropped tcpip.MultiCounterStat + outgoingRequestsSent tcpip.MultiCounterStat + repliesReceived tcpip.MultiCounterStat + outgoingRepliesDropped tcpip.MultiCounterStat + outgoingRepliesSent tcpip.MultiCounterStat +} + +func (m *multiCounterARPStats) init(a, b *tcpip.ARPStats) { + m.packetsReceived.Init(a.PacketsReceived, b.PacketsReceived) + m.disabledPacketsReceived.Init(a.DisabledPacketsReceived, b.DisabledPacketsReceived) + m.malformedPacketsReceived.Init(a.MalformedPacketsReceived, b.MalformedPacketsReceived) + m.requestsReceived.Init(a.RequestsReceived, b.RequestsReceived) + m.requestsReceivedUnknownTargetAddress.Init(a.RequestsReceivedUnknownTargetAddress, b.RequestsReceivedUnknownTargetAddress) + m.outgoingRequestInterfaceHasNoLocalAddressErrors.Init(a.OutgoingRequestInterfaceHasNoLocalAddressErrors, b.OutgoingRequestInterfaceHasNoLocalAddressErrors) + m.outgoingRequestBadLocalAddressErrors.Init(a.OutgoingRequestBadLocalAddressErrors, b.OutgoingRequestBadLocalAddressErrors) + m.outgoingRequestsDropped.Init(a.OutgoingRequestsDropped, b.OutgoingRequestsDropped) + m.outgoingRequestsSent.Init(a.OutgoingRequestsSent, b.OutgoingRequestsSent) + m.repliesReceived.Init(a.RepliesReceived, b.RepliesReceived) + m.outgoingRepliesDropped.Init(a.OutgoingRepliesDropped, b.OutgoingRepliesDropped) + m.outgoingRepliesSent.Init(a.OutgoingRepliesSent, b.OutgoingRepliesSent) +} + +// LINT.ThenChange(../../tcpip.go:ARPStats) diff --git a/pkg/tcpip/network/hash/hash.go b/pkg/tcpip/network/hash/hash.go new file mode 100644 index 0000000..71fd827 --- /dev/null +++ b/pkg/tcpip/network/hash/hash.go @@ -0,0 +1,93 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hash contains utility functions for hashing. +package hash + +import ( + "encoding/binary" + + "github.com/sagernet/gvisor/pkg/rand" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +var hashIV = RandN32(1)[0] + +// RandN32 generates a slice of n cryptographic random 32-bit numbers. +func RandN32(n int) []uint32 { + b := make([]byte, 4*n) + if _, err := rand.Read(b); err != nil { + panic("unable to get random numbers: " + err.Error()) + } + r := make([]uint32, n) + for i := range r { + r[i] = binary.LittleEndian.Uint32(b[4*i : (4*i + 4)]) + } + return r +} + +// Hash3Words calculates the Jenkins hash of 3 32-bit words. This is adapted +// from linux. +func Hash3Words(a, b, c, initval uint32) uint32 { + const iv = 0xdeadbeef + (3 << 2) + initval += iv + + a += initval + b += initval + c += initval + + c ^= b + c -= rol32(b, 14) + a ^= c + a -= rol32(c, 11) + b ^= a + b -= rol32(a, 25) + c ^= b + c -= rol32(b, 16) + a ^= c + a -= rol32(c, 4) + b ^= a + b -= rol32(a, 14) + c ^= b + c -= rol32(b, 24) + + return c +} + +// IPv4FragmentHash computes the hash of the IPv4 fragment as suggested in RFC 791. +func IPv4FragmentHash(h header.IPv4) uint32 { + x := uint32(h.ID())<<16 | uint32(h.Protocol()) + t := h.SourceAddress().As4() + y := uint32(t[0]) | uint32(t[1])<<8 | uint32(t[2])<<16 | uint32(t[3])<<24 + t = h.DestinationAddress().As4() + z := uint32(t[0]) | uint32(t[1])<<8 | uint32(t[2])<<16 | uint32(t[3])<<24 + return Hash3Words(x, y, z, hashIV) +} + +// IPv6FragmentHash computes the hash of the ipv6 fragment. +// Unlike IPv4, the protocol is not used to compute the hash. +// RFC 2640 (sec 4.5) is not very sharp on this aspect. +// As a reference, also Linux ignores the protocol to compute +// the hash (inet6_hash_frag). +func IPv6FragmentHash(h header.IPv6, id uint32) uint32 { + t := h.SourceAddress().As16() + y := uint32(t[0]) | uint32(t[1])<<8 | uint32(t[2])<<16 | uint32(t[3])<<24 + t = h.DestinationAddress().As16() + z := uint32(t[0]) | uint32(t[1])<<8 | uint32(t[2])<<16 | uint32(t[3])<<24 + return Hash3Words(id, y, z, hashIV) +} + +func rol32(v, shift uint32) uint32 { + return (v << shift) | (v >> ((-shift) & 31)) +} diff --git a/pkg/tcpip/network/hash/hash_state_autogen.go b/pkg/tcpip/network/hash/hash_state_autogen.go new file mode 100644 index 0000000..9467fe2 --- /dev/null +++ b/pkg/tcpip/network/hash/hash_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package hash diff --git a/pkg/tcpip/network/internal/fragmentation/fragmentation.go b/pkg/tcpip/network/internal/fragmentation/fragmentation.go new file mode 100644 index 0000000..a53603c --- /dev/null +++ b/pkg/tcpip/network/internal/fragmentation/fragmentation.go @@ -0,0 +1,375 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fragmentation contains the implementation of IP fragmentation. +// It is based on RFC 791, RFC 815 and RFC 8200. +package fragmentation + +import ( + "errors" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // HighFragThreshold is the threshold at which we start trimming old + // fragmented packets. Linux uses a default value of 4 MB. See + // net.ipv4.ipfrag_high_thresh for more information. + HighFragThreshold = 4 << 20 // 4MB + + // LowFragThreshold is the threshold we reach to when we start dropping + // older fragmented packets. It's important that we keep enough room for newer + // packets to be re-assembled. Hence, this needs to be lower than + // HighFragThreshold enough. Linux uses a default value of 3 MB. See + // net.ipv4.ipfrag_low_thresh for more information. + LowFragThreshold = 3 << 20 // 3MB + + // minBlockSize is the minimum block size for fragments. + minBlockSize = 1 +) + +var ( + // ErrInvalidArgs indicates to the caller that an invalid argument was + // provided. + ErrInvalidArgs = errors.New("invalid args") + + // ErrFragmentOverlap indicates that, during reassembly, a fragment overlaps + // with another one. + ErrFragmentOverlap = errors.New("overlapping fragments") + + // ErrFragmentConflict indicates that, during reassembly, some fragments are + // in conflict with one another. + ErrFragmentConflict = errors.New("conflicting fragments") +) + +// FragmentID is the identifier for a fragment. +// +// +stateify savable +type FragmentID struct { + // Source is the source address of the fragment. + Source tcpip.Address + + // Destination is the destination address of the fragment. + Destination tcpip.Address + + // ID is the identification value of the fragment. + // + // This is a uint32 because IPv6 uses a 32-bit identification value. + ID uint32 + + // The protocol for the packet. + Protocol uint8 +} + +// Fragmentation is the main structure that other modules +// of the stack should use to implement IP Fragmentation. +// +// +stateify savable +type Fragmentation struct { + mu sync.Mutex `state:"nosave"` + highLimit int + lowLimit int + reassemblers map[FragmentID]*reassembler + rList reassemblerList + memSize int + timeout time.Duration + blockSize uint16 + clock tcpip.Clock + releaseJob *tcpip.Job + timeoutHandler TimeoutHandler +} + +// TimeoutHandler is consulted if a packet reassembly has timed out. +type TimeoutHandler interface { + // OnReassemblyTimeout will be called with the first fragment (or nil, if the + // first fragment has not been received) of a packet whose reassembly has + // timed out. + OnReassemblyTimeout(pkt *stack.PacketBuffer) +} + +// NewFragmentation creates a new Fragmentation. +// +// blockSize specifies the fragment block size, in bytes. +// +// highMemoryLimit specifies the limit on the memory consumed +// by the fragments stored by Fragmentation (overhead of internal data-structures +// is not accounted). Fragments are dropped when the limit is reached. +// +// lowMemoryLimit specifies the limit on which we will reach by dropping +// fragments after reaching highMemoryLimit. +// +// reassemblingTimeout specifies the maximum time allowed to reassemble a packet. +// Fragments are lazily evicted only when a new a packet with an +// already existing fragmentation-id arrives after the timeout. +func NewFragmentation(blockSize uint16, highMemoryLimit, lowMemoryLimit int, reassemblingTimeout time.Duration, clock tcpip.Clock, timeoutHandler TimeoutHandler) *Fragmentation { + if lowMemoryLimit >= highMemoryLimit { + lowMemoryLimit = highMemoryLimit + } + + if lowMemoryLimit < 0 { + lowMemoryLimit = 0 + } + + if blockSize < minBlockSize { + blockSize = minBlockSize + } + + f := &Fragmentation{ + reassemblers: make(map[FragmentID]*reassembler), + highLimit: highMemoryLimit, + lowLimit: lowMemoryLimit, + timeout: reassemblingTimeout, + blockSize: blockSize, + clock: clock, + timeoutHandler: timeoutHandler, + } + f.releaseJob = tcpip.NewJob(f.clock, &f.mu, f.releaseReassemblersLocked) + + return f +} + +// Process processes an incoming fragment belonging to an ID and returns a +// complete packet and its protocol number when all the packets belonging to +// that ID have been received. +// +// [first, last] is the range of the fragment bytes. +// +// first must be a multiple of the block size f is configured with. The size +// of the fragment data must be a multiple of the block size, unless there are +// no fragments following this fragment (more set to false). +// +// proto is the protocol number marked in the fragment being processed. It has +// to be given here outside of the FragmentID struct because IPv6 should not use +// the protocol to identify a fragment. +func (f *Fragmentation) Process( + id FragmentID, first, last uint16, more bool, proto uint8, pkt *stack.PacketBuffer) ( + *stack.PacketBuffer, uint8, bool, error, +) { + if first > last { + return nil, 0, false, fmt.Errorf("first=%d is greater than last=%d: %w", first, last, ErrInvalidArgs) + } + + if first%f.blockSize != 0 { + return nil, 0, false, fmt.Errorf("first=%d is not a multiple of block size=%d: %w", first, f.blockSize, ErrInvalidArgs) + } + + fragmentSize := last - first + 1 + if more && fragmentSize%f.blockSize != 0 { + return nil, 0, false, fmt.Errorf("fragment size=%d bytes is not a multiple of block size=%d on non-final fragment: %w", fragmentSize, f.blockSize, ErrInvalidArgs) + } + + if l := pkt.Data().Size(); l != int(fragmentSize) { + return nil, 0, false, fmt.Errorf("got fragment size=%d bytes not equal to the expected fragment size=%d bytes (first=%d last=%d): %w", l, fragmentSize, first, last, ErrInvalidArgs) + } + + f.mu.Lock() + if f.reassemblers == nil { + return nil, 0, false, fmt.Errorf("Release() called before fragmentation processing could finish") + } + + r, ok := f.reassemblers[id] + if !ok { + r = newReassembler(id, f.clock) + f.reassemblers[id] = r + wasEmpty := f.rList.Empty() + f.rList.PushFront(r) + if wasEmpty { + // If we have just pushed a first reassembler into an empty list, we + // should kickstart the release job. The release job will keep + // rescheduling itself until the list becomes empty. + f.releaseReassemblersLocked() + } + } + f.mu.Unlock() + + resPkt, firstFragmentProto, done, memConsumed, err := r.process(first, last, more, proto, pkt) + if err != nil { + // We probably got an invalid sequence of fragments. Just + // discard the reassembler and move on. + f.mu.Lock() + f.release(r, false /* timedOut */) + f.mu.Unlock() + return nil, 0, false, fmt.Errorf("fragmentation processing error: %w", err) + } + f.mu.Lock() + f.memSize += memConsumed + if done { + f.release(r, false /* timedOut */) + } + // Evict reassemblers if we are consuming more memory than highLimit until + // we reach lowLimit. + if f.memSize > f.highLimit { + for f.memSize > f.lowLimit { + tail := f.rList.Back() + if tail == nil { + break + } + f.release(tail, false /* timedOut */) + } + } + f.mu.Unlock() + return resPkt, firstFragmentProto, done, nil +} + +// Release releases all underlying resources. +func (f *Fragmentation) Release() { + f.mu.Lock() + defer f.mu.Unlock() + for _, r := range f.reassemblers { + f.release(r, false /* timedOut */) + } + f.reassemblers = nil +} + +func (f *Fragmentation) release(r *reassembler, timedOut bool) { + // Before releasing a fragment we need to check if r is already marked as done. + // Otherwise, we would delete it twice. + if r.checkDoneOrMark() { + return + } + + delete(f.reassemblers, r.id) + f.rList.Remove(r) + f.memSize -= r.memSize + if f.memSize < 0 { + log.Warningf("memory counter < 0 (%d), this is an accounting bug that requires investigation", f.memSize) + f.memSize = 0 + } + + if h := f.timeoutHandler; timedOut && h != nil { + h.OnReassemblyTimeout(r.pkt) + } + if r.pkt != nil { + r.pkt.DecRef() + r.pkt = nil + } + for _, h := range r.holes { + if h.pkt != nil { + h.pkt.DecRef() + h.pkt = nil + } + } + r.holes = nil +} + +// releaseReassemblersLocked releases already-expired reassemblers, then +// schedules the job to call back itself for the remaining reassemblers if +// any. This function must be called with f.mu locked. +func (f *Fragmentation) releaseReassemblersLocked() { + now := f.clock.NowMonotonic() + for { + // The reassembler at the end of the list is the oldest. + r := f.rList.Back() + if r == nil { + // The list is empty. + break + } + elapsed := now.Sub(r.createdAt) + if f.timeout > elapsed { + // If the oldest reassembler has not expired, schedule the release + // job so that this function is called back when it has expired. + f.releaseJob.Schedule(f.timeout - elapsed) + break + } + // If the oldest reassembler has already expired, release it. + f.release(r, true /* timedOut*/) + } +} + +// PacketFragmenter is the book-keeping struct for packet fragmentation. +type PacketFragmenter struct { + transportHeader []byte + data buffer.Buffer + reserve int + fragmentPayloadLen int + fragmentCount int + currentFragment int + fragmentOffset int +} + +// MakePacketFragmenter prepares the struct needed for packet fragmentation. +// +// pkt is the packet to be fragmented. +// +// fragmentPayloadLen is the maximum number of bytes of fragmentable data a fragment can +// have. +// +// reserve is the number of bytes that should be reserved for the headers in +// each generated fragment. +func MakePacketFragmenter(pkt *stack.PacketBuffer, fragmentPayloadLen uint32, reserve int) PacketFragmenter { + // As per RFC 8200 Section 4.5, some IPv6 extension headers should not be + // repeated in each fragment. However we do not currently support any header + // of that kind yet, so the following computation is valid for both IPv4 and + // IPv6. + // TODO(gvisor.dev/issue/3912): Once Authentication or ESP Headers are + // supported for outbound packets, the fragmentable data should not include + // these headers. + var fragmentableData buffer.Buffer + fragmentableData.Append(pkt.TransportHeader().View()) + pktBuf := pkt.Data().ToBuffer() + fragmentableData.Merge(&pktBuf) + fragmentCount := (uint32(fragmentableData.Size()) + fragmentPayloadLen - 1) / fragmentPayloadLen + + return PacketFragmenter{ + data: fragmentableData, + reserve: reserve, + fragmentPayloadLen: int(fragmentPayloadLen), + fragmentCount: int(fragmentCount), + } +} + +// BuildNextFragment returns a packet with the payload of the next fragment, +// along with the fragment's offset, the number of bytes copied and a boolean +// indicating if there are more fragments left or not. If this function is +// called again after it indicated that no more fragments were left, it will +// panic. +// +// Note that the returned packet will not have its network and link headers +// populated, but space for them will be reserved. The transport header will be +// stored in the packet's data. +func (pf *PacketFragmenter) BuildNextFragment() (*stack.PacketBuffer, int, int, bool) { + if pf.currentFragment >= pf.fragmentCount { + panic("BuildNextFragment should not be called again after the last fragment was returned") + } + + fragPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: pf.reserve, + }) + + // Copy data for the fragment. + copied := fragPkt.Data().ReadFrom(&pf.data, pf.fragmentPayloadLen) + + offset := pf.fragmentOffset + pf.fragmentOffset += copied + pf.currentFragment++ + more := pf.currentFragment != pf.fragmentCount + + return fragPkt, offset, copied, more +} + +// RemainingFragmentCount returns the number of fragments left to be built. +func (pf *PacketFragmenter) RemainingFragmentCount() int { + return pf.fragmentCount - pf.currentFragment +} + +// Release frees resources owned by the packet fragmenter. +func (pf *PacketFragmenter) Release() { + pf.data.Release() +} diff --git a/pkg/tcpip/network/internal/fragmentation/fragmentation_state_autogen.go b/pkg/tcpip/network/internal/fragmentation/fragmentation_state_autogen.go new file mode 100644 index 0000000..14a7685 --- /dev/null +++ b/pkg/tcpip/network/internal/fragmentation/fragmentation_state_autogen.go @@ -0,0 +1,246 @@ +// automatically generated by stateify. + +package fragmentation + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (f *FragmentID) StateTypeName() string { + return "pkg/tcpip/network/internal/fragmentation.FragmentID" +} + +func (f *FragmentID) StateFields() []string { + return []string{ + "Source", + "Destination", + "ID", + "Protocol", + } +} + +func (f *FragmentID) beforeSave() {} + +// +checklocksignore +func (f *FragmentID) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.Source) + stateSinkObject.Save(1, &f.Destination) + stateSinkObject.Save(2, &f.ID) + stateSinkObject.Save(3, &f.Protocol) +} + +func (f *FragmentID) afterLoad(context.Context) {} + +// +checklocksignore +func (f *FragmentID) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.Source) + stateSourceObject.Load(1, &f.Destination) + stateSourceObject.Load(2, &f.ID) + stateSourceObject.Load(3, &f.Protocol) +} + +func (f *Fragmentation) StateTypeName() string { + return "pkg/tcpip/network/internal/fragmentation.Fragmentation" +} + +func (f *Fragmentation) StateFields() []string { + return []string{ + "highLimit", + "lowLimit", + "reassemblers", + "rList", + "memSize", + "timeout", + "blockSize", + "clock", + "releaseJob", + "timeoutHandler", + } +} + +func (f *Fragmentation) beforeSave() {} + +// +checklocksignore +func (f *Fragmentation) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.highLimit) + stateSinkObject.Save(1, &f.lowLimit) + stateSinkObject.Save(2, &f.reassemblers) + stateSinkObject.Save(3, &f.rList) + stateSinkObject.Save(4, &f.memSize) + stateSinkObject.Save(5, &f.timeout) + stateSinkObject.Save(6, &f.blockSize) + stateSinkObject.Save(7, &f.clock) + stateSinkObject.Save(8, &f.releaseJob) + stateSinkObject.Save(9, &f.timeoutHandler) +} + +func (f *Fragmentation) afterLoad(context.Context) {} + +// +checklocksignore +func (f *Fragmentation) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.highLimit) + stateSourceObject.Load(1, &f.lowLimit) + stateSourceObject.Load(2, &f.reassemblers) + stateSourceObject.Load(3, &f.rList) + stateSourceObject.Load(4, &f.memSize) + stateSourceObject.Load(5, &f.timeout) + stateSourceObject.Load(6, &f.blockSize) + stateSourceObject.Load(7, &f.clock) + stateSourceObject.Load(8, &f.releaseJob) + stateSourceObject.Load(9, &f.timeoutHandler) +} + +func (h *hole) StateTypeName() string { + return "pkg/tcpip/network/internal/fragmentation.hole" +} + +func (h *hole) StateFields() []string { + return []string{ + "first", + "last", + "filled", + "final", + "pkt", + } +} + +func (h *hole) beforeSave() {} + +// +checklocksignore +func (h *hole) StateSave(stateSinkObject state.Sink) { + h.beforeSave() + stateSinkObject.Save(0, &h.first) + stateSinkObject.Save(1, &h.last) + stateSinkObject.Save(2, &h.filled) + stateSinkObject.Save(3, &h.final) + stateSinkObject.Save(4, &h.pkt) +} + +func (h *hole) afterLoad(context.Context) {} + +// +checklocksignore +func (h *hole) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &h.first) + stateSourceObject.Load(1, &h.last) + stateSourceObject.Load(2, &h.filled) + stateSourceObject.Load(3, &h.final) + stateSourceObject.Load(4, &h.pkt) +} + +func (r *reassembler) StateTypeName() string { + return "pkg/tcpip/network/internal/fragmentation.reassembler" +} + +func (r *reassembler) StateFields() []string { + return []string{ + "reassemblerEntry", + "id", + "memSize", + "proto", + "holes", + "filled", + "done", + "createdAt", + "pkt", + } +} + +func (r *reassembler) beforeSave() {} + +// +checklocksignore +func (r *reassembler) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.reassemblerEntry) + stateSinkObject.Save(1, &r.id) + stateSinkObject.Save(2, &r.memSize) + stateSinkObject.Save(3, &r.proto) + stateSinkObject.Save(4, &r.holes) + stateSinkObject.Save(5, &r.filled) + stateSinkObject.Save(6, &r.done) + stateSinkObject.Save(7, &r.createdAt) + stateSinkObject.Save(8, &r.pkt) +} + +func (r *reassembler) afterLoad(context.Context) {} + +// +checklocksignore +func (r *reassembler) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.reassemblerEntry) + stateSourceObject.Load(1, &r.id) + stateSourceObject.Load(2, &r.memSize) + stateSourceObject.Load(3, &r.proto) + stateSourceObject.Load(4, &r.holes) + stateSourceObject.Load(5, &r.filled) + stateSourceObject.Load(6, &r.done) + stateSourceObject.Load(7, &r.createdAt) + stateSourceObject.Load(8, &r.pkt) +} + +func (l *reassemblerList) StateTypeName() string { + return "pkg/tcpip/network/internal/fragmentation.reassemblerList" +} + +func (l *reassemblerList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *reassemblerList) beforeSave() {} + +// +checklocksignore +func (l *reassemblerList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *reassemblerList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *reassemblerList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *reassemblerEntry) StateTypeName() string { + return "pkg/tcpip/network/internal/fragmentation.reassemblerEntry" +} + +func (e *reassemblerEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *reassemblerEntry) beforeSave() {} + +// +checklocksignore +func (e *reassemblerEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *reassemblerEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *reassemblerEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*FragmentID)(nil)) + state.Register((*Fragmentation)(nil)) + state.Register((*hole)(nil)) + state.Register((*reassembler)(nil)) + state.Register((*reassemblerList)(nil)) + state.Register((*reassemblerEntry)(nil)) +} diff --git a/pkg/tcpip/network/internal/fragmentation/reassembler.go b/pkg/tcpip/network/internal/fragmentation/reassembler.go new file mode 100644 index 0000000..e490487 --- /dev/null +++ b/pkg/tcpip/network/internal/fragmentation/reassembler.go @@ -0,0 +1,185 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fragmentation + +import ( + "math" + "sort" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// +stateify savable +type hole struct { + first uint16 + last uint16 + filled bool + final bool + // pkt is the fragment packet if hole is filled. We keep the whole pkt rather + // than the fragmented payload to prevent binding to specific buffer types. + pkt *stack.PacketBuffer +} + +// +stateify savable +type reassembler struct { + reassemblerEntry + id FragmentID + memSize int + proto uint8 + mu sync.Mutex `state:"nosave"` + holes []hole + filled int + done bool + createdAt tcpip.MonotonicTime + pkt *stack.PacketBuffer +} + +func newReassembler(id FragmentID, clock tcpip.Clock) *reassembler { + r := &reassembler{ + id: id, + createdAt: clock.NowMonotonic(), + } + r.holes = append(r.holes, hole{ + first: 0, + last: math.MaxUint16, + filled: false, + final: true, + }) + return r +} + +func (r *reassembler) process(first, last uint16, more bool, proto uint8, pkt *stack.PacketBuffer) (*stack.PacketBuffer, uint8, bool, int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.done { + // A concurrent goroutine might have already reassembled + // the packet and emptied the heap while this goroutine + // was waiting on the mutex. We don't have to do anything in this case. + return nil, 0, false, 0, nil + } + + var holeFound bool + var memConsumed int + for i := range r.holes { + currentHole := &r.holes[i] + + if last < currentHole.first || currentHole.last < first { + continue + } + // For IPv6, overlaps with an existing fragment are explicitly forbidden by + // RFC 8200 section 4.5: + // If any of the fragments being reassembled overlap with any other + // fragments being reassembled for the same packet, reassembly of that + // packet must be abandoned and all the fragments that have been received + // for that packet must be discarded, and no ICMP error messages should be + // sent. + // + // It is not explicitly forbidden for IPv4, but to keep parity with Linux we + // disallow it as well: + // https://github.com/torvalds/linux/blob/38525c6/net/ipv4/inet_fragment.c#L349 + if first < currentHole.first || currentHole.last < last { + // Incoming fragment only partially fits in the free hole. + return nil, 0, false, 0, ErrFragmentOverlap + } + if !more { + if !currentHole.final || currentHole.filled && currentHole.last != last { + // We have another final fragment, which does not perfectly overlap. + return nil, 0, false, 0, ErrFragmentConflict + } + } + + holeFound = true + if currentHole.filled { + // Incoming fragment is a duplicate. + continue + } + + // We are populating the current hole with the payload and creating a new + // hole for any unfilled ranges on either end. + if first > currentHole.first { + r.holes = append(r.holes, hole{ + first: currentHole.first, + last: first - 1, + filled: false, + final: false, + }) + } + if last < currentHole.last && more { + r.holes = append(r.holes, hole{ + first: last + 1, + last: currentHole.last, + filled: false, + final: currentHole.final, + }) + currentHole.final = false + } + memConsumed = pkt.MemSize() + r.memSize += memConsumed + // Update the current hole to precisely match the incoming fragment. + r.holes[i] = hole{ + first: first, + last: last, + filled: true, + final: currentHole.final, + pkt: pkt.Clone(), + } + r.filled++ + // For IPv6, it is possible to have different Protocol values between + // fragments of a packet (because, unlike IPv4, the Protocol is not used to + // identify a fragment). In this case, only the Protocol of the first + // fragment must be used as per RFC 8200 Section 4.5. + // + // TODO(gvisor.dev/issue/3648): During reassembly of an IPv6 packet, IP + // options received in the first fragment should be used - and they should + // override options from following fragments. + if first == 0 { + if r.pkt != nil { + r.pkt.DecRef() + } + r.pkt = pkt.Clone() + r.proto = proto + } + break + } + if !holeFound { + // Incoming fragment is beyond end. + return nil, 0, false, 0, ErrFragmentConflict + } + + // Check if all the holes have been filled and we are ready to reassemble. + if r.filled < len(r.holes) { + return nil, 0, false, memConsumed, nil + } + + sort.Slice(r.holes, func(i, j int) bool { + return r.holes[i].first < r.holes[j].first + }) + + resPkt := r.holes[0].pkt.Clone() + for i := 1; i < len(r.holes); i++ { + stack.MergeFragment(resPkt, r.holes[i].pkt) + } + return resPkt, r.proto, true /* done */, memConsumed, nil +} + +func (r *reassembler) checkDoneOrMark() bool { + r.mu.Lock() + prev := r.done + r.done = true + r.mu.Unlock() + return prev +} diff --git a/pkg/tcpip/network/internal/fragmentation/reassembler_list.go b/pkg/tcpip/network/internal/fragmentation/reassembler_list.go new file mode 100644 index 0000000..949a0ac --- /dev/null +++ b/pkg/tcpip/network/internal/fragmentation/reassembler_list.go @@ -0,0 +1,239 @@ +package fragmentation + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type reassemblerElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (reassemblerElementMapper) linkerFor(elem *reassembler) *reassembler { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type reassemblerList struct { + head *reassembler + tail *reassembler +} + +// Reset resets list l to the empty state. +func (l *reassemblerList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *reassemblerList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *reassemblerList) Front() *reassembler { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *reassemblerList) Back() *reassembler { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *reassemblerList) Len() (count int) { + for e := l.Front(); e != nil; e = (reassemblerElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *reassemblerList) PushFront(e *reassembler) { + linker := reassemblerElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + reassemblerElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *reassemblerList) PushFrontList(m *reassemblerList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + reassemblerElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + reassemblerElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *reassemblerList) PushBack(e *reassembler) { + linker := reassemblerElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + reassemblerElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *reassemblerList) PushBackList(m *reassemblerList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + reassemblerElementMapper{}.linkerFor(l.tail).SetNext(m.head) + reassemblerElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *reassemblerList) InsertAfter(b, e *reassembler) { + bLinker := reassemblerElementMapper{}.linkerFor(b) + eLinker := reassemblerElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + reassemblerElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *reassemblerList) InsertBefore(a, e *reassembler) { + aLinker := reassemblerElementMapper{}.linkerFor(a) + eLinker := reassemblerElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + reassemblerElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *reassemblerList) Remove(e *reassembler) { + linker := reassemblerElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + reassemblerElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + reassemblerElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type reassemblerEntry struct { + next *reassembler + prev *reassembler +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *reassemblerEntry) Next() *reassembler { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *reassemblerEntry) Prev() *reassembler { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *reassemblerEntry) SetNext(elem *reassembler) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *reassemblerEntry) SetPrev(elem *reassembler) { + e.prev = elem +} diff --git a/pkg/tcpip/network/internal/ip/duplicate_address_detection.go b/pkg/tcpip/network/internal/ip/duplicate_address_detection.go new file mode 100644 index 0000000..6be50f2 --- /dev/null +++ b/pkg/tcpip/network/internal/ip/duplicate_address_detection.go @@ -0,0 +1,304 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ip holds IPv4/IPv6 common utilities. +package ip + +import ( + "bytes" + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +type extendRequest int + +const ( + notRequested extendRequest = iota + requested + extended +) + +// +stateify savable +type dadState struct { + nonce []byte + extendRequest extendRequest + + done *bool + timer tcpip.Timer + + completionHandlers []stack.DADCompletionHandler +} + +// DADProtocol is a protocol whose core state machine can be represented by DAD. +type DADProtocol interface { + // SendDADMessage attempts to send a DAD probe message. + SendDADMessage(tcpip.Address, []byte) tcpip.Error +} + +// DADOptions holds options for DAD. +// +// +stateify savable +type DADOptions struct { + Clock tcpip.Clock + // TODO(b/341946753): Restore when netstack is savable. + SecureRNG io.Reader `state:"nosave"` + NonceSize uint8 + ExtendDADTransmits uint8 + Protocol DADProtocol + NICID tcpip.NICID +} + +// DAD performs duplicate address detection for addresses. +// +// +stateify savable +type DAD struct { + opts DADOptions + configs stack.DADConfigurations + + protocolMU sync.Locker `state:"nosave"` + addresses map[tcpip.Address]dadState +} + +// Init initializes the DAD state. +// +// Must only be called once for the lifetime of d; Init will panic if it is +// called twice. +// +// The lock will only be taken when timers fire. +func (d *DAD) Init(protocolMU sync.Locker, configs stack.DADConfigurations, opts DADOptions) { + if d.addresses != nil { + panic("attempted to initialize DAD state twice") + } + + if opts.NonceSize != 0 && opts.ExtendDADTransmits == 0 { + panic(fmt.Sprintf("given a non-zero value for NonceSize (%d) but zero for ExtendDADTransmits", opts.NonceSize)) + } + + configs.Validate() + + *d = DAD{ + opts: opts, + configs: configs, + protocolMU: protocolMU, + addresses: make(map[tcpip.Address]dadState), + } +} + +// CheckDuplicateAddressLocked performs DAD for an address, calling the +// completion handler once DAD resolves. +// +// If DAD is already performing for the provided address, h will be called when +// the currently running process completes. +// +// Precondition: d.protocolMU must be locked. +func (d *DAD) CheckDuplicateAddressLocked(addr tcpip.Address, h stack.DADCompletionHandler) stack.DADCheckAddressDisposition { + if d.configs.DupAddrDetectTransmits == 0 { + return stack.DADDisabled + } + + ret := stack.DADAlreadyRunning + s, ok := d.addresses[addr] + if !ok { + ret = stack.DADStarting + + remaining := d.configs.DupAddrDetectTransmits + + // Protected by d.protocolMU. + done := false + + s = dadState{ + done: &done, + timer: d.opts.Clock.AfterFunc(0, func() { + dadDone := remaining == 0 + + nonce, earlyReturn := func() ([]byte, bool) { + d.protocolMU.Lock() + defer d.protocolMU.Unlock() + + if done { + return nil, true + } + + s, ok := d.addresses[addr] + if !ok { + panic(fmt.Sprintf("dad: timer fired but missing state for %s on NIC(%d)", addr, d.opts.NICID)) + } + + // As per RFC 7527 section 4 + // + // If any probe is looped back within RetransTimer milliseconds + // after having sent DupAddrDetectTransmits NS(DAD) messages, the + // interface continues with another MAX_MULTICAST_SOLICIT number of + // NS(DAD) messages transmitted RetransTimer milliseconds apart. + if dadDone && s.extendRequest == requested { + dadDone = false + remaining = d.opts.ExtendDADTransmits + s.extendRequest = extended + } + + if !dadDone && d.opts.NonceSize != 0 { + if s.nonce == nil { + s.nonce = make([]byte, d.opts.NonceSize) + } + + if n, err := io.ReadFull(d.opts.SecureRNG, s.nonce); err != nil { + panic(fmt.Sprintf("SecureRNG.Read(...): %s", err)) + } else if n != len(s.nonce) { + panic(fmt.Sprintf("expected to read %d bytes from secure RNG, only read %d bytes", len(s.nonce), n)) + } + } + + d.addresses[addr] = s + return s.nonce, false + }() + if earlyReturn { + return + } + + var err tcpip.Error + if !dadDone { + err = d.opts.Protocol.SendDADMessage(addr, nonce) + } + + d.protocolMU.Lock() + defer d.protocolMU.Unlock() + + if done { + return + } + + s, ok := d.addresses[addr] + if !ok { + panic(fmt.Sprintf("dad: timer fired but missing state for %s on NIC(%d)", addr, d.opts.NICID)) + } + + if !dadDone && err == nil { + remaining-- + s.timer.Reset(d.configs.RetransmitTimer) + return + } + + // At this point we know that either DAD has resolved or we hit an error + // sending the last DAD message. Either way, clear the DAD state. + done = false + s.timer.Stop() + delete(d.addresses, addr) + + var res stack.DADResult = &stack.DADSucceeded{} + if err != nil { + res = &stack.DADError{Err: err} + } + for _, h := range s.completionHandlers { + h(res) + } + }), + } + } + + s.completionHandlers = append(s.completionHandlers, h) + d.addresses[addr] = s + return ret +} + +// ExtendIfNonceEqualLockedDisposition enumerates the possible results from +// ExtendIfNonceEqualLocked. +type ExtendIfNonceEqualLockedDisposition int + +const ( + // Extended indicates that the DAD process was extended. + Extended ExtendIfNonceEqualLockedDisposition = iota + + // AlreadyExtended indicates that the DAD process was already extended. + AlreadyExtended + + // NoDADStateFound indicates that DAD state was not found for the address. + NoDADStateFound + + // NonceDisabled indicates that nonce values are not sent with DAD messages. + NonceDisabled + + // NonceNotEqual indicates that the nonce value passed and the nonce in the + // last send DAD message are not equal. + NonceNotEqual +) + +// ExtendIfNonceEqualLocked extends the DAD process if the provided nonce is the +// same as the nonce sent in the last DAD message. +// +// Precondition: d.protocolMU must be locked. +func (d *DAD) ExtendIfNonceEqualLocked(addr tcpip.Address, nonce []byte) ExtendIfNonceEqualLockedDisposition { + s, ok := d.addresses[addr] + if !ok { + return NoDADStateFound + } + + if d.opts.NonceSize == 0 { + return NonceDisabled + } + + if s.extendRequest != notRequested { + return AlreadyExtended + } + + // As per RFC 7527 section 4 + // + // If any probe is looped back within RetransTimer milliseconds after having + // sent DupAddrDetectTransmits NS(DAD) messages, the interface continues + // with another MAX_MULTICAST_SOLICIT number of NS(DAD) messages transmitted + // RetransTimer milliseconds apart. + // + // If a DAD message has already been sent and the nonce value we observed is + // the same as the nonce value we last sent, then we assume our probe was + // looped back and request an extension to the DAD process. + // + // Note, the first DAD message is sent asynchronously so we need to make sure + // that we sent a DAD message by checking if we have a nonce value set. + if s.nonce != nil && bytes.Equal(s.nonce, nonce) { + s.extendRequest = requested + d.addresses[addr] = s + return Extended + } + + return NonceNotEqual +} + +// StopLocked stops a currently running DAD process. +// +// Precondition: d.protocolMU must be locked. +func (d *DAD) StopLocked(addr tcpip.Address, reason stack.DADResult) { + s, ok := d.addresses[addr] + if !ok { + return + } + + *s.done = true + s.timer.Stop() + delete(d.addresses, addr) + + for _, h := range s.completionHandlers { + h(reason) + } +} + +// SetConfigsLocked sets the DAD configurations. +// +// Precondition: d.protocolMU must be locked. +func (d *DAD) SetConfigsLocked(c stack.DADConfigurations) { + c.Validate() + d.configs = c +} diff --git a/pkg/tcpip/network/internal/ip/errors.go b/pkg/tcpip/network/internal/ip/errors.go new file mode 100644 index 0000000..e09614a --- /dev/null +++ b/pkg/tcpip/network/internal/ip/errors.go @@ -0,0 +1,129 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ip + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// ForwardingError represents an error that occurred while trying to forward +// a packet. +type ForwardingError interface { + isForwardingError() + fmt.Stringer +} + +// ErrTTLExceeded indicates that the received packet's TTL has been exceeded. +type ErrTTLExceeded struct{} + +func (*ErrTTLExceeded) isForwardingError() {} + +func (*ErrTTLExceeded) String() string { return "ttl exceeded" } + +// ErrOutgoingDeviceNoBufferSpace indicates that the outgoing device does not +// have enough space to hold a buffer. +type ErrOutgoingDeviceNoBufferSpace struct{} + +func (*ErrOutgoingDeviceNoBufferSpace) isForwardingError() {} + +func (*ErrOutgoingDeviceNoBufferSpace) String() string { return "no device buffer space" } + +// ErrParameterProblem indicates the received packet had a problem with an IP +// parameter. +type ErrParameterProblem struct{} + +func (*ErrParameterProblem) isForwardingError() {} + +func (*ErrParameterProblem) String() string { return "parameter problem" } + +// ErrInitializingSourceAddress indicates the received packet had a source +// address that may only be used on the local network as part of initialization +// work. +type ErrInitializingSourceAddress struct{} + +func (*ErrInitializingSourceAddress) isForwardingError() {} + +func (*ErrInitializingSourceAddress) String() string { return "initializing source address" } + +// ErrLinkLocalSourceAddress indicates the received packet had a link-local +// source address. +type ErrLinkLocalSourceAddress struct{} + +func (*ErrLinkLocalSourceAddress) isForwardingError() {} + +func (*ErrLinkLocalSourceAddress) String() string { return "link local source address" } + +// ErrLinkLocalDestinationAddress indicates the received packet had a link-local +// destination address. +type ErrLinkLocalDestinationAddress struct{} + +func (*ErrLinkLocalDestinationAddress) isForwardingError() {} + +func (*ErrLinkLocalDestinationAddress) String() string { return "link local destination address" } + +// ErrHostUnreachable indicates that the destination host could not be reached. +type ErrHostUnreachable struct{} + +func (*ErrHostUnreachable) isForwardingError() {} + +func (*ErrHostUnreachable) String() string { return "no route to host" } + +// ErrMessageTooLong indicates the packet was too big for the outgoing MTU. +// +// +stateify savable +type ErrMessageTooLong struct{} + +func (*ErrMessageTooLong) isForwardingError() {} + +func (*ErrMessageTooLong) String() string { return "message too long" } + +// ErrNoMulticastPendingQueueBufferSpace indicates that a multicast packet +// could not be added to the pending packet queue due to insufficient buffer +// space. +// +// +stateify savable +type ErrNoMulticastPendingQueueBufferSpace struct{} + +func (*ErrNoMulticastPendingQueueBufferSpace) isForwardingError() {} + +func (*ErrNoMulticastPendingQueueBufferSpace) String() string { return "no buffer space" } + +// ErrUnexpectedMulticastInputInterface indicates that the interface that the +// packet arrived on did not match the routes expected input interface. +type ErrUnexpectedMulticastInputInterface struct{} + +func (*ErrUnexpectedMulticastInputInterface) isForwardingError() {} + +func (*ErrUnexpectedMulticastInputInterface) String() string { return "unexpected input interface" } + +// ErrUnknownOutputEndpoint indicates that the output endpoint associated with +// a route could not be found. +type ErrUnknownOutputEndpoint struct{} + +func (*ErrUnknownOutputEndpoint) isForwardingError() {} + +func (*ErrUnknownOutputEndpoint) String() string { return "unknown endpoint" } + +// ErrOther indicates the packet coould not be forwarded for a reason +// captured by the contained error. +type ErrOther struct { + Err tcpip.Error +} + +func (*ErrOther) isForwardingError() {} + +func (e *ErrOther) String() string { return fmt.Sprintf("other tcpip error: %s", e.Err) } diff --git a/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go b/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go new file mode 100644 index 0000000..1eaeee7 --- /dev/null +++ b/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go @@ -0,0 +1,1192 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ip + +import ( + "fmt" + "math/rand" + "time" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +const ( + // As per RFC 2236 section 3, + // + // When a host joins a multicast group, it should immediately transmit + // an unsolicited Version 2 Membership Report for that group, in case it + // is the first member of that group on the network. To cover the + // possibility of the initial Membership Report being lost or damaged, + // it is recommended that it be repeated once or twice after short + // delays [Unsolicited Report Interval]. (A simple way to accomplish + // this is to send the initial Version 2 Membership Report and then act + // as if a Group-Specific Query was received for that group, and set a + // timer appropriately). + // + // As per RFC 2710 section 4, + // + // When a node starts listening to a multicast address on an interface, + // it should immediately transmit an unsolicited Report for that address + // on that interface, in case it is the first listener on the link. To + // cover the possibility of the initial Report being lost or damaged, it + // is recommended that it be repeated once or twice after short delays + // [Unsolicited Report Interval]. (A simple way to accomplish this is + // to send the initial Report and then act as if a Multicast-Address- + // Specific Query was received for that address, and set a timer + // appropriately). + unsolicitedTransmissionCount = 2 + + // Responses to queries may be delayed, but we only send a response to a + // query once. A response to a query can be handled by any pending + // unsolicited transmission count, but we should send at least one report + // after sending a query. + // + // As per RFC 2236 section 3, + // + // When a host receives a General Query, it sets delay timers for each + // group (excluding the all-systems group) of which it is a member on + // the interface from which it received the query. + // + // As per RFC 2710 section 4, + // + // When a node receives a General Query, it sets a delay timer for each + // multicast address to which it is listening on the interface from + // which it received the Query, EXCLUDING the link-scope all-nodes + // address and any multicast addresses of scope 0 (reserved) or 1 + // (node-local). + minQueryResponseTransmissionCount = 1 + + // DefaultRobustnessVariable is the default robustness variable + // + // As per RFC 3810 section 9.1 (for MLDv2), + // + // The Robustness Variable allows tuning for the expected packet loss on + // a link. If a link is expected to be lossy, the value of the + // Robustness Variable may be increased. MLD is robust to [Robustness + // Variable] - 1 packet losses. The value of the Robustness Variable + // MUST NOT be zero, and SHOULD NOT be one. Default value: 2. + // + // As per RFC 3376 section 8.1 (for IGMPv3), + // + // The Robustness Variable allows tuning for the expected packet loss on + // a network. If a network is expected to be lossy, the Robustness + // Variable may be increased. IGMP is robust to (Robustness Variable - + // 1) packet losses. The Robustness Variable MUST NOT be zero, and + // SHOULD NOT be one. Default: 2 + DefaultRobustnessVariable = 2 + + // DefaultQueryInterval is the default query interval. + // + // As per RFC 3810 section 9.2 (for MLDv2), + // + // The Query Interval variable denotes the interval between General + // Queries sent by the Querier. Default value: 125 seconds. + // + // As per RFC 3376 section 8.2 (for IGMPv3), + // + // The Query Interval is the interval between General Queries sent by + // the Querier. Default: 125 seconds. + DefaultQueryInterval = 125 * time.Second +) + +// multicastGroupState holds the Generic Multicast Protocol state for a +// multicast group. +// +// +stateify savable +type multicastGroupState struct { + // joins is the number of times the group has been joined. + joins uint64 + + // transmissionLeft is the number of transmissions left to send. + transmissionLeft uint8 + + // lastToSendReport is true if we sent the last report for the group. It is + // used to track whether there are other hosts on the subnet that are also + // members of the group. + // + // Defined in RFC 2236 section 6 page 9 for IGMPv2 and RFC 2710 section 5 page + // 8 for MLDv1. + lastToSendReport bool + + // delayedReportJob is used to delay sending responses to membership report + // messages in order to reduce duplicate reports from multiple hosts on the + // interface. + // + // Must not be nil. + delayedReportJob *tcpip.Job + + // delyedReportJobFiresAt is the time when the delayed report job will fire. + // + // A zero value indicates that the job is not scheduled. + // TODO(b/341946753): Restore when netstack is savable. + delayedReportJobFiresAt time.Time `state:"nosave"` + + // queriedIncludeSources holds sources that were queried for. + // + // Indicates that there is a pending source-specific query response for the + // multicast address. + queriedIncludeSources map[tcpip.Address]struct{} + + deleteScheduled bool +} + +func (m *multicastGroupState) cancelDelayedReportJob() { + m.delayedReportJob.Cancel() + m.delayedReportJobFiresAt = time.Time{} + m.transmissionLeft = 0 +} + +func (m *multicastGroupState) clearQueriedIncludeSources() { + for source := range m.queriedIncludeSources { + delete(m.queriedIncludeSources, source) + } +} + +// GenericMulticastProtocolOptions holds options for the generic multicast +// protocol. +// +// +stateify savable +type GenericMulticastProtocolOptions struct { + // Rand is the source of random numbers. + // TODO(b/341946753): Restore when netstack is savable. + Rand *rand.Rand `state:"nosave"` + + // Clock is the clock used to create timers. + Clock tcpip.Clock + + // Protocol is the implementation of the variant of multicast group protocol + // in use. + Protocol MulticastGroupProtocol + + // MaxUnsolicitedReportDelay is the maximum amount of time to wait between + // transmitting unsolicited reports. + // + // Unsolicited reports are transmitted when a group is newly joined. + MaxUnsolicitedReportDelay time.Duration +} + +// MulticastGroupProtocolV2ReportRecordType is the type of a +// MulticastGroupProtocolv2 multicast address record. +type MulticastGroupProtocolV2ReportRecordType int + +// MulticastGroupProtocolv2 multicast address record types. +const ( + _ MulticastGroupProtocolV2ReportRecordType = iota + MulticastGroupProtocolV2ReportRecordModeIsInclude + MulticastGroupProtocolV2ReportRecordModeIsExclude + MulticastGroupProtocolV2ReportRecordChangeToIncludeMode + MulticastGroupProtocolV2ReportRecordChangeToExcludeMode + MulticastGroupProtocolV2ReportRecordAllowNewSources + MulticastGroupProtocolV2ReportRecordBlockOldSources +) + +// MulticastGroupProtocolV2ReportBuilder is a builder for a V2 report. +type MulticastGroupProtocolV2ReportBuilder interface { + // AddRecord adds a record to the report. + AddRecord(recordType MulticastGroupProtocolV2ReportRecordType, groupAddress tcpip.Address) + + // Send sends the report. + // + // Does nothing if no records were added. + // + // It is invalid to use this builder after this method is called. + Send() (sent bool, err tcpip.Error) +} + +// MulticastGroupProtocol is a multicast group protocol whose core state machine +// can be represented by GenericMulticastProtocolState. +type MulticastGroupProtocol interface { + // Enabled indicates whether the generic multicast protocol will be + // performed. + // + // When enabled, the protocol may transmit report and leave messages when + // joining and leaving multicast groups respectively, and handle incoming + // packets. + // + // When disabled, the protocol will still keep track of locally joined groups, + // it just won't transmit and handle packets, or update groups' state. + Enabled() bool + + // SendReport sends a multicast report for the specified group address. + // + // Returns false if the caller should queue the report to be sent later. Note, + // returning false does not mean that the receiver hit an error. + SendReport(groupAddress tcpip.Address) (sent bool, err tcpip.Error) + + // SendLeave sends a multicast leave for the specified group address. + SendLeave(groupAddress tcpip.Address) tcpip.Error + + // ShouldPerformProtocol returns true iff the protocol should be performed for + // the specified group. + ShouldPerformProtocol(tcpip.Address) bool + + // NewReportV2Builder creates a new V2 builder. + NewReportV2Builder() MulticastGroupProtocolV2ReportBuilder + + // V2QueryMaxRespCodeToV2Delay takes a V2 query's maximum response code and + // returns the V2 delay. + V2QueryMaxRespCodeToV2Delay(code uint16) time.Duration + + // V2QueryMaxRespCodeToV1Delay takes a V2 query's maximum response code and + // returns the V1 delay. + V2QueryMaxRespCodeToV1Delay(code uint16) time.Duration +} + +type protocolMode int + +const ( + protocolModeV2 protocolMode = iota + protocolModeV1 + protocolModeV1Compatibility +) + +// GenericMulticastProtocolState is the per interface generic multicast protocol +// state. +// +// There is actually no protocol named "Generic Multicast Protocol". Instead, +// the term used to refer to a generic multicast protocol that applies to both +// IPv4 and IPv6. Specifically, Generic Multicast Protocol is the core state +// machine of IGMPv2 as defined by RFC 2236 and MLDv1 as defined by RFC 2710. +// +// Callers must synchronize accesses to the generic multicast protocol state; +// GenericMulticastProtocolState obtains no locks in any of its methods. The +// only exception to this is GenericMulticastProtocolState's timer/job callbacks +// which will obtain the lock provided to the GenericMulticastProtocolState when +// it is initialized. +// +// GenericMulticastProtocolState.Init MUST be called before calling any of +// the methods on GenericMulticastProtocolState. +// +// GenericMulticastProtocolState.MakeAllNonMemberLocked MUST be called when the +// multicast group protocol is disabled so that leave messages may be sent. +// +// +stateify savable +type GenericMulticastProtocolState struct { + // Do not allow overwriting this state. + _ sync.NoCopy `state:"nosave"` + + opts GenericMulticastProtocolOptions + + // memberships holds group addresses and their associated state. + memberships map[tcpip.Address]multicastGroupState + + // protocolMU is the mutex used to protect the protocol. + protocolMU *sync.RWMutex `state:"nosave"` + + // V2 state. + robustnessVariable uint8 + queryInterval time.Duration + mode protocolMode + modeTimer tcpip.Timer + + generalQueryV2Timer tcpip.Timer + // TODO(b/341946753): Restore when netstack is savable. + generalQueryV2TimerFiresAt time.Time `state:"nosave"` + + stateChangedReportV2Timer tcpip.Timer + stateChangedReportV2TimerSet bool +} + +// GetV1ModeLocked returns the V1 configuration. +// +// Precondition: g.protocolMU must be read locked. +func (g *GenericMulticastProtocolState) GetV1ModeLocked() bool { + switch g.mode { + case protocolModeV2, protocolModeV1Compatibility: + return false + case protocolModeV1: + return true + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } +} + +func (g *GenericMulticastProtocolState) stopModeTimer() { + if g.modeTimer != nil { + g.modeTimer.Stop() + } +} + +// SetV1ModeLocked sets the V1 configuration. +// +// Returns the previous configuration. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) SetV1ModeLocked(v bool) bool { + if g.GetV1ModeLocked() == v { + return v + } + + if v { + g.stopModeTimer() + g.cancelV2ReportTimers() + g.mode = protocolModeV1 + return false + } + + g.mode = protocolModeV2 + return true +} + +func (g *GenericMulticastProtocolState) cancelV2ReportTimers() { + if g.generalQueryV2Timer != nil { + g.generalQueryV2Timer.Stop() + g.generalQueryV2TimerFiresAt = time.Time{} + } + + if g.stateChangedReportV2Timer != nil { + g.stateChangedReportV2Timer.Stop() + g.stateChangedReportV2TimerSet = false + } +} + +// Init initializes the Generic Multicast Protocol state. +// +// Must only be called once for the lifetime of g; Init will panic if it is +// called twice. +// +// The GenericMulticastProtocolState will only grab the lock when timers/jobs +// fire. +// +// Note: the methods on opts.Protocol will always be called while protocolMU is +// held. +func (g *GenericMulticastProtocolState) Init(protocolMU *sync.RWMutex, opts GenericMulticastProtocolOptions) { + if g.memberships != nil { + panic("attempted to initialize generic membership protocol state twice") + } + + *g = GenericMulticastProtocolState{ + opts: opts, + memberships: make(map[tcpip.Address]multicastGroupState), + protocolMU: protocolMU, + robustnessVariable: DefaultRobustnessVariable, + queryInterval: DefaultQueryInterval, + mode: protocolModeV2, + } +} + +// MakeAllNonMemberLocked transitions all groups to the non-member state. +// +// The groups will still be considered joined locally. +// +// MUST be called when the multicast group protocol is disabled. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) MakeAllNonMemberLocked() { + if !g.opts.Protocol.Enabled() { + return + } + + g.stopModeTimer() + g.cancelV2ReportTimers() + + var v2ReportBuilder MulticastGroupProtocolV2ReportBuilder + var handler func(tcpip.Address, *multicastGroupState) + switch g.mode { + case protocolModeV2: + v2ReportBuilder = g.opts.Protocol.NewReportV2Builder() + handler = func(groupAddress tcpip.Address, info *multicastGroupState) { + info.cancelDelayedReportJob() + + // Send a report immediately to announce us leaving the group. + v2ReportBuilder.AddRecord( + MulticastGroupProtocolV2ReportRecordChangeToIncludeMode, + groupAddress, + ) + } + case protocolModeV1Compatibility: + g.mode = protocolModeV2 + fallthrough + case protocolModeV1: + handler = g.transitionToNonMemberLocked + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + + for groupAddress, info := range g.memberships { + if !g.shouldPerformForGroup(groupAddress) { + continue + } + + handler(groupAddress, &info) + + if info.deleteScheduled { + delete(g.memberships, groupAddress) + } else { + info.transmissionLeft = 0 + g.memberships[groupAddress] = info + } + } + + if v2ReportBuilder != nil { + // Nothing meaningful we can do with the error here - this method may be + // called when an interface is being disabled when we expect sends to + // fail. + _, _ = v2ReportBuilder.Send() + } +} + +// InitializeGroupsLocked initializes each group, as if they were newly joined +// but without affecting the groups' join count. +// +// Must only be called after calling MakeAllNonMember as a group should not be +// initialized while it is not in the non-member state. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) InitializeGroupsLocked() { + if !g.opts.Protocol.Enabled() { + return + } + + var v2ReportBuilder MulticastGroupProtocolV2ReportBuilder + switch g.mode { + case protocolModeV2: + v2ReportBuilder = g.opts.Protocol.NewReportV2Builder() + case protocolModeV1Compatibility, protocolModeV1: + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + + for groupAddress, info := range g.memberships { + g.initializeNewMemberLocked(groupAddress, &info, v2ReportBuilder) + g.memberships[groupAddress] = info + } + + if v2ReportBuilder == nil { + return + } + + if sent, err := v2ReportBuilder.Send(); sent && err == nil { + g.scheduleStateChangedTimer() + } else { + // Nothing meaningful we could do with the error here - the interface may + // not yet have an address. This is okay because we would either schedule a + // report to be sent later or we will be notified when an address is added, + // at which point we will try to send messages again. + for groupAddress, info := range g.memberships { + if !g.shouldPerformForGroup(groupAddress) { + continue + } + + // Revert the transmissions count since we did not successfully send. + info.transmissionLeft++ + g.memberships[groupAddress] = info + } + } +} + +// SendQueuedReportsLocked attempts to send reports for groups that failed to +// send reports during their last attempt. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) SendQueuedReportsLocked() { + if g.stateChangedReportV2TimerSet { + return + } + + for groupAddress, info := range g.memberships { + if info.delayedReportJobFiresAt.IsZero() { + switch g.mode { + case protocolModeV2: + g.sendV2ReportAndMaybeScheduleChangedTimer(groupAddress, &info, MulticastGroupProtocolV2ReportRecordChangeToExcludeMode) + case protocolModeV1Compatibility, protocolModeV1: + g.maybeSendReportLocked(groupAddress, &info) + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + + g.memberships[groupAddress] = info + } + } +} + +// JoinGroupLocked handles joining a new group. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) JoinGroupLocked(groupAddress tcpip.Address) { + info, ok := g.memberships[groupAddress] + if ok { + info.joins++ + if info.joins > 1 { + // The group has already been joined. + g.memberships[groupAddress] = info + return + } + } else { + info = multicastGroupState{ + // Since we just joined the group, its count is 1. + joins: 1, + lastToSendReport: false, + delayedReportJob: tcpip.NewJob(g.opts.Clock, g.protocolMU, func() { + if !g.opts.Protocol.Enabled() { + panic(fmt.Sprintf("delayed report job fired for group %s while the multicast group protocol is disabled", groupAddress)) + } + + info, ok := g.memberships[groupAddress] + if !ok { + panic(fmt.Sprintf("expected to find group state for group = %s", groupAddress)) + } + + info.delayedReportJobFiresAt = time.Time{} + + switch g.mode { + case protocolModeV2: + reportBuilder := g.opts.Protocol.NewReportV2Builder() + reportBuilder.AddRecord(MulticastGroupProtocolV2ReportRecordModeIsExclude, groupAddress) + // Nothing meaningful we can do with the error here - we only try to + // send a delayed report once. + _, _ = reportBuilder.Send() + case protocolModeV1Compatibility, protocolModeV1: + g.maybeSendReportLocked(groupAddress, &info) + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + + info.clearQueriedIncludeSources() + g.memberships[groupAddress] = info + }), + queriedIncludeSources: make(map[tcpip.Address]struct{}), + } + } + + info.deleteScheduled = false + info.clearQueriedIncludeSources() + info.delayedReportJobFiresAt = time.Time{} + info.lastToSendReport = false + g.initializeNewMemberLocked(groupAddress, &info, nil /* callersV2ReportBuilder */) + g.memberships[groupAddress] = info +} + +// IsLocallyJoinedRLocked returns true if the group is locally joined. +// +// Precondition: g.protocolMU must be read locked. +func (g *GenericMulticastProtocolState) IsLocallyJoinedRLocked(groupAddress tcpip.Address) bool { + info, ok := g.memberships[groupAddress] + return ok && !info.deleteScheduled +} + +func (g *GenericMulticastProtocolState) sendV2ReportAndMaybeScheduleChangedTimer( + groupAddress tcpip.Address, + info *multicastGroupState, + recordType MulticastGroupProtocolV2ReportRecordType, +) bool { + if info.transmissionLeft == 0 { + return false + } + + successfullySentAndHasMore := false + + // Send a report immediately to announce us leaving the group. + reportBuilder := g.opts.Protocol.NewReportV2Builder() + reportBuilder.AddRecord(recordType, groupAddress) + if sent, err := reportBuilder.Send(); sent && err == nil { + info.transmissionLeft-- + + successfullySentAndHasMore = info.transmissionLeft != 0 + + // Use the interface-wide state changed report for further transmissions. + if successfullySentAndHasMore { + g.scheduleStateChangedTimer() + } + } + + return successfullySentAndHasMore +} + +func (g *GenericMulticastProtocolState) scheduleStateChangedTimer() { + if g.stateChangedReportV2TimerSet { + return + } + + delay := g.calculateDelayTimerDuration(g.opts.MaxUnsolicitedReportDelay) + if g.stateChangedReportV2Timer == nil { + // TODO(https://issuetracker.google.com/264799098): Create timer on + // initialization instead of lazily creating the timer since the timer + // does not change after being created. + g.stateChangedReportV2Timer = g.opts.Clock.AfterFunc(delay, func() { + g.protocolMU.Lock() + defer g.protocolMU.Unlock() + + reportBuilder := g.opts.Protocol.NewReportV2Builder() + nonEmptyReport := false + for groupAddress, info := range g.memberships { + if info.transmissionLeft == 0 || !g.shouldPerformForGroup(groupAddress) { + continue + } + + info.transmissionLeft-- + nonEmptyReport = true + + mode := MulticastGroupProtocolV2ReportRecordChangeToExcludeMode + if info.deleteScheduled { + mode = MulticastGroupProtocolV2ReportRecordChangeToIncludeMode + } + reportBuilder.AddRecord(mode, groupAddress) + + if info.deleteScheduled && info.transmissionLeft == 0 { + // No more transmissions left so we can actually delete the + // membership. + delete(g.memberships, groupAddress) + } else { + g.memberships[groupAddress] = info + } + } + + // Nothing meaningful we can do with the error here. We will retry + // sending a state changed report again anyways. + _, _ = reportBuilder.Send() + + if nonEmptyReport { + g.stateChangedReportV2Timer.Reset(g.calculateDelayTimerDuration(g.opts.MaxUnsolicitedReportDelay)) + } else { + g.stateChangedReportV2TimerSet = false + } + }) + } else { + g.stateChangedReportV2Timer.Reset(delay) + } + g.stateChangedReportV2TimerSet = true +} + +// LeaveGroupLocked handles leaving the group. +// +// Returns false if the group is not currently joined. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) LeaveGroupLocked(groupAddress tcpip.Address) bool { + info, ok := g.memberships[groupAddress] + if !ok || info.joins == 0 { + return false + } + + info.joins-- + if info.joins != 0 { + // If we still have outstanding joins, then do nothing further. + g.memberships[groupAddress] = info + return true + } + + info.deleteScheduled = true + info.cancelDelayedReportJob() + + if !g.shouldPerformForGroup(groupAddress) { + delete(g.memberships, groupAddress) + return true + } + + switch g.mode { + case protocolModeV2: + info.transmissionLeft = g.robustnessVariable + if g.sendV2ReportAndMaybeScheduleChangedTimer(groupAddress, &info, MulticastGroupProtocolV2ReportRecordChangeToIncludeMode) { + g.memberships[groupAddress] = info + } else { + delete(g.memberships, groupAddress) + } + case protocolModeV1Compatibility, protocolModeV1: + g.transitionToNonMemberLocked(groupAddress, &info) + delete(g.memberships, groupAddress) + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + + return true +} + +// HandleQueryV2Locked handles a V2 query. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) HandleQueryV2Locked(groupAddress tcpip.Address, maxResponseCode uint16, sources header.AddressIterator, robustnessVariable uint8, queryInterval time.Duration) { + if !g.opts.Protocol.Enabled() { + return + } + + switch g.mode { + case protocolModeV1Compatibility, protocolModeV1: + g.handleQueryInnerLocked(groupAddress, g.opts.Protocol.V2QueryMaxRespCodeToV1Delay(maxResponseCode)) + return + case protocolModeV2: + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + + if robustnessVariable != 0 { + g.robustnessVariable = robustnessVariable + } + + if queryInterval != 0 { + g.queryInterval = queryInterval + } + + maxResponseTime := g.calculateDelayTimerDuration(g.opts.Protocol.V2QueryMaxRespCodeToV2Delay(maxResponseCode)) + + // As per RFC 3376 section 5.2, + // + // 1. If there is a pending response to a previous General Query + // scheduled sooner than the selected delay, no additional response + // needs to be scheduled. + // + // 2. If the received Query is a General Query, the interface timer is + // used to schedule a response to the General Query after the + // selected delay. Any previously pending response to a General + // Query is canceled. + // + // 3. If the received Query is a Group-Specific Query or a Group-and- + // Source-Specific Query and there is no pending response to a + // previous Query for this group, then the group timer is used to + // schedule a report. If the received Query is a Group-and-Source- + // Specific Query, the list of queried sources is recorded to be used + // when generating a response. + // + // 4. If there already is a pending response to a previous Query + // scheduled for this group, and either the new Query is a Group- + // Specific Query or the recorded source-list associated with the + // group is empty, then the group source-list is cleared and a single + // response is scheduled using the group timer. The new response is + // scheduled to be sent at the earliest of the remaining time for the + // pending report and the selected delay. + // + // 5. If the received Query is a Group-and-Source-Specific Query and + // there is a pending response for this group with a non-empty + // source-list, then the group source list is augmented to contain + // the list of sources in the new Query and a single response is + // scheduled using the group timer. The new response is scheduled to + // be sent at the earliest of the remaining time for the pending + // report and the selected delay. + // + // As per RFC 3810 section 6.2, + // + // 1. If there is a pending response to a previous General Query + // scheduled sooner than the selected delay, no additional response + // needs to be scheduled. + // + // 2. If the received Query is a General Query, the Interface Timer is + // used to schedule a response to the General Query after the + // selected delay. Any previously pending response to a General + // Query is canceled. + // + // 3. If the received Query is a Multicast Address Specific Query or a + // Multicast Address and Source Specific Query and there is no + // pending response to a previous Query for this multicast address, + // then the Multicast Address Timer is used to schedule a report. If + // the received Query is a Multicast Address and Source Specific + // Query, the list of queried sources is recorded to be used when + // generating a response. + // + // 4. If there is already a pending response to a previous Query + // scheduled for this multicast address, and either the new Query is + // a Multicast Address Specific Query or the recorded source list + // associated with the multicast address is empty, then the multicast + // address source list is cleared and a single response is scheduled, + // using the Multicast Address Timer. The new response is scheduled + // to be sent at the earliest of the remaining time for the pending + // report and the selected delay. + // + // 5. If the received Query is a Multicast Address and Source Specific + // Query and there is a pending response for this multicast address + // with a non-empty source list, then the multicast address source + // list is augmented to contain the list of sources in the new Query, + // and a single response is scheduled using the Multicast Address + // Timer. The new response is scheduled to be sent at the earliest + // of the remaining time for the pending report and the selected + // delay. + now := g.opts.Clock.Now() + if !g.generalQueryV2TimerFiresAt.IsZero() && g.generalQueryV2TimerFiresAt.Sub(now) <= maxResponseTime { + return + } + + if groupAddress.Unspecified() { + if g.generalQueryV2Timer == nil { + // TODO(https://issuetracker.google.com/264799098): Create timer on + // initialization instead of lazily creating the timer since the timer + // does not change after being created. + g.generalQueryV2Timer = g.opts.Clock.AfterFunc(maxResponseTime, func() { + g.protocolMU.Lock() + defer g.protocolMU.Unlock() + + g.generalQueryV2TimerFiresAt = time.Time{} + + // As per RFC 3810 section 6.3, + // + // If the expired timer is the Interface Timer (i.e., there is a + // pending response to a General Query), then one Current State + // Record is sent for each multicast address for which the specified + // interface has listening state, as described in section 4.2. The + // Current State Record carries the multicast address and its + // associated filter mode (MODE_IS_INCLUDE or MODE_IS_EXCLUDE) and + // Source list. Multiple Current State Records are packed into + // individual Report messages, to the extent possible. + // + // As per RFC 3376 section 5.2, + // + // If the expired timer is the interface timer (i.e., it is a pending + // response to a General Query), then one Current-State Record is + // sent for each multicast address for which the specified interface + // has reception state, as described in section 3.2. The Current- + // State Record carries the multicast address and its associated + // filter mode (MODE_IS_INCLUDE or MODE_IS_EXCLUDE) and source list. + // Multiple Current-State Records are packed into individual Report + // messages, to the extent possible. + reportBuilder := g.opts.Protocol.NewReportV2Builder() + for groupAddress, info := range g.memberships { + if info.deleteScheduled || !g.shouldPerformForGroup(groupAddress) { + continue + } + + // A MODE_IS_EXCLUDE record without any sources indicates that we are + // interested in traffic from all sources for the group. + // + // We currently only hold groups if we have an active interest in the + // group. + reportBuilder.AddRecord( + MulticastGroupProtocolV2ReportRecordModeIsExclude, + groupAddress, + ) + } + + _, _ = reportBuilder.Send() + }) + } else { + g.generalQueryV2Timer.Reset(maxResponseTime) + } + g.generalQueryV2TimerFiresAt = now.Add(maxResponseTime) + return + } + + if info, ok := g.memberships[groupAddress]; ok && !info.deleteScheduled && g.shouldPerformForGroup(groupAddress) { + if info.delayedReportJobFiresAt.IsZero() || (!sources.Done() && len(info.queriedIncludeSources) != 0) { + for { + source, ok := sources.Next() + if !ok { + break + } + + info.queriedIncludeSources[source] = struct{}{} + } + } else { + info.clearQueriedIncludeSources() + } + g.setDelayTimerForAddressLocked(groupAddress, &info, maxResponseTime) + g.memberships[groupAddress] = info + } +} + +// HandleQueryLocked handles a query message with the specified maximum response +// time. +// +// If the group address is unspecified, then reports will be scheduled for all +// joined groups. +// +// Report(s) will be scheduled to be sent after a random duration between 0 and +// the maximum response time. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) HandleQueryLocked(groupAddress tcpip.Address, maxResponseTime time.Duration) { + if !g.opts.Protocol.Enabled() { + return + } + + switch g.mode { + case protocolModeV2, protocolModeV1Compatibility: + // As per 3376 section 8.12 (for IGMPv3), + // + // The Older Version Querier Interval is the time-out for transitioning + // a host back to IGMPv3 mode once an older version query is heard. + // When an older version query is received, hosts set their Older + // Version Querier Present Timer to Older Version Querier Interval. + // + // This value MUST be ((the Robustness Variable) times (the Query + // Interval in the last Query received)) plus (one Query Response + // Interval). + // + // As per RFC 3810 section 9.12 (for MLDv2), + // + // The Older Version Querier Present Timeout is the time-out for + // transitioning a host back to MLDv2 Host Compatibility Mode. When an + // MLDv1 query is received, MLDv2 hosts set their Older Version Querier + // Present Timer to [Older Version Querier Present Timeout]. + // + // This value MUST be ([Robustness Variable] times (the [Query Interval] + // in the last Query received)) plus ([Query Response Interval]). + modeRevertDelay := time.Duration(g.robustnessVariable) * g.queryInterval + if g.modeTimer == nil { + // TODO(https://issuetracker.google.com/264799098): Create timer on + // initialization instead of lazily creating the timer since the timer + // does not change after being created. + g.modeTimer = g.opts.Clock.AfterFunc(modeRevertDelay, func() { + g.protocolMU.Lock() + defer g.protocolMU.Unlock() + g.mode = protocolModeV2 + }) + } else { + g.modeTimer.Reset(modeRevertDelay) + } + g.mode = protocolModeV1Compatibility + g.cancelV2ReportTimers() + case protocolModeV1: + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } + g.handleQueryInnerLocked(groupAddress, maxResponseTime) +} + +func (g *GenericMulticastProtocolState) handleQueryInnerLocked(groupAddress tcpip.Address, maxResponseTime time.Duration) { + maxResponseTime = g.calculateDelayTimerDuration(maxResponseTime) + + // As per RFC 2236 section 2.4 (for IGMPv2), + // + // In a Membership Query message, the group address field is set to zero + // when sending a General Query, and set to the group address being + // queried when sending a Group-Specific Query. + // + // As per RFC 2710 section 3.6 (for MLDv1), + // + // In a Query message, the Multicast Address field is set to zero when + // sending a General Query, and set to a specific IPv6 multicast address + // when sending a Multicast-Address-Specific Query. + if groupAddress.Unspecified() { + // This is a general query as the group address is unspecified. + for groupAddress, info := range g.memberships { + g.setDelayTimerForAddressLocked(groupAddress, &info, maxResponseTime) + g.memberships[groupAddress] = info + } + } else if info, ok := g.memberships[groupAddress]; ok && !info.deleteScheduled { + g.setDelayTimerForAddressLocked(groupAddress, &info, maxResponseTime) + g.memberships[groupAddress] = info + } +} + +// HandleReportLocked handles a report message. +// +// If the report is for a joined group, any active delayed report will be +// cancelled and the host state for the group transitions to idle. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) HandleReportLocked(groupAddress tcpip.Address) { + if !g.opts.Protocol.Enabled() { + return + } + + // As per RFC 2236 section 3 pages 3-4 (for IGMPv2), + // + // If the host receives another host's Report (version 1 or 2) while it has + // a timer running, it stops its timer for the specified group and does not + // send a Report + // + // As per RFC 2710 section 4 page 6 (for MLDv1), + // + // If a node receives another node's Report from an interface for a + // multicast address while it has a timer running for that same address + // on that interface, it stops its timer and does not send a Report for + // that address, thus suppressing duplicate reports on the link. + if info, ok := g.memberships[groupAddress]; ok { + info.cancelDelayedReportJob() + info.lastToSendReport = false + g.memberships[groupAddress] = info + } +} + +// initializeNewMemberLocked initializes a new group membership. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) initializeNewMemberLocked(groupAddress tcpip.Address, info *multicastGroupState, callersV2ReportBuilder MulticastGroupProtocolV2ReportBuilder) { + if !g.shouldPerformForGroup(groupAddress) { + return + } + + info.lastToSendReport = false + + switch g.mode { + case protocolModeV2: + info.transmissionLeft = g.robustnessVariable + if callersV2ReportBuilder == nil { + g.sendV2ReportAndMaybeScheduleChangedTimer(groupAddress, info, MulticastGroupProtocolV2ReportRecordChangeToExcludeMode) + } else { + callersV2ReportBuilder.AddRecord(MulticastGroupProtocolV2ReportRecordChangeToExcludeMode, groupAddress) + info.transmissionLeft-- + } + case protocolModeV1Compatibility, protocolModeV1: + info.transmissionLeft = unsolicitedTransmissionCount + g.maybeSendReportLocked(groupAddress, info) + default: + panic(fmt.Sprintf("unrecognized mode = %d", g.mode)) + } +} + +func (g *GenericMulticastProtocolState) shouldPerformForGroup(groupAddress tcpip.Address) bool { + return g.opts.Protocol.ShouldPerformProtocol(groupAddress) && g.opts.Protocol.Enabled() +} + +// maybeSendReportLocked attempts to send a report for a group. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) maybeSendReportLocked(groupAddress tcpip.Address, info *multicastGroupState) { + if info.transmissionLeft == 0 { + return + } + + // As per RFC 2236 section 3 page 5 (for IGMPv2), + // + // When a host joins a multicast group, it should immediately transmit an + // unsolicited Version 2 Membership Report for that group" ... "it is + // recommended that it be repeated". + // + // As per RFC 2710 section 4 page 6 (for MLDv1), + // + // When a node starts listening to a multicast address on an interface, + // it should immediately transmit an unsolicited Report for that address + // on that interface, in case it is the first listener on the link. To + // cover the possibility of the initial Report being lost or damaged, it + // is recommended that it be repeated once or twice after short delays + // [Unsolicited Report Interval]. + // + // TODO(gvisor.dev/issue/4901): Support a configurable number of initial + // unsolicited reports. + sent, err := g.opts.Protocol.SendReport(groupAddress) + if err == nil && sent { + info.lastToSendReport = true + + info.transmissionLeft-- + if info.transmissionLeft > 0 { + g.setDelayTimerForAddressLocked( + groupAddress, + info, + g.calculateDelayTimerDuration(g.opts.MaxUnsolicitedReportDelay), + ) + } + } +} + +// maybeSendLeave attempts to send a leave message. +func (g *GenericMulticastProtocolState) maybeSendLeave(groupAddress tcpip.Address, lastToSendReport bool) { + if !g.shouldPerformForGroup(groupAddress) || !lastToSendReport { + return + } + + // Okay to ignore the error here as if packet write failed, the multicast + // routers will eventually drop our membership anyways. If the interface is + // being disabled or removed, the generic multicast protocol's should be + // cleared eventually. + // + // As per RFC 2236 section 3 page 5 (for IGMPv2), + // + // When a router receives a Report, it adds the group being reported to + // the list of multicast group memberships on the network on which it + // received the Report and sets the timer for the membership to the + // [Group Membership Interval]. Repeated Reports refresh the timer. If + // no Reports are received for a particular group before this timer has + // expired, the router assumes that the group has no local members and + // that it need not forward remotely-originated multicasts for that + // group onto the attached network. + // + // As per RFC 2710 section 4 page 5 (for MLDv1), + // + // When a router receives a Report from a link, if the reported address + // is not already present in the router's list of multicast address + // having listeners on that link, the reported address is added to the + // list, its timer is set to [Multicast Listener Interval], and its + // appearance is made known to the router's multicast routing component. + // If a Report is received for a multicast address that is already + // present in the router's list, the timer for that address is reset to + // [Multicast Listener Interval]. If an address's timer expires, it is + // assumed that there are no longer any listeners for that address + // present on the link, so it is deleted from the list and its + // disappearance is made known to the multicast routing component. + // + // The requirement to send a leave message is also optional (it MAY be + // skipped): + // + // As per RFC 2236 section 6 page 8 (for IGMPv2), + // + // "send leave" for the group on the interface. If the interface + // state says the Querier is running IGMPv1, this action SHOULD be + // skipped. If the flag saying we were the last host to report is + // cleared, this action MAY be skipped. The Leave Message is sent to + // the ALL-ROUTERS group (224.0.0.2). + // + // As per RFC 2710 section 5 page 8 (for MLDv1), + // + // "send done" for the address on the interface. If the flag saying + // we were the last node to report is cleared, this action MAY be + // skipped. The Done message is sent to the link-scope all-routers + // address (FF02::2). + _ = g.opts.Protocol.SendLeave(groupAddress) +} + +// transitionToNonMemberLocked transitions the given multicast group the the +// non-member/listener state. +// +// Precondition: g.protocolMU must be locked. +func (g *GenericMulticastProtocolState) transitionToNonMemberLocked(groupAddress tcpip.Address, info *multicastGroupState) { + info.cancelDelayedReportJob() + g.maybeSendLeave(groupAddress, info.lastToSendReport) + info.lastToSendReport = false +} + +// setDelayTimerForAddressLocked sets timer to send a delayed report. +// +// Precondition: g.protocolMU MUST be locked. +func (g *GenericMulticastProtocolState) setDelayTimerForAddressLocked(groupAddress tcpip.Address, info *multicastGroupState, maxResponseTime time.Duration) { + if !g.shouldPerformForGroup(groupAddress) { + return + } + + if info.transmissionLeft < minQueryResponseTransmissionCount { + info.transmissionLeft = minQueryResponseTransmissionCount + } + + // As per RFC 2236 section 3 page 3 (for IGMPv2), + // + // If a timer for the group is already running, it is reset to the random + // value only if the requested Max Response Time is less than the remaining + // value of the running timer. + // + // As per RFC 2710 section 4 page 5 (for MLDv1), + // + // If a timer for any address is already running, it is reset to the new + // random value only if the requested Maximum Response Delay is less than + // the remaining value of the running timer. + now := g.opts.Clock.Now() + if !info.delayedReportJobFiresAt.IsZero() && info.delayedReportJobFiresAt.Sub(now) <= maxResponseTime { + // The timer is scheduled to fire before the maximum response time so we + // leave our timer as is. + return + } + + info.delayedReportJob.Cancel() + info.delayedReportJob.Schedule(maxResponseTime) + info.delayedReportJobFiresAt = now.Add(maxResponseTime) +} + +// calculateDelayTimerDuration returns a random time between (0, maxRespTime]. +func (g *GenericMulticastProtocolState) calculateDelayTimerDuration(maxRespTime time.Duration) time.Duration { + // As per RFC 2236 section 3 page 3 (for IGMPv2), + // + // When a host receives a Group-Specific Query, it sets a delay timer to a + // random value selected from the range (0, Max Response Time]... + // + // As per RFC 2710 section 4 page 6 (for MLDv1), + // + // When a node receives a Multicast-Address-Specific Query, if it is + // listening to the queried Multicast Address on the interface from + // which the Query was received, it sets a delay timer for that address + // to a random value selected from the range [0, Maximum Response Delay], + // as above. + if maxRespTime == 0 { + return 0 + } + return time.Duration(g.opts.Rand.Int63n(int64(maxRespTime))) +} diff --git a/pkg/tcpip/network/internal/ip/ip_state_autogen.go b/pkg/tcpip/network/internal/ip/ip_state_autogen.go new file mode 100644 index 0000000..2aaa1cc --- /dev/null +++ b/pkg/tcpip/network/internal/ip/ip_state_autogen.go @@ -0,0 +1,435 @@ +// automatically generated by stateify. + +package ip + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (d *dadState) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.dadState" +} + +func (d *dadState) StateFields() []string { + return []string{ + "nonce", + "extendRequest", + "done", + "timer", + "completionHandlers", + } +} + +func (d *dadState) beforeSave() {} + +// +checklocksignore +func (d *dadState) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.nonce) + stateSinkObject.Save(1, &d.extendRequest) + stateSinkObject.Save(2, &d.done) + stateSinkObject.Save(3, &d.timer) + stateSinkObject.Save(4, &d.completionHandlers) +} + +func (d *dadState) afterLoad(context.Context) {} + +// +checklocksignore +func (d *dadState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.nonce) + stateSourceObject.Load(1, &d.extendRequest) + stateSourceObject.Load(2, &d.done) + stateSourceObject.Load(3, &d.timer) + stateSourceObject.Load(4, &d.completionHandlers) +} + +func (d *DADOptions) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.DADOptions" +} + +func (d *DADOptions) StateFields() []string { + return []string{ + "Clock", + "NonceSize", + "ExtendDADTransmits", + "Protocol", + "NICID", + } +} + +func (d *DADOptions) beforeSave() {} + +// +checklocksignore +func (d *DADOptions) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.Clock) + stateSinkObject.Save(1, &d.NonceSize) + stateSinkObject.Save(2, &d.ExtendDADTransmits) + stateSinkObject.Save(3, &d.Protocol) + stateSinkObject.Save(4, &d.NICID) +} + +func (d *DADOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (d *DADOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.Clock) + stateSourceObject.Load(1, &d.NonceSize) + stateSourceObject.Load(2, &d.ExtendDADTransmits) + stateSourceObject.Load(3, &d.Protocol) + stateSourceObject.Load(4, &d.NICID) +} + +func (d *DAD) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.DAD" +} + +func (d *DAD) StateFields() []string { + return []string{ + "opts", + "configs", + "addresses", + } +} + +func (d *DAD) beforeSave() {} + +// +checklocksignore +func (d *DAD) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.opts) + stateSinkObject.Save(1, &d.configs) + stateSinkObject.Save(2, &d.addresses) +} + +func (d *DAD) afterLoad(context.Context) {} + +// +checklocksignore +func (d *DAD) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.opts) + stateSourceObject.Load(1, &d.configs) + stateSourceObject.Load(2, &d.addresses) +} + +func (e *ErrMessageTooLong) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.ErrMessageTooLong" +} + +func (e *ErrMessageTooLong) StateFields() []string { + return []string{} +} + +func (e *ErrMessageTooLong) beforeSave() {} + +// +checklocksignore +func (e *ErrMessageTooLong) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrMessageTooLong) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrMessageTooLong) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNoMulticastPendingQueueBufferSpace) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.ErrNoMulticastPendingQueueBufferSpace" +} + +func (e *ErrNoMulticastPendingQueueBufferSpace) StateFields() []string { + return []string{} +} + +func (e *ErrNoMulticastPendingQueueBufferSpace) beforeSave() {} + +// +checklocksignore +func (e *ErrNoMulticastPendingQueueBufferSpace) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNoMulticastPendingQueueBufferSpace) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNoMulticastPendingQueueBufferSpace) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (m *multicastGroupState) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.multicastGroupState" +} + +func (m *multicastGroupState) StateFields() []string { + return []string{ + "joins", + "transmissionLeft", + "lastToSendReport", + "delayedReportJob", + "queriedIncludeSources", + "deleteScheduled", + } +} + +func (m *multicastGroupState) beforeSave() {} + +// +checklocksignore +func (m *multicastGroupState) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.joins) + stateSinkObject.Save(1, &m.transmissionLeft) + stateSinkObject.Save(2, &m.lastToSendReport) + stateSinkObject.Save(3, &m.delayedReportJob) + stateSinkObject.Save(4, &m.queriedIncludeSources) + stateSinkObject.Save(5, &m.deleteScheduled) +} + +func (m *multicastGroupState) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multicastGroupState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.joins) + stateSourceObject.Load(1, &m.transmissionLeft) + stateSourceObject.Load(2, &m.lastToSendReport) + stateSourceObject.Load(3, &m.delayedReportJob) + stateSourceObject.Load(4, &m.queriedIncludeSources) + stateSourceObject.Load(5, &m.deleteScheduled) +} + +func (g *GenericMulticastProtocolOptions) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.GenericMulticastProtocolOptions" +} + +func (g *GenericMulticastProtocolOptions) StateFields() []string { + return []string{ + "Clock", + "Protocol", + "MaxUnsolicitedReportDelay", + } +} + +func (g *GenericMulticastProtocolOptions) beforeSave() {} + +// +checklocksignore +func (g *GenericMulticastProtocolOptions) StateSave(stateSinkObject state.Sink) { + g.beforeSave() + stateSinkObject.Save(0, &g.Clock) + stateSinkObject.Save(1, &g.Protocol) + stateSinkObject.Save(2, &g.MaxUnsolicitedReportDelay) +} + +func (g *GenericMulticastProtocolOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (g *GenericMulticastProtocolOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &g.Clock) + stateSourceObject.Load(1, &g.Protocol) + stateSourceObject.Load(2, &g.MaxUnsolicitedReportDelay) +} + +func (g *GenericMulticastProtocolState) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.GenericMulticastProtocolState" +} + +func (g *GenericMulticastProtocolState) StateFields() []string { + return []string{ + "opts", + "memberships", + "robustnessVariable", + "queryInterval", + "mode", + "modeTimer", + "generalQueryV2Timer", + "stateChangedReportV2Timer", + "stateChangedReportV2TimerSet", + } +} + +func (g *GenericMulticastProtocolState) beforeSave() {} + +// +checklocksignore +func (g *GenericMulticastProtocolState) StateSave(stateSinkObject state.Sink) { + g.beforeSave() + stateSinkObject.Save(0, &g.opts) + stateSinkObject.Save(1, &g.memberships) + stateSinkObject.Save(2, &g.robustnessVariable) + stateSinkObject.Save(3, &g.queryInterval) + stateSinkObject.Save(4, &g.mode) + stateSinkObject.Save(5, &g.modeTimer) + stateSinkObject.Save(6, &g.generalQueryV2Timer) + stateSinkObject.Save(7, &g.stateChangedReportV2Timer) + stateSinkObject.Save(8, &g.stateChangedReportV2TimerSet) +} + +func (g *GenericMulticastProtocolState) afterLoad(context.Context) {} + +// +checklocksignore +func (g *GenericMulticastProtocolState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &g.opts) + stateSourceObject.Load(1, &g.memberships) + stateSourceObject.Load(2, &g.robustnessVariable) + stateSourceObject.Load(3, &g.queryInterval) + stateSourceObject.Load(4, &g.mode) + stateSourceObject.Load(5, &g.modeTimer) + stateSourceObject.Load(6, &g.generalQueryV2Timer) + stateSourceObject.Load(7, &g.stateChangedReportV2Timer) + stateSourceObject.Load(8, &g.stateChangedReportV2TimerSet) +} + +func (m *MultiCounterIPForwardingStats) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.MultiCounterIPForwardingStats" +} + +func (m *MultiCounterIPForwardingStats) StateFields() []string { + return []string{ + "Unrouteable", + "ExhaustedTTL", + "InitializingSource", + "LinkLocalSource", + "LinkLocalDestination", + "PacketTooBig", + "HostUnreachable", + "ExtensionHeaderProblem", + "UnexpectedMulticastInputInterface", + "UnknownOutputEndpoint", + "NoMulticastPendingQueueBufferSpace", + "OutgoingDeviceNoBufferSpace", + "Errors", + "OutgoingDeviceClosedForSend", + } +} + +func (m *MultiCounterIPForwardingStats) beforeSave() {} + +// +checklocksignore +func (m *MultiCounterIPForwardingStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.Unrouteable) + stateSinkObject.Save(1, &m.ExhaustedTTL) + stateSinkObject.Save(2, &m.InitializingSource) + stateSinkObject.Save(3, &m.LinkLocalSource) + stateSinkObject.Save(4, &m.LinkLocalDestination) + stateSinkObject.Save(5, &m.PacketTooBig) + stateSinkObject.Save(6, &m.HostUnreachable) + stateSinkObject.Save(7, &m.ExtensionHeaderProblem) + stateSinkObject.Save(8, &m.UnexpectedMulticastInputInterface) + stateSinkObject.Save(9, &m.UnknownOutputEndpoint) + stateSinkObject.Save(10, &m.NoMulticastPendingQueueBufferSpace) + stateSinkObject.Save(11, &m.OutgoingDeviceNoBufferSpace) + stateSinkObject.Save(12, &m.Errors) + stateSinkObject.Save(13, &m.OutgoingDeviceClosedForSend) +} + +func (m *MultiCounterIPForwardingStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *MultiCounterIPForwardingStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.Unrouteable) + stateSourceObject.Load(1, &m.ExhaustedTTL) + stateSourceObject.Load(2, &m.InitializingSource) + stateSourceObject.Load(3, &m.LinkLocalSource) + stateSourceObject.Load(4, &m.LinkLocalDestination) + stateSourceObject.Load(5, &m.PacketTooBig) + stateSourceObject.Load(6, &m.HostUnreachable) + stateSourceObject.Load(7, &m.ExtensionHeaderProblem) + stateSourceObject.Load(8, &m.UnexpectedMulticastInputInterface) + stateSourceObject.Load(9, &m.UnknownOutputEndpoint) + stateSourceObject.Load(10, &m.NoMulticastPendingQueueBufferSpace) + stateSourceObject.Load(11, &m.OutgoingDeviceNoBufferSpace) + stateSourceObject.Load(12, &m.Errors) + stateSourceObject.Load(13, &m.OutgoingDeviceClosedForSend) +} + +func (m *MultiCounterIPStats) StateTypeName() string { + return "pkg/tcpip/network/internal/ip.MultiCounterIPStats" +} + +func (m *MultiCounterIPStats) StateFields() []string { + return []string{ + "PacketsReceived", + "ValidPacketsReceived", + "DisabledPacketsReceived", + "InvalidDestinationAddressesReceived", + "InvalidSourceAddressesReceived", + "PacketsDelivered", + "PacketsSent", + "OutgoingPacketErrors", + "MalformedPacketsReceived", + "MalformedFragmentsReceived", + "IPTablesPreroutingDropped", + "IPTablesInputDropped", + "IPTablesForwardDropped", + "IPTablesOutputDropped", + "IPTablesPostroutingDropped", + "OptionTimestampReceived", + "OptionRecordRouteReceived", + "OptionRouterAlertReceived", + "OptionUnknownReceived", + "Forwarding", + } +} + +func (m *MultiCounterIPStats) beforeSave() {} + +// +checklocksignore +func (m *MultiCounterIPStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.PacketsReceived) + stateSinkObject.Save(1, &m.ValidPacketsReceived) + stateSinkObject.Save(2, &m.DisabledPacketsReceived) + stateSinkObject.Save(3, &m.InvalidDestinationAddressesReceived) + stateSinkObject.Save(4, &m.InvalidSourceAddressesReceived) + stateSinkObject.Save(5, &m.PacketsDelivered) + stateSinkObject.Save(6, &m.PacketsSent) + stateSinkObject.Save(7, &m.OutgoingPacketErrors) + stateSinkObject.Save(8, &m.MalformedPacketsReceived) + stateSinkObject.Save(9, &m.MalformedFragmentsReceived) + stateSinkObject.Save(10, &m.IPTablesPreroutingDropped) + stateSinkObject.Save(11, &m.IPTablesInputDropped) + stateSinkObject.Save(12, &m.IPTablesForwardDropped) + stateSinkObject.Save(13, &m.IPTablesOutputDropped) + stateSinkObject.Save(14, &m.IPTablesPostroutingDropped) + stateSinkObject.Save(15, &m.OptionTimestampReceived) + stateSinkObject.Save(16, &m.OptionRecordRouteReceived) + stateSinkObject.Save(17, &m.OptionRouterAlertReceived) + stateSinkObject.Save(18, &m.OptionUnknownReceived) + stateSinkObject.Save(19, &m.Forwarding) +} + +func (m *MultiCounterIPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *MultiCounterIPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.PacketsReceived) + stateSourceObject.Load(1, &m.ValidPacketsReceived) + stateSourceObject.Load(2, &m.DisabledPacketsReceived) + stateSourceObject.Load(3, &m.InvalidDestinationAddressesReceived) + stateSourceObject.Load(4, &m.InvalidSourceAddressesReceived) + stateSourceObject.Load(5, &m.PacketsDelivered) + stateSourceObject.Load(6, &m.PacketsSent) + stateSourceObject.Load(7, &m.OutgoingPacketErrors) + stateSourceObject.Load(8, &m.MalformedPacketsReceived) + stateSourceObject.Load(9, &m.MalformedFragmentsReceived) + stateSourceObject.Load(10, &m.IPTablesPreroutingDropped) + stateSourceObject.Load(11, &m.IPTablesInputDropped) + stateSourceObject.Load(12, &m.IPTablesForwardDropped) + stateSourceObject.Load(13, &m.IPTablesOutputDropped) + stateSourceObject.Load(14, &m.IPTablesPostroutingDropped) + stateSourceObject.Load(15, &m.OptionTimestampReceived) + stateSourceObject.Load(16, &m.OptionRecordRouteReceived) + stateSourceObject.Load(17, &m.OptionRouterAlertReceived) + stateSourceObject.Load(18, &m.OptionUnknownReceived) + stateSourceObject.Load(19, &m.Forwarding) +} + +func init() { + state.Register((*dadState)(nil)) + state.Register((*DADOptions)(nil)) + state.Register((*DAD)(nil)) + state.Register((*ErrMessageTooLong)(nil)) + state.Register((*ErrNoMulticastPendingQueueBufferSpace)(nil)) + state.Register((*multicastGroupState)(nil)) + state.Register((*GenericMulticastProtocolOptions)(nil)) + state.Register((*GenericMulticastProtocolState)(nil)) + state.Register((*MultiCounterIPForwardingStats)(nil)) + state.Register((*MultiCounterIPStats)(nil)) +} diff --git a/pkg/tcpip/network/internal/ip/stats.go b/pkg/tcpip/network/internal/ip/stats.go new file mode 100644 index 0000000..31573e6 --- /dev/null +++ b/pkg/tcpip/network/internal/ip/stats.go @@ -0,0 +1,219 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ip + +import "github.com/sagernet/gvisor/pkg/tcpip" + +// LINT.IfChange(MultiCounterIPForwardingStats) + +// MultiCounterIPForwardingStats holds IP forwarding statistics. Each counter +// may have several versions. +// +// +stateify savable +type MultiCounterIPForwardingStats struct { + // Unrouteable is the number of IP packets received which were dropped + // because the netstack could not construct a route to their + // destination. + Unrouteable tcpip.MultiCounterStat + + // ExhaustedTTL is the number of IP packets received which were dropped + // because their TTL was exhausted. + ExhaustedTTL tcpip.MultiCounterStat + + // InitializingSource is the number of IP packets which were dropped + // because they contained a source address that may only be used on the local + // network as part of initialization work. + InitializingSource tcpip.MultiCounterStat + + // LinkLocalSource is the number of IP packets which were dropped + // because they contained a link-local source address. + LinkLocalSource tcpip.MultiCounterStat + + // LinkLocalDestination is the number of IP packets which were dropped + // because they contained a link-local destination address. + LinkLocalDestination tcpip.MultiCounterStat + + // PacketTooBig is the number of IP packets which were dropped because they + // were too big for the outgoing MTU. + PacketTooBig tcpip.MultiCounterStat + + // HostUnreachable is the number of IP packets received which could not be + // successfully forwarded due to an unresolvable next hop. + HostUnreachable tcpip.MultiCounterStat + + // ExtensionHeaderProblem is the number of IP packets which were dropped + // because of a problem encountered when processing an IPv6 extension + // header. + ExtensionHeaderProblem tcpip.MultiCounterStat + + // UnexpectedMulticastInputInterface is the number of multicast packets that + // were received on an interface that did not match the corresponding route's + // expected input interface. + UnexpectedMulticastInputInterface tcpip.MultiCounterStat + + // UnknownOutputEndpoint is the number of packets that could not be forwarded + // because the output endpoint could not be found. + UnknownOutputEndpoint tcpip.MultiCounterStat + + // NoMulticastPendingQueueBufferSpace is the number of multicast packets that + // were dropped due to insufficient buffer space in the pending packet queue. + NoMulticastPendingQueueBufferSpace tcpip.MultiCounterStat + + // OutgoingDeviceNoBufferSpace is the number of packets that were dropped due + // to insufficient space in the outgoing device. + OutgoingDeviceNoBufferSpace tcpip.MultiCounterStat + + // Errors is the number of IP packets received which could not be + // successfully forwarded. + Errors tcpip.MultiCounterStat + + // OutgoingDeviceClosedForSend is the number of packets that were dropped due + // to the outgoing device being closed for send. + OutgoingDeviceClosedForSend tcpip.MultiCounterStat +} + +// Init sets internal counters to track a and b counters. +func (m *MultiCounterIPForwardingStats) Init(a, b *tcpip.IPForwardingStats) { + m.Unrouteable.Init(a.Unrouteable, b.Unrouteable) + m.Errors.Init(a.Errors, b.Errors) + m.InitializingSource.Init(a.InitializingSource, b.InitializingSource) + m.LinkLocalSource.Init(a.LinkLocalSource, b.LinkLocalSource) + m.LinkLocalDestination.Init(a.LinkLocalDestination, b.LinkLocalDestination) + m.ExtensionHeaderProblem.Init(a.ExtensionHeaderProblem, b.ExtensionHeaderProblem) + m.PacketTooBig.Init(a.PacketTooBig, b.PacketTooBig) + m.ExhaustedTTL.Init(a.ExhaustedTTL, b.ExhaustedTTL) + m.HostUnreachable.Init(a.HostUnreachable, b.HostUnreachable) + m.UnexpectedMulticastInputInterface.Init(a.UnexpectedMulticastInputInterface, b.UnexpectedMulticastInputInterface) + m.UnknownOutputEndpoint.Init(a.UnknownOutputEndpoint, b.UnknownOutputEndpoint) + m.NoMulticastPendingQueueBufferSpace.Init(a.NoMulticastPendingQueueBufferSpace, b.NoMulticastPendingQueueBufferSpace) + m.OutgoingDeviceNoBufferSpace.Init(a.OutgoingDeviceNoBufferSpace, b.OutgoingDeviceNoBufferSpace) + m.OutgoingDeviceClosedForSend.Init(a.OutgoingDeviceClosedForSend, b.OutgoingDeviceClosedForSend) +} + +// LINT.ThenChange(../../../tcpip.go:IPForwardingStats) + +// LINT.IfChange(MultiCounterIPStats) + +// MultiCounterIPStats holds IP statistics, each counter may have several +// versions. +// +// +stateify savable +type MultiCounterIPStats struct { + // PacketsReceived is the number of IP packets received from the link + // layer. + PacketsReceived tcpip.MultiCounterStat + + // ValidPacketsReceived is the number of valid IP packets that reached the IP + // layer. + ValidPacketsReceived tcpip.MultiCounterStat + + // DisabledPacketsReceived is the number of IP packets received from + // the link layer when the IP layer is disabled. + DisabledPacketsReceived tcpip.MultiCounterStat + + // InvalidDestinationAddressesReceived is the number of IP packets + // received with an unknown or invalid destination address. + InvalidDestinationAddressesReceived tcpip.MultiCounterStat + + // InvalidSourceAddressesReceived is the number of IP packets received + // with a source address that should never have been received on the + // wire. + InvalidSourceAddressesReceived tcpip.MultiCounterStat + + // PacketsDelivered is the number of incoming IP packets successfully + // delivered to the transport layer. + PacketsDelivered tcpip.MultiCounterStat + + // PacketsSent is the number of IP packets sent via WritePacket. + PacketsSent tcpip.MultiCounterStat + + // OutgoingPacketErrors is the number of IP packets which failed to + // write to a link-layer endpoint. + OutgoingPacketErrors tcpip.MultiCounterStat + + // MalformedPacketsReceived is the number of IP Packets that were + // dropped due to the IP packet header failing validation checks. + MalformedPacketsReceived tcpip.MultiCounterStat + + // MalformedFragmentsReceived is the number of IP Fragments that were + // dropped due to the fragment failing validation checks. + MalformedFragmentsReceived tcpip.MultiCounterStat + + // IPTablesPreroutingDropped is the number of IP packets dropped in the + // Prerouting chain. + IPTablesPreroutingDropped tcpip.MultiCounterStat + + // IPTablesInputDropped is the number of IP packets dropped in the + // Input chain. + IPTablesInputDropped tcpip.MultiCounterStat + + // IPTablesForwardDropped is the number of IP packets dropped in the + // Forward chain. + IPTablesForwardDropped tcpip.MultiCounterStat + + // IPTablesOutputDropped is the number of IP packets dropped in the + // Output chain. + IPTablesOutputDropped tcpip.MultiCounterStat + + // IPTablesPostroutingDropped is the number of IP packets dropped in + // the Postrouting chain. + IPTablesPostroutingDropped tcpip.MultiCounterStat + + // TODO(https://gvisor.dev/issues/5529): Move the IPv4-only option + // stats out of IPStats. + + // OptionTimestampReceived is the number of Timestamp options seen. + OptionTimestampReceived tcpip.MultiCounterStat + + // OptionRecordRouteReceived is the number of Record Route options + // seen. + OptionRecordRouteReceived tcpip.MultiCounterStat + + // OptionRouterAlertReceived is the number of Router Alert options + // seen. + OptionRouterAlertReceived tcpip.MultiCounterStat + + // OptionUnknownReceived is the number of unknown IP options seen. + OptionUnknownReceived tcpip.MultiCounterStat + + // Forwarding collects stats related to IP forwarding. + Forwarding MultiCounterIPForwardingStats +} + +// Init sets internal counters to track a and b counters. +func (m *MultiCounterIPStats) Init(a, b *tcpip.IPStats) { + m.PacketsReceived.Init(a.PacketsReceived, b.PacketsReceived) + m.ValidPacketsReceived.Init(a.ValidPacketsReceived, b.ValidPacketsReceived) + m.DisabledPacketsReceived.Init(a.DisabledPacketsReceived, b.DisabledPacketsReceived) + m.InvalidDestinationAddressesReceived.Init(a.InvalidDestinationAddressesReceived, b.InvalidDestinationAddressesReceived) + m.InvalidSourceAddressesReceived.Init(a.InvalidSourceAddressesReceived, b.InvalidSourceAddressesReceived) + m.PacketsDelivered.Init(a.PacketsDelivered, b.PacketsDelivered) + m.PacketsSent.Init(a.PacketsSent, b.PacketsSent) + m.OutgoingPacketErrors.Init(a.OutgoingPacketErrors, b.OutgoingPacketErrors) + m.MalformedPacketsReceived.Init(a.MalformedPacketsReceived, b.MalformedPacketsReceived) + m.MalformedFragmentsReceived.Init(a.MalformedFragmentsReceived, b.MalformedFragmentsReceived) + m.IPTablesPreroutingDropped.Init(a.IPTablesPreroutingDropped, b.IPTablesPreroutingDropped) + m.IPTablesInputDropped.Init(a.IPTablesInputDropped, b.IPTablesInputDropped) + m.IPTablesForwardDropped.Init(a.IPTablesForwardDropped, b.IPTablesForwardDropped) + m.IPTablesOutputDropped.Init(a.IPTablesOutputDropped, b.IPTablesOutputDropped) + m.IPTablesPostroutingDropped.Init(a.IPTablesPostroutingDropped, b.IPTablesPostroutingDropped) + m.OptionTimestampReceived.Init(a.OptionTimestampReceived, b.OptionTimestampReceived) + m.OptionRecordRouteReceived.Init(a.OptionRecordRouteReceived, b.OptionRecordRouteReceived) + m.OptionRouterAlertReceived.Init(a.OptionRouterAlertReceived, b.OptionRouterAlertReceived) + m.OptionUnknownReceived.Init(a.OptionUnknownReceived, b.OptionUnknownReceived) + m.Forwarding.Init(&a.Forwarding, &b.Forwarding) +} + +// LINT.ThenChange(../../../tcpip.go:IPStats) diff --git a/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go b/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go new file mode 100644 index 0000000..1af40ce --- /dev/null +++ b/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go @@ -0,0 +1,137 @@ +// automatically generated by stateify. + +package multicast + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (r *RouteTable) StateTypeName() string { + return "pkg/tcpip/network/internal/multicast.RouteTable" +} + +func (r *RouteTable) StateFields() []string { + return []string{ + "installedRoutes", + "pendingRoutes", + "cleanupPendingRoutesTimer", + "isCleanupRoutineRunning", + "config", + } +} + +func (r *RouteTable) beforeSave() {} + +// +checklocksignore +func (r *RouteTable) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.installedRoutes) + stateSinkObject.Save(1, &r.pendingRoutes) + stateSinkObject.Save(2, &r.cleanupPendingRoutesTimer) + stateSinkObject.Save(3, &r.isCleanupRoutineRunning) + stateSinkObject.Save(4, &r.config) +} + +func (r *RouteTable) afterLoad(context.Context) {} + +// +checklocksignore +func (r *RouteTable) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.installedRoutes) + stateSourceObject.Load(1, &r.pendingRoutes) + stateSourceObject.Load(2, &r.cleanupPendingRoutesTimer) + stateSourceObject.Load(3, &r.isCleanupRoutineRunning) + stateSourceObject.Load(4, &r.config) +} + +func (r *InstalledRoute) StateTypeName() string { + return "pkg/tcpip/network/internal/multicast.InstalledRoute" +} + +func (r *InstalledRoute) StateFields() []string { + return []string{ + "MulticastRoute", + "lastUsedTimestamp", + } +} + +func (r *InstalledRoute) beforeSave() {} + +// +checklocksignore +func (r *InstalledRoute) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.MulticastRoute) + stateSinkObject.Save(1, &r.lastUsedTimestamp) +} + +func (r *InstalledRoute) afterLoad(context.Context) {} + +// +checklocksignore +func (r *InstalledRoute) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.MulticastRoute) + stateSourceObject.Load(1, &r.lastUsedTimestamp) +} + +func (p *PendingRoute) StateTypeName() string { + return "pkg/tcpip/network/internal/multicast.PendingRoute" +} + +func (p *PendingRoute) StateFields() []string { + return []string{ + "packets", + "expiration", + } +} + +func (p *PendingRoute) beforeSave() {} + +// +checklocksignore +func (p *PendingRoute) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.packets) + stateSinkObject.Save(1, &p.expiration) +} + +func (p *PendingRoute) afterLoad(context.Context) {} + +// +checklocksignore +func (p *PendingRoute) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.packets) + stateSourceObject.Load(1, &p.expiration) +} + +func (c *Config) StateTypeName() string { + return "pkg/tcpip/network/internal/multicast.Config" +} + +func (c *Config) StateFields() []string { + return []string{ + "MaxPendingQueueSize", + "Clock", + } +} + +func (c *Config) beforeSave() {} + +// +checklocksignore +func (c *Config) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.MaxPendingQueueSize) + stateSinkObject.Save(1, &c.Clock) +} + +func (c *Config) afterLoad(context.Context) {} + +// +checklocksignore +func (c *Config) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.MaxPendingQueueSize) + stateSourceObject.Load(1, &c.Clock) +} + +func init() { + state.Register((*RouteTable)(nil)) + state.Register((*InstalledRoute)(nil)) + state.Register((*PendingRoute)(nil)) + state.Register((*Config)(nil)) +} diff --git a/pkg/tcpip/network/internal/multicast/route_table.go b/pkg/tcpip/network/internal/multicast/route_table.go new file mode 100644 index 0000000..490e83d --- /dev/null +++ b/pkg/tcpip/network/internal/multicast/route_table.go @@ -0,0 +1,446 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package multicast contains utilities for supporting multicast routing. +package multicast + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// RouteTable represents a multicast routing table. +// +// +stateify savable +type RouteTable struct { + // Internally, installed and pending routes are stored and locked separately + // A couple of reasons for structuring the table this way: + // + // 1. We can avoid write locking installed routes when pending packets are + // being queued. In other words, the happy path of reading installed + // routes doesn't require an exclusive lock. + // 2. The cleanup process for expired routes only needs to operate on pending + // routes. Like above, a write lock on the installed routes can be + // avoided. + // 3. This structure is similar to the Linux implementation: + // https://github.com/torvalds/linux/blob/cffb2b72d3e/include/linux/mroute_base.h#L250 + + // The installedMu lock should typically be acquired before the pendingMu + // lock. This ensures that installed routes can continue to be read even when + // the pending routes are write locked. + + installedMu sync.RWMutex `state:"nosave"` + // Maintaining pointers ensures that the installed routes are exclusively + // locked only when a route is being installed. + // +checklocks:installedMu + installedRoutes map[stack.UnicastSourceAndMulticastDestination]*InstalledRoute + + pendingMu sync.RWMutex `state:"nosave"` + // +checklocks:pendingMu + pendingRoutes map[stack.UnicastSourceAndMulticastDestination]PendingRoute + // cleanupPendingRoutesTimer is a timer that triggers a routine to remove + // pending routes that are expired. + // +checklocks:pendingMu + cleanupPendingRoutesTimer tcpip.Timer + // +checklocks:pendingMu + isCleanupRoutineRunning bool + + config Config +} + +var ( + // ErrNoBufferSpace indicates that no buffer space is available in the + // pending route packet queue. + ErrNoBufferSpace = errors.New("unable to queue packet, no buffer space available") + + // ErrMissingClock indicates that a clock was not provided as part of the + // Config, but is required. + ErrMissingClock = errors.New("clock must not be nil") + + // ErrAlreadyInitialized indicates that RouteTable.Init was already invoked. + ErrAlreadyInitialized = errors.New("table is already initialized") +) + +// InstalledRoute represents a route that is in the installed state. +// +// If a route is in the installed state, then it may be used to forward +// multicast packets. +// +// +stateify savable +type InstalledRoute struct { + stack.MulticastRoute + + lastUsedTimestampMu sync.RWMutex `state:"nosave"` + // +checklocks:lastUsedTimestampMu + lastUsedTimestamp tcpip.MonotonicTime +} + +// LastUsedTimestamp returns a monotonic timestamp that corresponds to the last +// time the route was used or updated. +func (r *InstalledRoute) LastUsedTimestamp() tcpip.MonotonicTime { + r.lastUsedTimestampMu.RLock() + defer r.lastUsedTimestampMu.RUnlock() + + return r.lastUsedTimestamp +} + +// SetLastUsedTimestamp sets the time that the route was last used. +// +// The timestamp is only updated if it occurs after the currently set +// timestamp. Callers should invoke this anytime the route is used to forward a +// packet. +func (r *InstalledRoute) SetLastUsedTimestamp(monotonicTime tcpip.MonotonicTime) { + r.lastUsedTimestampMu.Lock() + defer r.lastUsedTimestampMu.Unlock() + + if monotonicTime.After(r.lastUsedTimestamp) { + r.lastUsedTimestamp = monotonicTime + } +} + +// PendingRoute represents a route that is in the "pending" state. +// +// A route is in the pending state if an installed route does not yet exist +// for the entry. For such routes, packets are added to an expiring queue until +// a route is installed. +// +// +stateify savable +type PendingRoute struct { + packets []*stack.PacketBuffer + + // expiration is the timestamp at which the pending route should be expired. + // + // If this value is before the current time, then this pending route will + // be dropped. + expiration tcpip.MonotonicTime +} + +func (p *PendingRoute) releasePackets() { + for _, pkt := range p.packets { + pkt.DecRef() + } +} + +func (p *PendingRoute) isExpired(currentTime tcpip.MonotonicTime) bool { + return currentTime.After(p.expiration) +} + +const ( + // DefaultMaxPendingQueueSize corresponds to the number of elements that can + // be in the packet queue for a pending route. + // + // Matches the Linux default queue size: + // https://github.com/torvalds/linux/blob/26291c54e11/net/ipv6/ip6mr.c#L1186 + DefaultMaxPendingQueueSize uint8 = 3 + + // DefaultPendingRouteExpiration is the default maximum lifetime of a pending + // route. + // + // Matches the Linux default: + // https://github.com/torvalds/linux/blob/26291c54e11/net/ipv6/ip6mr.c#L991 + DefaultPendingRouteExpiration time.Duration = 10 * time.Second + + // DefaultCleanupInterval is the default frequency of the routine that + // expires pending routes. + // + // Matches the Linux default: + // https://github.com/torvalds/linux/blob/26291c54e11/net/ipv6/ip6mr.c#L793 + DefaultCleanupInterval time.Duration = 10 * time.Second +) + +// Config represents the options for configuring a RouteTable. +// +// +stateify savable +type Config struct { + // MaxPendingQueueSize corresponds to the maximum number of queued packets + // for a pending route. + // + // If the caller attempts to queue a packet and the queue already contains + // MaxPendingQueueSize elements, then the packet will be rejected and should + // not be forwarded. + MaxPendingQueueSize uint8 + + // Clock represents the clock that should be used to obtain the current time. + // + // This field is required and must have a non-nil value. + Clock tcpip.Clock +} + +// DefaultConfig returns the default configuration for the table. +func DefaultConfig(clock tcpip.Clock) Config { + return Config{ + MaxPendingQueueSize: DefaultMaxPendingQueueSize, + Clock: clock, + } +} + +// Init initializes the RouteTable with the provided config. +// +// An error is returned if the config is not valid. +// +// Must be called before any other function on the table. +func (r *RouteTable) Init(config Config) error { + r.installedMu.Lock() + defer r.installedMu.Unlock() + r.pendingMu.Lock() + defer r.pendingMu.Unlock() + + if r.installedRoutes != nil { + return ErrAlreadyInitialized + } + + if config.Clock == nil { + return ErrMissingClock + } + + r.config = config + r.installedRoutes = make(map[stack.UnicastSourceAndMulticastDestination]*InstalledRoute) + r.pendingRoutes = make(map[stack.UnicastSourceAndMulticastDestination]PendingRoute) + + return nil +} + +// Close cleans up resources held by the table. +// +// Calling this will stop the cleanup routine and release any packets owned by +// the table. +func (r *RouteTable) Close() { + r.pendingMu.Lock() + defer r.pendingMu.Unlock() + + if r.cleanupPendingRoutesTimer != nil { + r.cleanupPendingRoutesTimer.Stop() + } + + for key, route := range r.pendingRoutes { + delete(r.pendingRoutes, key) + route.releasePackets() + } +} + +// maybeStopCleanupRoutine stops the pending routes cleanup routine if no +// pending routes exist. +// +// Returns true if the timer is not running. Otherwise, returns false. +// +// +checklocks:r.pendingMu +func (r *RouteTable) maybeStopCleanupRoutineLocked() bool { + if !r.isCleanupRoutineRunning { + return true + } + + if len(r.pendingRoutes) == 0 { + r.cleanupPendingRoutesTimer.Stop() + r.isCleanupRoutineRunning = false + return true + } + + return false +} + +func (r *RouteTable) cleanupPendingRoutes() { + currentTime := r.config.Clock.NowMonotonic() + r.pendingMu.Lock() + defer r.pendingMu.Unlock() + + for key, route := range r.pendingRoutes { + if route.isExpired(currentTime) { + delete(r.pendingRoutes, key) + route.releasePackets() + } + } + + if stopped := r.maybeStopCleanupRoutineLocked(); !stopped { + r.cleanupPendingRoutesTimer.Reset(DefaultCleanupInterval) + } +} + +func (r *RouteTable) newPendingRoute() PendingRoute { + return PendingRoute{ + packets: make([]*stack.PacketBuffer, 0, r.config.MaxPendingQueueSize), + expiration: r.config.Clock.NowMonotonic().Add(DefaultPendingRouteExpiration), + } +} + +// NewInstalledRoute instantiates an installed route for the table. +func (r *RouteTable) NewInstalledRoute(route stack.MulticastRoute) *InstalledRoute { + return &InstalledRoute{ + MulticastRoute: route, + lastUsedTimestamp: r.config.Clock.NowMonotonic(), + } +} + +// GetRouteResult represents the result of calling GetRouteOrInsertPending. +type GetRouteResult struct { + // GetRouteResultState signals the result of calling GetRouteOrInsertPending. + GetRouteResultState GetRouteResultState + + // InstalledRoute represents the existing installed route. This field will + // only be populated if the GetRouteResultState is InstalledRouteFound. + InstalledRoute *InstalledRoute +} + +// GetRouteResultState signals the result of calling GetRouteOrInsertPending. +type GetRouteResultState uint8 + +const ( + // InstalledRouteFound indicates that an InstalledRoute was found. + InstalledRouteFound GetRouteResultState = iota + + // PacketQueuedInPendingRoute indicates that the packet was queued in an + // existing pending route. + PacketQueuedInPendingRoute + + // NoRouteFoundAndPendingInserted indicates that no route was found and that + // a pending route was newly inserted into the RouteTable. + NoRouteFoundAndPendingInserted +) + +func (e GetRouteResultState) String() string { + switch e { + case InstalledRouteFound: + return "InstalledRouteFound" + case PacketQueuedInPendingRoute: + return "PacketQueuedInPendingRoute" + case NoRouteFoundAndPendingInserted: + return "NoRouteFoundAndPendingInserted" + default: + return fmt.Sprintf("%d", uint8(e)) + } +} + +// GetRouteOrInsertPending attempts to fetch the installed route that matches +// the provided key. +// +// If no matching installed route is found, then the pkt is cloned and queued +// in a pending route. The GetRouteResult.GetRouteResultState will indicate +// whether the pkt was queued in a new pending route or an existing one. +// +// If the relevant pending route queue is at max capacity, then returns false. +// Otherwise, returns true. +func (r *RouteTable) GetRouteOrInsertPending(key stack.UnicastSourceAndMulticastDestination, pkt *stack.PacketBuffer) (GetRouteResult, bool) { + r.installedMu.RLock() + defer r.installedMu.RUnlock() + + if route, ok := r.installedRoutes[key]; ok { + return GetRouteResult{GetRouteResultState: InstalledRouteFound, InstalledRoute: route}, true + } + + r.pendingMu.Lock() + defer r.pendingMu.Unlock() + + pendingRoute, getRouteResultState := r.getOrCreatePendingRouteRLocked(key) + if len(pendingRoute.packets) >= int(r.config.MaxPendingQueueSize) { + // The incoming packet is rejected if the pending queue is already at max + // capacity. This behavior matches the Linux implementation: + // https://github.com/torvalds/linux/blob/ae085d7f936/net/ipv4/ipmr.c#L1147 + return GetRouteResult{}, false + } + pendingRoute.packets = append(pendingRoute.packets, pkt.Clone()) + r.pendingRoutes[key] = pendingRoute + + if !r.isCleanupRoutineRunning { + // The cleanup routine isn't running, but should be. Start it. + if r.cleanupPendingRoutesTimer == nil { + r.cleanupPendingRoutesTimer = r.config.Clock.AfterFunc(DefaultCleanupInterval, r.cleanupPendingRoutes) + } else { + r.cleanupPendingRoutesTimer.Reset(DefaultCleanupInterval) + } + r.isCleanupRoutineRunning = true + } + + return GetRouteResult{GetRouteResultState: getRouteResultState, InstalledRoute: nil}, true +} + +// +checklocks:r.pendingMu +func (r *RouteTable) getOrCreatePendingRouteRLocked(key stack.UnicastSourceAndMulticastDestination) (PendingRoute, GetRouteResultState) { + if pendingRoute, ok := r.pendingRoutes[key]; ok { + return pendingRoute, PacketQueuedInPendingRoute + } + return r.newPendingRoute(), NoRouteFoundAndPendingInserted +} + +// AddInstalledRoute adds the provided route to the table. +// +// Packets that were queued while the route was in the pending state are +// returned. The caller assumes ownership of these packets and is responsible +// for forwarding and releasing them. If an installed route already exists for +// the provided key, then it is overwritten. +func (r *RouteTable) AddInstalledRoute(key stack.UnicastSourceAndMulticastDestination, route *InstalledRoute) []*stack.PacketBuffer { + r.installedMu.Lock() + defer r.installedMu.Unlock() + r.installedRoutes[key] = route + + r.pendingMu.Lock() + pendingRoute, ok := r.pendingRoutes[key] + delete(r.pendingRoutes, key) + // No need to reset the timer here. The cleanup routine is responsible for + // doing so. + _ = r.maybeStopCleanupRoutineLocked() + r.pendingMu.Unlock() + + // Ignore the pending route if it is expired. It may be in this state since + // the cleanup process is only run periodically. + if !ok || pendingRoute.isExpired(r.config.Clock.NowMonotonic()) { + pendingRoute.releasePackets() + return nil + } + + return pendingRoute.packets +} + +// RemoveInstalledRoute deletes any installed route that matches the provided +// key. +// +// Returns true if a route was removed. Otherwise returns false. +func (r *RouteTable) RemoveInstalledRoute(key stack.UnicastSourceAndMulticastDestination) bool { + r.installedMu.Lock() + defer r.installedMu.Unlock() + + if _, ok := r.installedRoutes[key]; ok { + delete(r.installedRoutes, key) + return true + } + + return false +} + +// RemoveAllInstalledRoutes removes all installed routes from the table. +func (r *RouteTable) RemoveAllInstalledRoutes() { + r.installedMu.Lock() + defer r.installedMu.Unlock() + + for key := range r.installedRoutes { + delete(r.installedRoutes, key) + } +} + +// GetLastUsedTimestamp returns a monotonic timestamp that represents the last +// time the route that matches the provided key was used or updated. +// +// Returns true if a matching route was found. Otherwise returns false. +func (r *RouteTable) GetLastUsedTimestamp(key stack.UnicastSourceAndMulticastDestination) (tcpip.MonotonicTime, bool) { + r.installedMu.RLock() + defer r.installedMu.RUnlock() + + if route, ok := r.installedRoutes[key]; ok { + return route.LastUsedTimestamp(), true + } + return tcpip.MonotonicTime{}, false +} diff --git a/pkg/tcpip/network/ipv4/icmp.go b/pkg/tcpip/network/ipv4/icmp.go new file mode 100644 index 0000000..5644c14 --- /dev/null +++ b/pkg/tcpip/network/ipv4/icmp.go @@ -0,0 +1,709 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv4 + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// icmpv4DestinationUnreachableSockError is a general ICMPv4 Destination +// Unreachable error. +// +// +stateify savable +type icmpv4DestinationUnreachableSockError struct{} + +// Origin implements tcpip.SockErrorCause. +func (*icmpv4DestinationUnreachableSockError) Origin() tcpip.SockErrOrigin { + return tcpip.SockExtErrorOriginICMP +} + +// Type implements tcpip.SockErrorCause. +func (*icmpv4DestinationUnreachableSockError) Type() uint8 { + return uint8(header.ICMPv4DstUnreachable) +} + +// Info implements tcpip.SockErrorCause. +func (*icmpv4DestinationUnreachableSockError) Info() uint32 { + return 0 +} + +var _ stack.TransportError = (*icmpv4DestinationHostUnreachableSockError)(nil) + +// icmpv4DestinationHostUnreachableSockError is an ICMPv4 Destination Host +// Unreachable error. +// +// It indicates that a packet was not able to reach the destination host. +// +// +stateify savable +type icmpv4DestinationHostUnreachableSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4DestinationHostUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv4HostUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv4DestinationHostUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationHostUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv4DestinationNetUnreachableSockError)(nil) + +// icmpv4DestinationNetUnreachableSockError is an ICMPv4 Destination Net +// Unreachable error. +// +// It indicates that a packet was not able to reach the destination network. +// +// +stateify savable +type icmpv4DestinationNetUnreachableSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4DestinationNetUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv4NetUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv4DestinationNetUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationNetworkUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv4DestinationPortUnreachableSockError)(nil) + +// icmpv4DestinationPortUnreachableSockError is an ICMPv4 Destination Port +// Unreachable error. +// +// It indicates that a packet reached the destination host, but the transport +// protocol was not active on the destination port. +// +// +stateify savable +type icmpv4DestinationPortUnreachableSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4DestinationPortUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv4PortUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv4DestinationPortUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationPortUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv4DestinationProtoUnreachableSockError)(nil) + +// icmpv4DestinationProtoUnreachableSockError is an ICMPv4 Destination Protocol +// Unreachable error. +// +// It indicates that a packet reached the destination host, but the transport +// protocol was not reachable +// +// +stateify savable +type icmpv4DestinationProtoUnreachableSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4DestinationProtoUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv4ProtoUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv4DestinationProtoUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationProtoUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv4SourceRouteFailedSockError)(nil) + +// icmpv4SourceRouteFailedSockError is an ICMPv4 Destination Unreachable error +// due to source route failed. +// +// +stateify savable +type icmpv4SourceRouteFailedSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4SourceRouteFailedSockError) Code() uint8 { + return uint8(header.ICMPv4SourceRouteFailed) +} + +// Kind implements stack.TransportError. +func (*icmpv4SourceRouteFailedSockError) Kind() stack.TransportErrorKind { + return stack.SourceRouteFailedTransportError +} + +var _ stack.TransportError = (*icmpv4SourceHostIsolatedSockError)(nil) + +// icmpv4SourceHostIsolatedSockError is an ICMPv4 Destination Unreachable error +// due to source host isolated (not on the network). +// +// +stateify savable +type icmpv4SourceHostIsolatedSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4SourceHostIsolatedSockError) Code() uint8 { + return uint8(header.ICMPv4SourceHostIsolated) +} + +// Kind implements stack.TransportError. +func (*icmpv4SourceHostIsolatedSockError) Kind() stack.TransportErrorKind { + return stack.SourceHostIsolatedTransportError +} + +var _ stack.TransportError = (*icmpv4DestinationHostUnknownSockError)(nil) + +// icmpv4DestinationHostUnknownSockError is an ICMPv4 Destination Unreachable +// error due to destination host unknown/down. +// +// +stateify savable +type icmpv4DestinationHostUnknownSockError struct { + icmpv4DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4DestinationHostUnknownSockError) Code() uint8 { + return uint8(header.ICMPv4DestinationHostUnknown) +} + +// Kind implements stack.TransportError. +func (*icmpv4DestinationHostUnknownSockError) Kind() stack.TransportErrorKind { + return stack.DestinationHostDownTransportError +} + +var _ stack.TransportError = (*icmpv4FragmentationNeededSockError)(nil) + +// icmpv4FragmentationNeededSockError is an ICMPv4 Destination Unreachable error +// due to fragmentation being required but the packet was set to not be +// fragmented. +// +// It indicates that a link exists on the path to the destination with an MTU +// that is too small to carry the packet. +// +// +stateify savable +type icmpv4FragmentationNeededSockError struct { + icmpv4DestinationUnreachableSockError + + mtu uint32 +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv4FragmentationNeededSockError) Code() uint8 { + return uint8(header.ICMPv4FragmentationNeeded) +} + +// Info implements tcpip.SockErrorCause. +func (e *icmpv4FragmentationNeededSockError) Info() uint32 { + return e.mtu +} + +// Kind implements stack.TransportError. +func (*icmpv4FragmentationNeededSockError) Kind() stack.TransportErrorKind { + return stack.PacketTooBigTransportError +} + +func (e *endpoint) checkLocalAddress(addr tcpip.Address) bool { + if e.nic.Spoofing() { + return true + } + + if addressEndpoint := e.AcquireAssignedAddress(addr, false, stack.NeverPrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + return true + } + return false +} + +// handleControl handles the case when an ICMP error packet contains the headers +// of the original packet that caused the ICMP one to be sent. This information +// is used to find out which transport endpoint must be notified about the ICMP +// packet. We only expect the payload, not the enclosing ICMP packet. +func (e *endpoint) handleControl(errInfo stack.TransportError, pkt *stack.PacketBuffer) { + h, ok := pkt.Data().PullUp(header.IPv4MinimumSize) + if !ok { + return + } + hdr := header.IPv4(h) + + // We don't use IsValid() here because ICMP only requires that the IP + // header plus 8 bytes of the transport header be included. So it's + // likely that it is truncated, which would cause IsValid to return + // false. + // + // Drop packet if it doesn't have the basic IPv4 header or if the + // original source address doesn't match an address we own. + srcAddr := hdr.SourceAddress() + if !e.checkLocalAddress(srcAddr) { + return + } + + hlen := int(hdr.HeaderLength()) + if pkt.Data().Size() < hlen || hdr.FragmentOffset() != 0 { + // We won't be able to handle this if it doesn't contain the + // full IPv4 header, or if it's a fragment not at offset 0 + // (because it won't have the transport header). + return + } + + // Keep needed information before trimming header. + p := hdr.TransportProtocol() + dstAddr := hdr.DestinationAddress() + // Skip the ip header, then deliver the error. + if _, ok := pkt.Data().Consume(hlen); !ok { + panic(fmt.Sprintf("could not consume the IP header of %d bytes", hlen)) + } + e.dispatcher.DeliverTransportError(srcAddr, dstAddr, ProtocolNumber, p, errInfo, pkt) +} + +func (e *endpoint) handleICMP(pkt *stack.PacketBuffer) { + received := e.stats.icmp.packetsReceived + h := header.ICMPv4(pkt.TransportHeader().Slice()) + if len(h) < header.ICMPv4MinimumSize { + received.invalid.Increment() + return + } + + // Only do in-stack processing if the checksum is correct. + if checksum.Checksum(h, pkt.Data().Checksum()) != 0xffff { + received.invalid.Increment() + // It's possible that a raw socket expects to receive this regardless + // of checksum errors. If it's an echo request we know it's safe because + // we are the only handler, however other types do not cope well with + // packets with checksum errors. + switch h.Type() { + case header.ICMPv4Echo: + e.dispatcher.DeliverTransportPacket(header.ICMPv4ProtocolNumber, pkt) + } + return + } + + iph := header.IPv4(pkt.NetworkHeader().Slice()) + var newOptions header.IPv4Options + if opts := iph.Options(); len(opts) != 0 { + // RFC 1122 section 3.2.2.6 (page 43) (and similar for other round trip + // type ICMP packets): + // If a Record Route and/or Time Stamp option is received in an + // ICMP Echo Request, this option (these options) SHOULD be + // updated to include the current host and included in the IP + // header of the Echo Reply message, without "truncation". + // Thus, the recorded route will be for the entire round trip. + // + // So we need to let the option processor know how it should handle them. + var op optionsUsage + if h.Type() == header.ICMPv4Echo { + op = &optionUsageEcho{} + } else { + op = &optionUsageReceive{} + } + var optProblem *header.IPv4OptParameterProblem + newOptions, _, optProblem = e.processIPOptions(pkt, opts, op) + if optProblem != nil { + if optProblem.NeedICMP { + _ = e.protocol.returnError(&icmpReasonParamProblem{ + pointer: optProblem.Pointer, + }, pkt, true /* deliveredLocally */) + e.stats.ip.MalformedPacketsReceived.Increment() + } + return + } + copied := copy(opts, newOptions) + if copied != len(newOptions) { + panic(fmt.Sprintf("copied %d bytes of new options, expected %d bytes", copied, len(newOptions))) + } + for i := copied; i < len(opts); i++ { + // Pad with 0 (EOL). RFC 791 page 23 says "The padding is zero". + opts[i] = byte(header.IPv4OptionListEndType) + } + } + + // TODO(b/112892170): Meaningfully handle all ICMP types. + switch h.Type() { + case header.ICMPv4Echo: + received.echoRequest.Increment() + e.dispatcher.DeliverTransportPacket(header.ICMPv4ProtocolNumber, pkt) + case header.ICMPv4EchoReply: + received.echoReply.Increment() + e.dispatcher.DeliverTransportPacket(header.ICMPv4ProtocolNumber, pkt) + case header.ICMPv4DstUnreachable: + received.dstUnreachable.Increment() + + mtu := h.MTU() + code := h.Code() + switch code { + case header.ICMPv4NetUnreachable, + header.ICMPv4DestinationNetworkUnknown, + header.ICMPv4NetUnreachableForTos, + header.ICMPv4NetProhibited: + e.handleControl(&icmpv4DestinationNetUnreachableSockError{}, pkt) + case header.ICMPv4HostUnreachable, + header.ICMPv4HostProhibited, + header.ICMPv4AdminProhibited, + header.ICMPv4HostUnreachableForTos, + header.ICMPv4HostPrecedenceViolation, + header.ICMPv4PrecedenceCutInEffect: + e.handleControl(&icmpv4DestinationHostUnreachableSockError{}, pkt) + case header.ICMPv4PortUnreachable: + e.handleControl(&icmpv4DestinationPortUnreachableSockError{}, pkt) + case header.ICMPv4FragmentationNeeded: + networkMTU, err := calculateNetworkMTU(uint32(mtu), header.IPv4MinimumSize) + if err != nil { + networkMTU = 0 + } + e.handleControl(&icmpv4FragmentationNeededSockError{mtu: networkMTU}, pkt) + case header.ICMPv4ProtoUnreachable: + e.handleControl(&icmpv4DestinationProtoUnreachableSockError{}, pkt) + case header.ICMPv4SourceRouteFailed: + e.handleControl(&icmpv4SourceRouteFailedSockError{}, pkt) + case header.ICMPv4SourceHostIsolated: + e.handleControl(&icmpv4SourceHostIsolatedSockError{}, pkt) + case header.ICMPv4DestinationHostUnknown: + e.handleControl(&icmpv4DestinationHostUnknownSockError{}, pkt) + } + case header.ICMPv4SrcQuench: + received.srcQuench.Increment() + + case header.ICMPv4Redirect: + received.redirect.Increment() + + case header.ICMPv4TimeExceeded: + received.timeExceeded.Increment() + + case header.ICMPv4ParamProblem: + received.paramProblem.Increment() + + case header.ICMPv4Timestamp: + received.timestamp.Increment() + + case header.ICMPv4TimestampReply: + received.timestampReply.Increment() + + case header.ICMPv4InfoRequest: + received.infoRequest.Increment() + + case header.ICMPv4InfoReply: + received.infoReply.Increment() + + default: + received.invalid.Increment() + } +} + +// ======= ICMP Error packet generation ========= + +// icmpReason is a marker interface for IPv4 specific ICMP errors. +type icmpReason interface { + isICMPReason() +} + +// icmpReasonNetworkProhibited is an error where the destination network is +// prohibited. +type icmpReasonNetworkProhibited struct{} + +func (*icmpReasonNetworkProhibited) isICMPReason() {} + +// icmpReasonHostProhibited is an error where the destination host is +// prohibited. +type icmpReasonHostProhibited struct{} + +func (*icmpReasonHostProhibited) isICMPReason() {} + +// icmpReasonAdministrativelyProhibited is an error where the destination is +// administratively prohibited. +type icmpReasonAdministrativelyProhibited struct{} + +func (*icmpReasonAdministrativelyProhibited) isICMPReason() {} + +// icmpReasonPortUnreachable is an error where the transport protocol has no +// listener and no alternative means to inform the sender. +type icmpReasonPortUnreachable struct{} + +func (*icmpReasonPortUnreachable) isICMPReason() {} + +// icmpReasonProtoUnreachable is an error where the transport protocol is +// not supported. +type icmpReasonProtoUnreachable struct{} + +func (*icmpReasonProtoUnreachable) isICMPReason() {} + +// icmpReasonTTLExceeded is an error where a packet's time to live exceeded in +// transit to its final destination, as per RFC 792 page 6, Time Exceeded +// Message. +type icmpReasonTTLExceeded struct{} + +func (*icmpReasonTTLExceeded) isICMPReason() {} + +// icmpReasonReassemblyTimeout is an error where insufficient fragments are +// received to complete reassembly of a packet within a configured time after +// the reception of the first-arriving fragment of that packet. +type icmpReasonReassemblyTimeout struct{} + +func (*icmpReasonReassemblyTimeout) isICMPReason() {} + +// icmpReasonParamProblem is an error to use to request a Parameter Problem +// message to be sent. +type icmpReasonParamProblem struct { + pointer byte +} + +func (*icmpReasonParamProblem) isICMPReason() {} + +// icmpReasonNetworkUnreachable is an error in which the network specified in +// the internet destination field of the datagram is unreachable. +type icmpReasonNetworkUnreachable struct{} + +func (*icmpReasonNetworkUnreachable) isICMPReason() {} + +// icmpReasonFragmentationNeeded is an error where a packet requires +// fragmentation while also having the Don't Fragment flag set, as per RFC 792 +// page 3, Destination Unreachable Message. +type icmpReasonFragmentationNeeded struct{} + +func (*icmpReasonFragmentationNeeded) isICMPReason() {} + +// icmpReasonHostUnreachable is an error in which the host specified in the +// internet destination field of the datagram is unreachable. +type icmpReasonHostUnreachable struct{} + +func (*icmpReasonHostUnreachable) isICMPReason() {} + +// returnError takes an error descriptor and generates the appropriate ICMP +// error packet for IPv4 and sends it back to the remote device that sent +// the problematic packet. It incorporates as much of that packet as +// possible as well as any error metadata as is available. returnError +// expects pkt to hold a valid IPv4 packet as per the wire format. +func (p *protocol) returnError(reason icmpReason, pkt *stack.PacketBuffer, deliveredLocally bool) tcpip.Error { + origIPHdr := header.IPv4(pkt.NetworkHeader().Slice()) + origIPHdrSrc := origIPHdr.SourceAddress() + origIPHdrDst := origIPHdr.DestinationAddress() + + // We check we are responding only when we are allowed to. + // See RFC 1812 section 4.3.2.7 (shown below). + // + // ========= + // 4.3.2.7 When Not to Send ICMP Errors + // + // An ICMP error message MUST NOT be sent as the result of receiving: + // + // o An ICMP error message, or + // + // o A packet which fails the IP header validation tests described in + // Section [5.2.2] (except where that section specifically permits + // the sending of an ICMP error message), or + // + // o A packet destined to an IP broadcast or IP multicast address, or + // + // o A packet sent as a Link Layer broadcast or multicast, or + // + // o Any fragment of a datagram other then the first fragment (i.e., a + // packet for which the fragment offset in the IP header is nonzero). + // + // TODO(gvisor.dev/issues/4058): Make sure we don't send ICMP errors in + // response to a non-initial fragment, but it currently can not happen. + if pkt.NetworkPacketInfo.LocalAddressBroadcast || header.IsV4MulticastAddress(origIPHdrDst) || origIPHdrSrc == header.IPv4Any { + return nil + } + + // If the packet wasn't delivered locally, do not use the packet's destination + // address as the response's source address as we should not not own the + // destination address of a packet we are forwarding. + localAddr := origIPHdrDst + if !deliveredLocally { + localAddr = tcpip.Address{} + } + + // Even if we were able to receive a packet from some remote, we may not have + // a route to it - the remote may be blocked via routing rules. We must always + // consult our routing table and find a route to the remote before sending any + // packet. + route, err := p.stack.FindRoute(pkt.NICID, localAddr, origIPHdrSrc, ProtocolNumber, false /* multicastLoop */) + if err != nil { + return err + } + defer route.Release() + + p.mu.Lock() + // We retrieve an endpoint using the newly constructed route's NICID rather + // than the packet's NICID. The packet's NICID corresponds to the NIC on + // which it arrived, which isn't necessarily the same as the NIC on which it + // will be transmitted. On the other hand, the route's NIC *is* guaranteed + // to be the NIC on which the packet will be transmitted. + netEP, ok := p.eps[route.NICID()] + p.mu.Unlock() + if !ok { + return &tcpip.ErrNotConnected{} + } + + transportHeader := pkt.TransportHeader().Slice() + + // Don't respond to icmp error packets. + if origIPHdr.Protocol() == uint8(header.ICMPv4ProtocolNumber) { + // We need to decide to explicitly name the packets we can respond to or + // the ones we can not respond to. The decision is somewhat arbitrary and + // if problems arise this could be reversed. It was judged less of a breach + // of protocol to not respond to unknown non-error packets than to respond + // to unknown error packets so we take the first approach. + if len(transportHeader) < header.ICMPv4MinimumSize { + // The packet is malformed. + return nil + } + switch header.ICMPv4(transportHeader).Type() { + case + header.ICMPv4EchoReply, + header.ICMPv4Echo, + header.ICMPv4Timestamp, + header.ICMPv4TimestampReply, + header.ICMPv4InfoRequest, + header.ICMPv4InfoReply: + default: + // Assume any type we don't know about may be an error type. + return nil + } + } + + sent := netEP.stats.icmp.packetsSent + icmpType, icmpCode, counter, pointer := func() (header.ICMPv4Type, header.ICMPv4Code, tcpip.MultiCounterStat, byte) { + switch reason := reason.(type) { + case *icmpReasonNetworkProhibited: + return header.ICMPv4DstUnreachable, header.ICMPv4NetProhibited, sent.dstUnreachable, 0 + case *icmpReasonHostProhibited: + return header.ICMPv4DstUnreachable, header.ICMPv4HostProhibited, sent.dstUnreachable, 0 + case *icmpReasonAdministrativelyProhibited: + return header.ICMPv4DstUnreachable, header.ICMPv4AdminProhibited, sent.dstUnreachable, 0 + case *icmpReasonPortUnreachable: + return header.ICMPv4DstUnreachable, header.ICMPv4PortUnreachable, sent.dstUnreachable, 0 + case *icmpReasonProtoUnreachable: + return header.ICMPv4DstUnreachable, header.ICMPv4ProtoUnreachable, sent.dstUnreachable, 0 + case *icmpReasonNetworkUnreachable: + return header.ICMPv4DstUnreachable, header.ICMPv4NetUnreachable, sent.dstUnreachable, 0 + case *icmpReasonHostUnreachable: + return header.ICMPv4DstUnreachable, header.ICMPv4HostUnreachable, sent.dstUnreachable, 0 + case *icmpReasonFragmentationNeeded: + return header.ICMPv4DstUnreachable, header.ICMPv4FragmentationNeeded, sent.dstUnreachable, 0 + case *icmpReasonTTLExceeded: + return header.ICMPv4TimeExceeded, header.ICMPv4TTLExceeded, sent.timeExceeded, 0 + case *icmpReasonReassemblyTimeout: + return header.ICMPv4TimeExceeded, header.ICMPv4ReassemblyTimeout, sent.timeExceeded, 0 + case *icmpReasonParamProblem: + return header.ICMPv4ParamProblem, header.ICMPv4UnusedCode, sent.paramProblem, reason.pointer + default: + panic(fmt.Sprintf("unsupported ICMP type %T", reason)) + } + }() + + if !p.allowICMPReply(icmpType, icmpCode) { + sent.rateLimited.Increment() + return nil + } + + // Now work out how much of the triggering packet we should return. + // As per RFC 1812 Section 4.3.2.3 + // + // ICMP datagram SHOULD contain as much of the original + // datagram as possible without the length of the ICMP + // datagram exceeding 576 bytes. + // + // NOTE: The above RFC referenced is different from the original + // recommendation in RFC 1122 and RFC 792 where it mentioned that at + // least 8 bytes of the payload must be included. Today linux and other + // systems implement the RFC 1812 definition and not the original + // requirement. We treat 8 bytes as the minimum but will try send more. + mtu := int(route.MTU()) + const maxIPData = header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize + if mtu > maxIPData { + mtu = maxIPData + } + available := mtu - header.ICMPv4MinimumSize + + if available < len(origIPHdr)+header.ICMPv4MinimumErrorPayloadSize { + return nil + } + + payloadLen := len(origIPHdr) + len(transportHeader) + pkt.Data().Size() + if payloadLen > available { + payloadLen = available + } + + // The buffers used by pkt may be used elsewhere in the system. + // For example, an AF_RAW or AF_PACKET socket may use what the transport + // protocol considers an unreachable destination. Thus we deep copy pkt to + // prevent multiple ownership and SR errors. The new copy is a vectorized + // view with the entire incoming IP packet reassembled and truncated as + // required. This is now the payload of the new ICMP packet and no longer + // considered a packet in its own right. + + payload := buffer.MakeWithView(pkt.NetworkHeader().View()) + payload.Append(pkt.TransportHeader().View()) + if dataCap := payloadLen - int(payload.Size()); dataCap > 0 { + buf := pkt.Data().ToBuffer() + buf.Truncate(int64(dataCap)) + payload.Merge(&buf) + } else { + payload.Truncate(int64(payloadLen)) + } + + icmpPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(route.MaxHeaderLength()) + header.ICMPv4MinimumSize, + Payload: payload, + }) + defer icmpPkt.DecRef() + + icmpPkt.TransportProtocolNumber = header.ICMPv4ProtocolNumber + + icmpHdr := header.ICMPv4(icmpPkt.TransportHeader().Push(header.ICMPv4MinimumSize)) + icmpHdr.SetCode(icmpCode) + icmpHdr.SetType(icmpType) + icmpHdr.SetPointer(pointer) + icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, icmpPkt.Data().Checksum())) + + if err := route.WritePacket( + stack.NetworkHeaderParams{ + Protocol: header.ICMPv4ProtocolNumber, + TTL: route.DefaultTTL(), + TOS: stack.DefaultTOS, + }, + icmpPkt, + ); err != nil { + sent.dropped.Increment() + return err + } + counter.Increment() + return nil +} + +// OnReassemblyTimeout implements fragmentation.TimeoutHandler. +func (p *protocol) OnReassemblyTimeout(pkt *stack.PacketBuffer) { + // OnReassemblyTimeout sends a Time Exceeded Message, as per RFC 792: + // + // If a host reassembling a fragmented datagram cannot complete the + // reassembly due to missing fragments within its time limit it discards the + // datagram, and it may send a time exceeded message. + // + // If fragment zero is not available then no time exceeded need be sent at + // all. + if pkt != nil { + p.returnError(&icmpReasonReassemblyTimeout{}, pkt, true /* deliveredLocally */) + } +} diff --git a/pkg/tcpip/network/ipv4/igmp.go b/pkg/tcpip/network/ipv4/igmp.go new file mode 100644 index 0000000..ebdb8c0 --- /dev/null +++ b/pkg/tcpip/network/ipv4/igmp.go @@ -0,0 +1,654 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv4 + +import ( + "fmt" + "math" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // v1RouterPresentTimeout from RFC 2236 Section 8.11, Page 18 + // See note on igmpState.igmpV1Present for more detail. + v1RouterPresentTimeout = 400 * time.Second + + // v1MaxRespTime from RFC 2236 Section 4, Page 5. "The IGMPv1 router + // will send General Queries with the Max Response Time set to 0. This MUST + // be interpreted as a value of 100 (10 seconds)." + // + // Note that the Max Response Time field is a value in units of deciseconds. + v1MaxRespTime = 10 * time.Second + + // UnsolicitedReportIntervalMax is the maximum delay between sending + // unsolicited IGMP reports. + // + // Obtained from RFC 2236 Section 8.10, Page 19. + UnsolicitedReportIntervalMax = 10 * time.Second +) + +type protocolMode int + +const ( + protocolModeV2OrV3 protocolMode = iota + protocolModeV1 + // protocolModeV1Compatibility is for maintaining compatibility with IGMPv1 + // Routers. + // + // Per RFC 2236 Section 4 Page 6: "The IGMPv1 router expects Version 1 + // Membership Reports in response to its Queries, and will not pay + // attention to Version 2 Membership Reports. Therefore, a state variable + // MUST be kept for each interface, describing whether the multicast + // Querier on that interface is running IGMPv1 or IGMPv2. This variable + // MUST be based upon whether or not an IGMPv1 query was heard in the last + // [Version 1 Router Present Timeout] seconds". + protocolModeV1Compatibility +) + +// IGMPVersion is the forced version of IGMP. +type IGMPVersion int + +const ( + _ IGMPVersion = iota + // IGMPVersion1 indicates IGMPv1. + IGMPVersion1 + // IGMPVersion2 indicates IGMPv2. Note that IGMP may still fallback to V1 + // compatibility mode as required by IGMPv2. + IGMPVersion2 + // IGMPVersion3 indicates IGMPv3. Note that IGMP may still fallback to V2 + // compatibility mode as required by IGMPv3. + IGMPVersion3 +) + +// IGMPEndpoint is a network endpoint that supports IGMP. +type IGMPEndpoint interface { + // SetIGMPVersion sets the IGMP version. + // + // Returns the previous IGMP version. + SetIGMPVersion(IGMPVersion) IGMPVersion + + // GetIGMPVersion returns the IGMP version. + GetIGMPVersion() IGMPVersion +} + +// IGMPOptions holds options for IGMP. +// +// +stateify savable +type IGMPOptions struct { + // Enabled indicates whether IGMP will be performed. + // + // When enabled, IGMP may transmit IGMP report and leave messages when + // joining and leaving multicast groups respectively, and handle incoming + // IGMP packets. + // + // This field is ignored and is always assumed to be false for interfaces + // without neighbouring nodes (e.g. loopback). + Enabled bool +} + +var _ ip.MulticastGroupProtocol = (*igmpState)(nil) + +// igmpState is the per-interface IGMP state. +// +// igmpState.init() MUST be called after creating an IGMP state. +// +// +stateify savable +type igmpState struct { + // The IPv4 endpoint this igmpState is for. + ep *endpoint + + genericMulticastProtocol ip.GenericMulticastProtocolState + + // mode is used to configure the version of IGMP to perform. + mode protocolMode + + // igmpV1Job is scheduled when this interface receives an IGMPv1 style + // message, upon expiration the igmpV1Present flag is cleared. + // igmpV1Job may not be nil once igmpState is initialized. + igmpV1Job *tcpip.Job +} + +// Enabled implements ip.MulticastGroupProtocol. +func (igmp *igmpState) Enabled() bool { + // No need to perform IGMP on loopback interfaces since they don't have + // neighbouring nodes. + return igmp.ep.protocol.options.IGMP.Enabled && !igmp.ep.nic.IsLoopback() && igmp.ep.Enabled() +} + +// SendReport implements ip.MulticastGroupProtocol. +// +// +checklocksread:igmp.ep.mu +func (igmp *igmpState) SendReport(groupAddress tcpip.Address) (bool, tcpip.Error) { + igmpType := header.IGMPv2MembershipReport + switch igmp.mode { + case protocolModeV2OrV3: + case protocolModeV1, protocolModeV1Compatibility: + igmpType = header.IGMPv1MembershipReport + default: + panic(fmt.Sprintf("unrecognized mode = %d", igmp.mode)) + } + return igmp.writePacket(groupAddress, groupAddress, igmpType) +} + +// SendLeave implements ip.MulticastGroupProtocol. +// +// +checklocksread:igmp.ep.mu +func (igmp *igmpState) SendLeave(groupAddress tcpip.Address) tcpip.Error { + // As per RFC 2236 Section 6, Page 8: "If the interface state says the + // Querier is running IGMPv1, this action SHOULD be skipped. If the flag + // saying we were the last host to report is cleared, this action MAY be + // skipped." + switch igmp.mode { + case protocolModeV2OrV3: + _, err := igmp.writePacket(header.IPv4AllRoutersGroup, groupAddress, header.IGMPLeaveGroup) + return err + case protocolModeV1, protocolModeV1Compatibility: + return nil + default: + panic(fmt.Sprintf("unrecognized mode = %d", igmp.mode)) + } +} + +// ShouldPerformProtocol implements ip.MulticastGroupProtocol. +func (igmp *igmpState) ShouldPerformProtocol(groupAddress tcpip.Address) bool { + // As per RFC 2236 section 6 page 10, + // + // The all-systems group (address 224.0.0.1) is handled as a special + // case. The host starts in Idle Member state for that group on every + // interface, never transitions to another state, and never sends a + // report for that group. + return groupAddress != header.IPv4AllSystems +} + +type igmpv3ReportBuilder struct { + igmp *igmpState + + records []header.IGMPv3ReportGroupAddressRecordSerializer +} + +// AddRecord implements ip.MulticastGroupProtocolV2ReportBuilder. +func (b *igmpv3ReportBuilder) AddRecord(genericRecordType ip.MulticastGroupProtocolV2ReportRecordType, groupAddress tcpip.Address) { + var recordType header.IGMPv3ReportRecordType + switch genericRecordType { + case ip.MulticastGroupProtocolV2ReportRecordModeIsInclude: + recordType = header.IGMPv3ReportRecordModeIsInclude + case ip.MulticastGroupProtocolV2ReportRecordModeIsExclude: + recordType = header.IGMPv3ReportRecordModeIsExclude + case ip.MulticastGroupProtocolV2ReportRecordChangeToIncludeMode: + recordType = header.IGMPv3ReportRecordChangeToIncludeMode + case ip.MulticastGroupProtocolV2ReportRecordChangeToExcludeMode: + recordType = header.IGMPv3ReportRecordChangeToExcludeMode + case ip.MulticastGroupProtocolV2ReportRecordAllowNewSources: + recordType = header.IGMPv3ReportRecordAllowNewSources + case ip.MulticastGroupProtocolV2ReportRecordBlockOldSources: + recordType = header.IGMPv3ReportRecordBlockOldSources + default: + panic(fmt.Sprintf("unrecognied genericRecordType = %d", genericRecordType)) + } + + b.records = append(b.records, header.IGMPv3ReportGroupAddressRecordSerializer{ + RecordType: recordType, + GroupAddress: groupAddress, + Sources: nil, + }) +} + +// Send implements ip.MulticastGroupProtocolV2ReportBuilder. +// +// +checklocksread:b.igmp.ep.mu +func (b *igmpv3ReportBuilder) Send() (sent bool, err tcpip.Error) { + if len(b.records) == 0 { + return false, err + } + + options := header.IPv4OptionsSerializer{ + &header.IPv4SerializableRouterAlertOption{}, + } + mtu := int(b.igmp.ep.MTU()) - int(options.Length()) + + allSentWithSpecifiedAddress := true + var firstErr tcpip.Error + for records := b.records; len(records) != 0; { + spaceLeft := mtu + maxRecords := 0 + + for ; maxRecords < len(records); maxRecords++ { + tmp := spaceLeft - records[maxRecords].Length() + if tmp > 0 { + spaceLeft = tmp + } else { + break + } + } + + serializer := header.IGMPv3ReportSerializer{Records: records[:maxRecords]} + records = records[maxRecords:] + + icmpView := buffer.NewViewSize(serializer.Length()) + serializer.SerializeInto(icmpView.AsSlice()) + if sentWithSpecifiedAddress, err := b.igmp.writePacketInner( + icmpView, + b.igmp.ep.stats.igmp.packetsSent.v3MembershipReport, + options, + header.IGMPv3RoutersAddress, + ); err != nil { + if firstErr != nil { + firstErr = nil + } + allSentWithSpecifiedAddress = false + } else if !sentWithSpecifiedAddress { + allSentWithSpecifiedAddress = false + } + } + + return allSentWithSpecifiedAddress, firstErr +} + +// NewReportV2Builder implements ip.MulticastGroupProtocol. +func (igmp *igmpState) NewReportV2Builder() ip.MulticastGroupProtocolV2ReportBuilder { + return &igmpv3ReportBuilder{igmp: igmp} +} + +// V2QueryMaxRespCodeToV2Delay implements ip.MulticastGroupProtocol. +func (*igmpState) V2QueryMaxRespCodeToV2Delay(code uint16) time.Duration { + if code > math.MaxUint8 { + panic(fmt.Sprintf("got IGMPv3 MaxRespCode = %d, want <= %d", code, math.MaxUint8)) + } + return header.IGMPv3MaximumResponseDelay(uint8(code)) +} + +// V2QueryMaxRespCodeToV1Delay implements ip.MulticastGroupProtocol. +func (*igmpState) V2QueryMaxRespCodeToV1Delay(code uint16) time.Duration { + return time.Duration(code) * time.Millisecond +} + +// init sets up an igmpState struct, and is required to be called before using +// a new igmpState. +// +// Must only be called once for the lifetime of igmp. +func (igmp *igmpState) init(ep *endpoint) { + igmp.ep = ep + igmp.genericMulticastProtocol.Init(&ep.mu, ip.GenericMulticastProtocolOptions{ + Rand: ep.protocol.stack.InsecureRNG(), + Clock: ep.protocol.stack.Clock(), + Protocol: igmp, + MaxUnsolicitedReportDelay: UnsolicitedReportIntervalMax, + }) + // As per RFC 2236 Page 9 says "No IGMPv1 Router Present ... is + // the initial state. + igmp.mode = protocolModeV2OrV3 + igmp.igmpV1Job = tcpip.NewJob(ep.protocol.stack.Clock(), &ep.mu, func() { + igmp.mode = protocolModeV2OrV3 + }) +} + +// +checklocks:igmp.ep.mu +func (igmp *igmpState) isSourceIPValidLocked(src tcpip.Address, messageType header.IGMPType) bool { + if messageType == header.IGMPMembershipQuery { + // RFC 2236 does not require the IGMP implementation to check the source IP + // for Membership Query messages. + return true + } + + // As per RFC 2236 section 10, + // + // Ignore the Report if you cannot identify the source address of the + // packet as belonging to a subnet assigned to the interface on which the + // packet was received. + // + // Ignore the Leave message if you cannot identify the source address of + // the packet as belonging to a subnet assigned to the interface on which + // the packet was received. + // + // Note: this rule applies to both V1 and V2 Membership Reports. + var isSourceIPValid bool + igmp.ep.addressableEndpointState.ForEachPrimaryEndpoint(func(addressEndpoint stack.AddressEndpoint) bool { + if subnet := addressEndpoint.Subnet(); subnet.Contains(src) { + isSourceIPValid = true + return false + } + return true + }) + + return isSourceIPValid +} + +// +checklocks:igmp.ep.mu +func (igmp *igmpState) isPacketValidLocked(pkt *stack.PacketBuffer, messageType header.IGMPType, hasRouterAlertOption bool) bool { + // We can safely assume that the IP header is valid if we got this far. + iph := header.IPv4(pkt.NetworkHeader().Slice()) + + // As per RFC 2236 section 2, + // + // All IGMP messages described in this document are sent with IP TTL 1, and + // contain the IP Router Alert option [RFC 2113] in their IP header. + if !hasRouterAlertOption || iph.TTL() != header.IGMPTTL { + return false + } + + return igmp.isSourceIPValidLocked(iph.SourceAddress(), messageType) +} + +// handleIGMP handles an IGMP packet. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) handleIGMP(pkt *stack.PacketBuffer, hasRouterAlertOption bool) { + received := igmp.ep.stats.igmp.packetsReceived + hdr, ok := pkt.Data().PullUp(pkt.Data().Size()) + if !ok { + received.invalid.Increment() + return + } + h := header.IGMP(hdr) + if len(h) < header.IGMPMinimumSize { + received.invalid.Increment() + return + } + + // As per RFC 1071 section 1.3, + // + // To check a checksum, the 1's complement sum is computed over the + // same set of octets, including the checksum field. If the result + // is all 1 bits (-0 in 1's complement arithmetic), the check + // succeeds. + if pkt.Data().Checksum() != 0xFFFF { + received.checksumErrors.Increment() + return + } + + isValid := func(minimumSize int) bool { + return len(hdr) >= minimumSize && igmp.isPacketValidLocked(pkt, h.Type(), hasRouterAlertOption) + } + + switch h.Type() { + case header.IGMPMembershipQuery: + received.membershipQuery.Increment() + if len(h) >= header.IGMPv3QueryMinimumSize { + if isValid(header.IGMPv3QueryMinimumSize) { + igmp.handleMembershipQueryV3(header.IGMPv3Query(h)) + } else { + received.invalid.Increment() + } + return + } else if !isValid(header.IGMPQueryMinimumSize) { + received.invalid.Increment() + return + } + igmp.handleMembershipQuery(h.GroupAddress(), h.MaxRespTime()) + case header.IGMPv1MembershipReport: + received.v1MembershipReport.Increment() + if !isValid(header.IGMPReportMinimumSize) { + received.invalid.Increment() + return + } + igmp.handleMembershipReport(h.GroupAddress()) + case header.IGMPv2MembershipReport: + received.v2MembershipReport.Increment() + if !isValid(header.IGMPReportMinimumSize) { + received.invalid.Increment() + return + } + igmp.handleMembershipReport(h.GroupAddress()) + case header.IGMPLeaveGroup: + received.leaveGroup.Increment() + if !isValid(header.IGMPLeaveMessageMinimumSize) { + received.invalid.Increment() + return + } + // As per RFC 2236 Section 6, Page 7: "IGMP messages other than Query or + // Report, are ignored in all states" + + default: + // As per RFC 2236 Section 2.1 Page 3: "Unrecognized message types should + // be silently ignored. New message types may be used by newer versions of + // IGMP, by multicast routing protocols, or other uses." + received.unrecognized.Increment() + } +} + +func (igmp *igmpState) resetV1Present() { + igmp.igmpV1Job.Cancel() + switch igmp.mode { + case protocolModeV2OrV3, protocolModeV1: + case protocolModeV1Compatibility: + igmp.mode = protocolModeV2OrV3 + default: + panic(fmt.Sprintf("unrecognized mode = %d", igmp.mode)) + } +} + +// handleMembershipQuery handles a membership query. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) handleMembershipQuery(groupAddress tcpip.Address, maxRespTime time.Duration) { + // As per RFC 2236 Section 6, Page 10: If the maximum response time is zero + // then change the state to note that an IGMPv1 router is present and + // schedule the query received Job. + if maxRespTime == 0 && igmp.Enabled() { + switch igmp.mode { + case protocolModeV2OrV3, protocolModeV1Compatibility: + igmp.igmpV1Job.Cancel() + igmp.igmpV1Job.Schedule(v1RouterPresentTimeout) + igmp.mode = protocolModeV1Compatibility + case protocolModeV1: + default: + panic(fmt.Sprintf("unrecognized mode = %d", igmp.mode)) + } + + maxRespTime = v1MaxRespTime + } + + igmp.genericMulticastProtocol.HandleQueryLocked(groupAddress, maxRespTime) +} + +// handleMembershipQueryV3 handles a membership query. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) handleMembershipQueryV3(igmpHdr header.IGMPv3Query) { + sources, ok := igmpHdr.Sources() + if !ok { + return + } + + igmp.genericMulticastProtocol.HandleQueryV2Locked( + igmpHdr.GroupAddress(), + uint16(igmpHdr.MaximumResponseCode()), + sources, + igmpHdr.QuerierRobustnessVariable(), + igmpHdr.QuerierQueryInterval(), + ) +} + +// handleMembershipReport handles a membership report. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) handleMembershipReport(groupAddress tcpip.Address) { + igmp.genericMulticastProtocol.HandleReportLocked(groupAddress) +} + +// writePacket assembles and sends an IGMP packet. +// +// +checklocksread:igmp.ep.mu +func (igmp *igmpState) writePacket(destAddress tcpip.Address, groupAddress tcpip.Address, igmpType header.IGMPType) (bool, tcpip.Error) { + igmpView := buffer.NewViewSize(header.IGMPReportMinimumSize) + igmpData := header.IGMP(igmpView.AsSlice()) + igmpData.SetType(igmpType) + igmpData.SetGroupAddress(groupAddress) + igmpData.SetChecksum(header.IGMPCalculateChecksum(igmpData)) + + var reportType tcpip.MultiCounterStat + sentStats := igmp.ep.stats.igmp.packetsSent + switch igmpType { + case header.IGMPv1MembershipReport: + reportType = sentStats.v1MembershipReport + case header.IGMPv2MembershipReport: + reportType = sentStats.v2MembershipReport + case header.IGMPLeaveGroup: + reportType = sentStats.leaveGroup + default: + panic(fmt.Sprintf("unrecognized igmp type = %d", igmpType)) + } + + return igmp.writePacketInner( + igmpView, + reportType, + header.IPv4OptionsSerializer{ + &header.IPv4SerializableRouterAlertOption{}, + }, + destAddress, + ) +} + +// +checklocksread:igmp.ep.mu +func (igmp *igmpState) writePacketInner(buf *buffer.View, reportStat tcpip.MultiCounterStat, options header.IPv4OptionsSerializer, destAddress tcpip.Address) (bool, tcpip.Error) { + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(igmp.ep.MaxHeaderLength()), + Payload: buffer.MakeWithView(buf), + }) + defer pkt.DecRef() + + addressEndpoint := igmp.ep.acquireOutgoingPrimaryAddressRLocked(destAddress, tcpip.Address{} /* srcHint */, false /* allowExpired */) + if addressEndpoint == nil { + return false, nil + } + localAddr := addressEndpoint.AddressWithPrefix().Address + addressEndpoint.DecRef() + addressEndpoint = nil + if err := igmp.ep.addIPHeader(localAddr, destAddress, pkt, stack.NetworkHeaderParams{ + Protocol: header.IGMPProtocolNumber, + TTL: header.IGMPTTL, + TOS: stack.DefaultTOS, + }, options); err != nil { + panic(fmt.Sprintf("failed to add IP header: %s", err)) + } + + sentStats := igmp.ep.stats.igmp.packetsSent + if err := igmp.ep.nic.WritePacketToRemote(header.EthernetAddressFromMulticastIPv4Address(destAddress), pkt); err != nil { + sentStats.dropped.Increment() + return false, err + } + reportStat.Increment() + return true, nil +} + +// joinGroup handles adding a new group to the membership map, setting up the +// IGMP state for the group, and sending and scheduling the required +// messages. +// +// If the group already exists in the membership map, returns +// *tcpip.ErrDuplicateAddress. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) joinGroup(groupAddress tcpip.Address) { + igmp.genericMulticastProtocol.JoinGroupLocked(groupAddress) +} + +// isInGroup returns true if the specified group has been joined locally. +// +// +checklocksread:igmp.ep.mu +func (igmp *igmpState) isInGroup(groupAddress tcpip.Address) bool { + return igmp.genericMulticastProtocol.IsLocallyJoinedRLocked(groupAddress) +} + +// leaveGroup handles removing the group from the membership map, cancels any +// delay timers associated with that group, and sends the Leave Group message +// if required. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) leaveGroup(groupAddress tcpip.Address) tcpip.Error { + // LeaveGroup returns false only if the group was not joined. + if igmp.genericMulticastProtocol.LeaveGroupLocked(groupAddress) { + return nil + } + + return &tcpip.ErrBadLocalAddress{} +} + +// softLeaveAll leaves all groups from the perspective of IGMP, but remains +// joined locally. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) softLeaveAll() { + igmp.genericMulticastProtocol.MakeAllNonMemberLocked() +} + +// initializeAll attempts to initialize the IGMP state for each group that has +// been joined locally. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) initializeAll() { + igmp.genericMulticastProtocol.InitializeGroupsLocked() +} + +// sendQueuedReports attempts to send any reports that are queued for sending. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) sendQueuedReports() { + igmp.genericMulticastProtocol.SendQueuedReportsLocked() +} + +// setVersion sets the IGMP version. +// +// +checklocks:igmp.ep.mu +func (igmp *igmpState) setVersion(v IGMPVersion) IGMPVersion { + prev := igmp.mode + igmp.igmpV1Job.Cancel() + + var prevGenericModeV1 bool + switch v { + case IGMPVersion3: + prevGenericModeV1 = igmp.genericMulticastProtocol.SetV1ModeLocked(false) + igmp.mode = protocolModeV2OrV3 + case IGMPVersion2: + // IGMPv1 and IGMPv2 map to V1 of the generic multicast protocol. + prevGenericModeV1 = igmp.genericMulticastProtocol.SetV1ModeLocked(true) + igmp.mode = protocolModeV2OrV3 + case IGMPVersion1: + // IGMPv1 and IGMPv2 map to V1 of the generic multicast protocol. + prevGenericModeV1 = igmp.genericMulticastProtocol.SetV1ModeLocked(true) + igmp.mode = protocolModeV1 + default: + panic(fmt.Sprintf("unrecognized version = %d", v)) + } + + return toIGMPVersion(prev, prevGenericModeV1) +} + +func toIGMPVersion(mode protocolMode, genericV1 bool) IGMPVersion { + switch mode { + case protocolModeV2OrV3, protocolModeV1Compatibility: + if genericV1 { + return IGMPVersion2 + } + return IGMPVersion3 + case protocolModeV1: + return IGMPVersion1 + default: + panic(fmt.Sprintf("unrecognized mode = %d", mode)) + } +} + +// getVersion returns the IGMP version. +// +// +checklocksread:igmp.ep.mu +func (igmp *igmpState) getVersion() IGMPVersion { + return toIGMPVersion(igmp.mode, igmp.genericMulticastProtocol.GetV1ModeLocked()) +} diff --git a/pkg/tcpip/network/ipv4/ipv4.go b/pkg/tcpip/network/ipv4/ipv4.go new file mode 100644 index 0000000..4a9402b --- /dev/null +++ b/pkg/tcpip/network/ipv4/ipv4.go @@ -0,0 +1,2405 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ipv4 contains the implementation of the ipv4 network protocol. +package ipv4 + +import ( + "fmt" + "math" + "reflect" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" + "github.com/sagernet/gvisor/pkg/tcpip/network/hash" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/fragmentation" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/multicast" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // ReassembleTimeout is the time a packet stays in the reassembly + // system before being evicted. + // As per RFC 791 section 3.2: + // The current recommendation for the initial timer setting is 15 seconds. + // This may be changed as experience with this protocol accumulates. + // + // Considering that it is an old recommendation, we use the same reassembly + // timeout that linux defines, which is 30 seconds: + // https://github.com/torvalds/linux/blob/47ec5303d73ea344e84f46660fff693c57641386/include/net/ip.h#L138 + ReassembleTimeout = 30 * time.Second + + // ProtocolNumber is the ipv4 protocol number. + ProtocolNumber = header.IPv4ProtocolNumber + + // MaxTotalSize is maximum size that can be encoded in the 16-bit + // TotalLength field of the ipv4 header. + MaxTotalSize = 0xffff + + // DefaultTTL is the default time-to-live value for this endpoint. + DefaultTTL = 64 + + // buckets is the number of identifier buckets. + buckets = 2048 + + // The size of a fragment block, in bytes, as per RFC 791 section 3.1, + // page 14. + fragmentblockSize = 8 +) + +const ( + forwardingDisabled = 0 + forwardingEnabled = 1 +) + +var martianPacketLogger = log.BasicRateLimitedLogger(time.Minute) + +var ipv4BroadcastAddr = header.IPv4Broadcast.WithPrefix() + +var ( + _ stack.LinkResolvableNetworkEndpoint = (*endpoint)(nil) + _ stack.ForwardingNetworkEndpoint = (*endpoint)(nil) + _ stack.MulticastForwardingNetworkEndpoint = (*endpoint)(nil) + _ stack.GroupAddressableEndpoint = (*endpoint)(nil) + _ stack.AddressableEndpoint = (*endpoint)(nil) + _ stack.NetworkEndpoint = (*endpoint)(nil) + _ IGMPEndpoint = (*endpoint)(nil) +) + +// +stateify savable +type endpoint struct { + nic stack.NetworkInterface + dispatcher stack.TransportDispatcher + protocol *protocol + stats sharedStats + + // enabled is set to 1 when the endpoint is enabled and 0 when it is + // disabled. + enabled atomicbitops.Uint32 + + // forwarding is set to forwardingEnabled when the endpoint has forwarding + // enabled and forwardingDisabled when it is disabled. + forwarding atomicbitops.Uint32 + + // multicastForwarding is set to forwardingEnabled when the endpoint has + // forwarding enabled and forwardingDisabled when it is disabled. + multicastForwarding atomicbitops.Uint32 + + // mu protects below. + mu sync.RWMutex `state:"nosave"` + + // +checklocks:mu + addressableEndpointState stack.AddressableEndpointState + + // +checklocks:mu + igmp igmpState +} + +// SetIGMPVersion implements IGMPEndpoint. +func (e *endpoint) SetIGMPVersion(v IGMPVersion) IGMPVersion { + e.mu.Lock() + defer e.mu.Unlock() + return e.setIGMPVersionLocked(v) +} + +// GetIGMPVersion implements IGMPEndpoint. +func (e *endpoint) GetIGMPVersion() IGMPVersion { + e.mu.RLock() + defer e.mu.RUnlock() + return e.getIGMPVersionLocked() +} + +// +checklocks:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) setIGMPVersionLocked(v IGMPVersion) IGMPVersion { + return e.igmp.setVersion(v) +} + +// +checklocksread:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) getIGMPVersionLocked() IGMPVersion { + return e.igmp.getVersion() +} + +// HandleLinkResolutionFailure implements stack.LinkResolvableNetworkEndpoint. +func (e *endpoint) HandleLinkResolutionFailure(pkt *stack.PacketBuffer) { + // If we are operating as a router, return an ICMP error to the original + // packet's sender. + if pkt.NetworkPacketInfo.IsForwardedPacket { + // TODO(gvisor.dev/issue/6005): Propagate asynchronously generated ICMP + // errors to local endpoints. + e.protocol.returnError(&icmpReasonHostUnreachable{}, pkt, false /* deliveredLocally */) + e.stats.ip.Forwarding.Errors.Increment() + e.stats.ip.Forwarding.HostUnreachable.Increment() + return + } + // handleControl expects the entire offending packet to be in the packet + // buffer's data field. + pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: pkt.ToBuffer(), + }) + defer pkt.DecRef() + pkt.NICID = e.nic.ID() + pkt.NetworkProtocolNumber = ProtocolNumber + // Use the same control type as an ICMPv4 destination host unreachable error + // since the host is considered unreachable if we cannot resolve the link + // address to the next hop. + e.handleControl(&icmpv4DestinationHostUnreachableSockError{}, pkt) +} + +// NewEndpoint creates a new ipv4 endpoint. +func (p *protocol) NewEndpoint(nic stack.NetworkInterface, dispatcher stack.TransportDispatcher) stack.NetworkEndpoint { + e := &endpoint{ + nic: nic, + dispatcher: dispatcher, + protocol: p, + } + e.mu.Lock() + e.addressableEndpointState.Init(e, stack.AddressableEndpointStateOptions{HiddenWhileDisabled: false}) + e.igmp.init(e) + e.mu.Unlock() + + tcpip.InitStatCounters(reflect.ValueOf(&e.stats.localStats).Elem()) + + stackStats := p.stack.Stats() + e.stats.ip.Init(&e.stats.localStats.IP, &stackStats.IP) + e.stats.icmp.init(&e.stats.localStats.ICMP, &stackStats.ICMP.V4) + e.stats.igmp.init(&e.stats.localStats.IGMP, &stackStats.IGMP) + + p.mu.Lock() + p.eps[nic.ID()] = e + p.mu.Unlock() + + return e +} + +func (p *protocol) findEndpointWithAddress(addr tcpip.Address) *endpoint { + p.mu.RLock() + defer p.mu.RUnlock() + + for _, e := range p.eps { + if addressEndpoint := e.AcquireAssignedAddress(addr, false /* allowTemp */, stack.NeverPrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + return e + } + } + + return nil +} + +func (p *protocol) getEndpointForNIC(id tcpip.NICID) (*endpoint, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + ep, ok := p.eps[id] + return ep, ok +} + +func (p *protocol) forgetEndpoint(nicID tcpip.NICID) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.eps, nicID) +} + +// Forwarding implements stack.ForwardingNetworkEndpoint. +func (e *endpoint) Forwarding() bool { + return e.forwarding.Load() == forwardingEnabled +} + +// setForwarding sets the forwarding status for the endpoint. +// +// Returns the previous forwarding status. +func (e *endpoint) setForwarding(v bool) bool { + forwarding := uint32(forwardingDisabled) + if v { + forwarding = forwardingEnabled + } + + return e.forwarding.Swap(forwarding) != forwardingDisabled +} + +// SetForwarding implements stack.ForwardingNetworkEndpoint. +func (e *endpoint) SetForwarding(forwarding bool) bool { + e.mu.Lock() + defer e.mu.Unlock() + + prevForwarding := e.setForwarding(forwarding) + if prevForwarding == forwarding { + return prevForwarding + } + + if forwarding { + // There does not seem to be an RFC requirement for a node to join the all + // routers multicast address but + // https://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml + // specifies the address as a group for all routers on a subnet so we join + // the group here. + if err := e.joinGroupLocked(header.IPv4AllRoutersGroup); err != nil { + // joinGroupLocked only returns an error if the group address is not a + // valid IPv4 multicast address. + panic(fmt.Sprintf("e.joinGroupLocked(%s): %s", header.IPv4AllRoutersGroup, err)) + } + + return prevForwarding + } + + switch err := e.leaveGroupLocked(header.IPv4AllRoutersGroup).(type) { + case nil: + case *tcpip.ErrBadLocalAddress: + // The endpoint may have already left the multicast group. + default: + panic(fmt.Sprintf("e.leaveGroupLocked(%s): %s", header.IPv4AllRoutersGroup, err)) + } + + return prevForwarding +} + +// MulticastForwarding implements stack.MulticastForwardingNetworkEndpoint. +func (e *endpoint) MulticastForwarding() bool { + return e.multicastForwarding.Load() == forwardingEnabled +} + +// SetMulticastForwarding implements stack.MulticastForwardingNetworkEndpoint. +func (e *endpoint) SetMulticastForwarding(forwarding bool) bool { + updatedForwarding := uint32(forwardingDisabled) + if forwarding { + updatedForwarding = forwardingEnabled + } + + return e.multicastForwarding.Swap(updatedForwarding) != forwardingDisabled +} + +// Enable implements stack.NetworkEndpoint. +func (e *endpoint) Enable() tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + return e.enableLocked() +} + +// +checklocks:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) enableLocked() tcpip.Error { + // If the NIC is not enabled, the endpoint can't do anything meaningful so + // don't enable the endpoint. + if !e.nic.Enabled() { + return &tcpip.ErrNotPermitted{} + } + + // If the endpoint is already enabled, there is nothing for it to do. + if !e.setEnabled(true) { + return nil + } + + // Must be called after Enabled has already been set. + e.addressableEndpointState.OnNetworkEndpointEnabledChanged() + + // Create an endpoint to receive broadcast packets on this interface. + ep, err := e.addressableEndpointState.AddAndAcquirePermanentAddress(ipv4BroadcastAddr, stack.AddressProperties{PEB: stack.NeverPrimaryEndpoint}) + if err != nil { + return err + } + // We have no need for the address endpoint. + ep.DecRef() + + // Groups may have been joined while the endpoint was disabled, or the + // endpoint may have left groups from the perspective of IGMP when the + // endpoint was disabled. Either way, we need to let routers know to + // send us multicast traffic. + e.igmp.initializeAll() + + // As per RFC 1122 section 3.3.7, all hosts should join the all-hosts + // multicast group. Note, the IANA calls the all-hosts multicast group the + // all-systems multicast group. + if err := e.joinGroupLocked(header.IPv4AllSystems); err != nil { + // joinGroupLocked only returns an error if the group address is not a valid + // IPv4 multicast address. + panic(fmt.Sprintf("e.joinGroupLocked(%s): %s", header.IPv4AllSystems, err)) + } + + return nil +} + +// Enabled implements stack.NetworkEndpoint. +func (e *endpoint) Enabled() bool { + return e.nic.Enabled() && e.isEnabled() +} + +// isEnabled returns true if the endpoint is enabled, regardless of the +// enabled status of the NIC. +func (e *endpoint) isEnabled() bool { + return e.enabled.Load() == 1 +} + +// setEnabled sets the enabled status for the endpoint. +// +// Returns true if the enabled status was updated. +func (e *endpoint) setEnabled(v bool) bool { + if v { + return e.enabled.Swap(1) == 0 + } + return e.enabled.Swap(0) == 1 +} + +// Disable implements stack.NetworkEndpoint. +func (e *endpoint) Disable() { + e.mu.Lock() + defer e.mu.Unlock() + e.disableLocked() +} + +// +checklocks:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) disableLocked() { + if !e.isEnabled() { + return + } + + // The endpoint may have already left the multicast group. + switch err := e.leaveGroupLocked(header.IPv4AllSystems).(type) { + case nil, *tcpip.ErrBadLocalAddress: + default: + panic(fmt.Sprintf("unexpected error when leaving group = %s: %s", header.IPv4AllSystems, err)) + } + + // Leave groups from the perspective of IGMP so that routers know that + // we are no longer interested in the group. + e.igmp.softLeaveAll() + + // The address may have already been removed. + switch err := e.addressableEndpointState.RemovePermanentAddress(ipv4BroadcastAddr.Address); err.(type) { + case nil, *tcpip.ErrBadLocalAddress: + default: + panic(fmt.Sprintf("unexpected error when removing address = %s: %s", ipv4BroadcastAddr.Address, err)) + } + + // Reset the IGMP V1 present flag. + // + // If the node comes back up on the same network, it will re-learn that it + // needs to perform IGMPv1. + e.igmp.resetV1Present() + + if !e.setEnabled(false) { + panic("should have only done work to disable the endpoint if it was enabled") + } + + // Must be called after Enabled has been set. + e.addressableEndpointState.OnNetworkEndpointEnabledChanged() +} + +// emitMulticastEvent emits a multicast forwarding event using the provided +// generator if a valid event dispatcher exists. +func (e *endpoint) emitMulticastEvent(eventGenerator func(stack.MulticastForwardingEventDispatcher)) { + e.protocol.mu.RLock() + defer e.protocol.mu.RUnlock() + + if mcastDisp := e.protocol.multicastForwardingDisp; mcastDisp != nil { + eventGenerator(mcastDisp) + } +} + +// DefaultTTL is the default time-to-live value for this endpoint. +func (e *endpoint) DefaultTTL() uint8 { + return e.protocol.DefaultTTL() +} + +// MTU implements stack.NetworkEndpoint. It returns the link-layer MTU minus the +// network layer max header length. +func (e *endpoint) MTU() uint32 { + networkMTU, err := calculateNetworkMTU(e.nic.MTU(), header.IPv4MinimumSize) + if err != nil { + return 0 + } + return networkMTU +} + +// MaxHeaderLength returns the maximum length needed by ipv4 headers (and +// underlying protocols). +func (e *endpoint) MaxHeaderLength() uint16 { + return e.nic.MaxHeaderLength() + header.IPv4MaximumHeaderSize +} + +// NetworkProtocolNumber implements stack.NetworkEndpoint. +func (e *endpoint) NetworkProtocolNumber() tcpip.NetworkProtocolNumber { + return e.protocol.Number() +} + +// getID returns a random uint16 number (other than zero) to be used as ID in +// the IPv4 header. +func (e *endpoint) getID() uint16 { + rng := e.protocol.stack.SecureRNG() + id := rng.Uint16() + for id == 0 { + id = rng.Uint16() + } + return id +} + +func (e *endpoint) addIPHeader(srcAddr, dstAddr tcpip.Address, pkt *stack.PacketBuffer, params stack.NetworkHeaderParams, options header.IPv4OptionsSerializer) tcpip.Error { + if expVal := params.ExperimentOptionValue; expVal != 0 { + options = append(options, &header.IPv4SerializableExperimentOption{Tag: expVal}) + } + hdrLen := header.IPv4MinimumSize + var optLen int + if options != nil { + optLen = int(options.Length()) + } + hdrLen += optLen + if hdrLen > header.IPv4MaximumHeaderSize { + return &tcpip.ErrMessageTooLong{} + } + ipH := header.IPv4(pkt.NetworkHeader().Push(hdrLen)) + length := pkt.Size() + if length > math.MaxUint16 { + return &tcpip.ErrMessageTooLong{} + } + + fields := header.IPv4Fields{ + TotalLength: uint16(length), + TTL: params.TTL, + TOS: params.TOS, + Protocol: uint8(params.Protocol), + SrcAddr: srcAddr, + DstAddr: dstAddr, + Options: options, + } + if params.DF { + // Treat want and do the same. + fields.Flags = header.IPv4FlagDontFragment + } else { + // RFC 6864 section 4.3 mandates uniqueness of ID values for + // non-atomic datagrams. + fields.ID = e.getID() + } + ipH.Encode(&fields) + + ipH.SetChecksum(^ipH.CalculateChecksum()) + pkt.NetworkProtocolNumber = ProtocolNumber + return nil +} + +// handleFragments fragments pkt and calls the handler function on each +// fragment. It returns the number of fragments handled and the number of +// fragments left to be processed. The IP header must already be present in the +// original packet. +func (e *endpoint) handleFragments(_ *stack.Route, networkMTU uint32, pkt *stack.PacketBuffer, handler func(*stack.PacketBuffer) tcpip.Error) (int, int, tcpip.Error) { + // Round the MTU down to align to 8 bytes. + fragmentPayloadSize := networkMTU &^ 7 + networkHeader := header.IPv4(pkt.NetworkHeader().Slice()) + pf := fragmentation.MakePacketFragmenter(pkt, fragmentPayloadSize, pkt.AvailableHeaderBytes()+len(networkHeader)) + defer pf.Release() + + var n int + for { + fragPkt, more := buildNextFragment(&pf, networkHeader) + err := handler(fragPkt) + fragPkt.DecRef() + if err != nil { + return n, pf.RemainingFragmentCount() + 1, err + } + n++ + if !more { + return n, pf.RemainingFragmentCount(), nil + } + } +} + +// WritePacket writes a packet to the given destination address and protocol. +func (e *endpoint) WritePacket(r *stack.Route, params stack.NetworkHeaderParams, pkt *stack.PacketBuffer) tcpip.Error { + if err := e.addIPHeader(r.LocalAddress(), r.RemoteAddress(), pkt, params, nil /* options */); err != nil { + return err + } + + return e.writePacket(r, pkt) +} + +func (e *endpoint) writePacket(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error { + netHeader := header.IPv4(pkt.NetworkHeader().Slice()) + dstAddr := netHeader.DestinationAddress() + + // iptables filtering. All packets that reach here are locally + // generated. + outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) + if ok := e.protocol.stack.IPTables().CheckOutput(pkt, r, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesOutputDropped.Increment() + return nil + } + + // If the packet is manipulated as per DNAT Output rules, handle packet + // based on destination address and do not send the packet to link + // layer. + // + // We should do this for every packet, rather than only DNATted packets, but + // removing this check short circuits broadcasts before they are sent out to + // other hosts. + if newDstAddr := netHeader.DestinationAddress(); dstAddr != newDstAddr { + if ep := e.protocol.findEndpointWithAddress(newDstAddr); ep != nil { + // Since we rewrote the packet but it is being routed back to us, we + // can safely assume the checksum is valid. + ep.handleLocalPacket(pkt, true /* canSkipRXChecksum */) + return nil + } + } + + return e.writePacketPostRouting(r, pkt, false /* headerIncluded */) +} + +func (e *endpoint) writePacketPostRouting(r *stack.Route, pkt *stack.PacketBuffer, headerIncluded bool) tcpip.Error { + if r.Loop()&stack.PacketLoop != 0 { + // If the packet was generated by the stack (not a raw/packet endpoint + // where a packet may be written with the header included), then we can + // safely assume the checksum is valid. + e.handleLocalPacket(pkt, !headerIncluded /* canSkipRXChecksum */) + } + if r.Loop()&stack.PacketOut == 0 { + return nil + } + + // Postrouting NAT can only change the source address, and does not alter the + // route or outgoing interface of the packet. + outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) + if ok := e.protocol.stack.IPTables().CheckPostrouting(pkt, r, e, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesPostroutingDropped.Increment() + return nil + } + + stats := e.stats.ip + + networkMTU, err := calculateNetworkMTU(e.nic.MTU(), uint32(len(pkt.NetworkHeader().Slice()))) + if err != nil { + stats.OutgoingPacketErrors.Increment() + return err + } + + if packetMustBeFragmented(pkt, networkMTU) { + h := header.IPv4(pkt.NetworkHeader().Slice()) + if h.Flags()&header.IPv4FlagDontFragment != 0 && pkt.NetworkPacketInfo.IsForwardedPacket { + // TODO(gvisor.dev/issue/5919): Handle error condition in which DontFragment + // is set but the packet must be fragmented for the non-forwarding case. + return &tcpip.ErrMessageTooLong{} + } + sent, remain, err := e.handleFragments(r, networkMTU, pkt, func(fragPkt *stack.PacketBuffer) tcpip.Error { + // TODO(gvisor.dev/issue/3884): Evaluate whether we want to send each + // fragment one by one using WritePacket() (current strategy) or if we + // want to create a PacketBufferList from the fragments and feed it to + // WritePackets(). It'll be faster but cost more memory. + return e.nic.WritePacket(r, fragPkt) + }) + stats.PacketsSent.IncrementBy(uint64(sent)) + stats.OutgoingPacketErrors.IncrementBy(uint64(remain)) + return err + } + + if err := e.nic.WritePacket(r, pkt); err != nil { + stats.OutgoingPacketErrors.Increment() + return err + } + stats.PacketsSent.Increment() + return nil +} + +// WriteHeaderIncludedPacket implements stack.NetworkEndpoint. +func (e *endpoint) WriteHeaderIncludedPacket(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error { + // The packet already has an IP header, but there are a few required + // checks. + h, ok := pkt.Data().PullUp(header.IPv4MinimumSize) + if !ok { + return &tcpip.ErrMalformedHeader{} + } + + hdrLen := header.IPv4(h).HeaderLength() + if hdrLen < header.IPv4MinimumSize { + return &tcpip.ErrMalformedHeader{} + } + + h, ok = pkt.Data().PullUp(int(hdrLen)) + if !ok { + return &tcpip.ErrMalformedHeader{} + } + ipH := header.IPv4(h) + + // Always set the total length. + pktSize := pkt.Data().Size() + ipH.SetTotalLength(uint16(pktSize)) + + // Set the source address when zero. + if ipH.SourceAddress() == header.IPv4Any { + ipH.SetSourceAddress(r.LocalAddress()) + } + + // Set the packet ID when zero. + if ipH.ID() == 0 { + // RFC 6864 section 4.3 mandates uniqueness of ID values for + // non-atomic datagrams, so assign an ID to all such datagrams + // according to the definition given in RFC 6864 section 4. + if ipH.Flags()&header.IPv4FlagDontFragment == 0 || ipH.Flags()&header.IPv4FlagMoreFragments != 0 || ipH.FragmentOffset() > 0 { + ipH.SetID(e.getID()) + } + } + + // Always set the checksum. + ipH.SetChecksum(0) + ipH.SetChecksum(^ipH.CalculateChecksum()) + + // Populate the packet buffer's network header and don't allow an invalid + // packet to be sent. + // + // Note that parsing only makes sure that the packet is well formed as per the + // wire format. We also want to check if the header's fields are valid before + // sending the packet. + if !parse.IPv4(pkt) || !header.IPv4(pkt.NetworkHeader().Slice()).IsValid(pktSize) { + return &tcpip.ErrMalformedHeader{} + } + + return e.writePacketPostRouting(r, pkt, true /* headerIncluded */) +} + +// forwardPacketWithRoute emits the pkt using the provided route. +// +// If updateOptions is true, then the IP options will be updated in the copied +// pkt using the outgoing endpoint. Otherwise, the caller is responsible for +// updating the options. +// +// This method should be invoked by the endpoint that received the pkt. +func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketBuffer, updateOptions bool) ip.ForwardingError { + h := header.IPv4(pkt.NetworkHeader().Slice()) + stk := e.protocol.stack + + inNicName := stk.FindNICNameFromID(e.nic.ID()) + outNicName := stk.FindNICNameFromID(route.NICID()) + if ok := stk.IPTables().CheckForward(pkt, inNicName, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesForwardDropped.Increment() + return nil + } + + // We need to do a deep copy of the IP packet because + // WriteHeaderIncludedPacket may modify the packet buffer, but we do + // not own it. + // + // TODO(https://gvisor.dev/issue/7473): For multicast, only create one deep + // copy and then clone. + newPkt := pkt.DeepCopyForForwarding(int(route.MaxHeaderLength())) + newHdr := header.IPv4(newPkt.NetworkHeader().Slice()) + defer newPkt.DecRef() + + forwardToEp, ok := e.protocol.getEndpointForNIC(route.NICID()) + if !ok { + return &ip.ErrUnknownOutputEndpoint{} + } + + if updateOptions { + if err := forwardToEp.updateOptionsForForwarding(newPkt); err != nil { + return err + } + } + + ttl := h.TTL() + // As per RFC 791 page 30, Time to Live, + // + // This field must be decreased at each point that the internet header + // is processed to reflect the time spent processing the datagram. + // Even if no local information is available on the time actually + // spent, the field must be decremented by 1. + newHdr.SetTTL(ttl - 1) + // We perform a full checksum as we may have updated options above. The IP + // header is relatively small so this is not expected to be an expensive + // operation. + newHdr.SetChecksum(0) + newHdr.SetChecksum(^newHdr.CalculateChecksum()) + + switch err := forwardToEp.writePacketPostRouting(route, newPkt, true /* headerIncluded */); err.(type) { + case nil: + return nil + case *tcpip.ErrMessageTooLong: + // As per RFC 792, page 4, Destination Unreachable: + // + // Another case is when a datagram must be fragmented to be forwarded by a + // gateway yet the Don't Fragment flag is on. In this case the gateway must + // discard the datagram and may return a destination unreachable message. + // + // WriteHeaderIncludedPacket checks for the presence of the Don't Fragment bit + // while sending the packet and returns this error iff fragmentation is + // necessary and the bit is also set. + _ = e.protocol.returnError(&icmpReasonFragmentationNeeded{}, pkt, false /* deliveredLocally */) + return &ip.ErrMessageTooLong{} + case *tcpip.ErrNoBufferSpace: + return &ip.ErrOutgoingDeviceNoBufferSpace{} + default: + return &ip.ErrOther{Err: err} + } +} + +// forwardUnicastPacket attempts to forward a packet to its final destination. +func (e *endpoint) forwardUnicastPacket(pkt *stack.PacketBuffer) ip.ForwardingError { + hView := pkt.NetworkHeader().View() + defer hView.Release() + h := header.IPv4(hView.AsSlice()) + + dstAddr := h.DestinationAddress() + + if err := validateAddressesForForwarding(h); err != nil { + return err + } + + ttl := h.TTL() + if ttl == 0 { + // As per RFC 792 page 6, Time Exceeded Message, + // + // If the gateway processing a datagram finds the time to live field + // is zero it must discard the datagram. The gateway may also notify + // the source host via the time exceeded message. + // + // We return the original error rather than the result of returning + // the ICMP packet because the original error is more relevant to + // the caller. + _ = e.protocol.returnError(&icmpReasonTTLExceeded{}, pkt, false /* deliveredLocally */) + return &ip.ErrTTLExceeded{} + } + + if err := e.updateOptionsForForwarding(pkt); err != nil { + return err + } + + stk := e.protocol.stack + + // Check if the destination is owned by the stack. + if ep := e.protocol.findEndpointWithAddress(dstAddr); ep != nil { + inNicName := stk.FindNICNameFromID(e.nic.ID()) + outNicName := stk.FindNICNameFromID(ep.nic.ID()) + if ok := stk.IPTables().CheckForward(pkt, inNicName, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesForwardDropped.Increment() + return nil + } + + // The packet originally arrived on e so provide its NIC as the input NIC. + ep.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) + return nil + } + + r, err := stk.FindRoute(0, tcpip.Address{}, dstAddr, ProtocolNumber, false /* multicastLoop */) + switch err.(type) { + case nil: + case *tcpip.ErrNetworkUnreachable: + // We return the original error rather than the result of returning + // the ICMP packet because the original error is more relevant to + // the caller. + _ = e.protocol.returnError(&icmpReasonNetworkUnreachable{}, pkt, false /* deliveredLocally */) + return &ip.ErrHostUnreachable{} + default: + return &ip.ErrOther{Err: err} + } + defer r.Release() + + // TODO(https://gvisor.dev/issue/7472): Unicast IP options should be updated + // using the output endpoint (instead of the input endpoint). In particular, + // RFC 1812 section 5.2.1 states the following: + // + // Processing of certain IP options requires that the router insert its IP + // address into the option. As noted in Section [5.2.4], the address + // inserted MUST be the address of the logical interface on which the + // packet is sent or the router's router-id if the packet is sent over an + // unnumbered interface. Thus, processing of these options cannot be + // completed until after the output interface is chosen. + return e.forwardPacketWithRoute(r, pkt, false /* updateOptions */) +} + +// HandlePacket is called by the link layer when new ipv4 packets arrive for +// this endpoint. +func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { + stats := e.stats.ip + + stats.PacketsReceived.Increment() + + if !e.isEnabled() { + stats.DisabledPacketsReceived.Increment() + return + } + + hView, ok := e.protocol.parseAndValidate(pkt) + if !ok { + stats.MalformedPacketsReceived.Increment() + return + } + h := header.IPv4(hView.AsSlice()) + defer hView.Release() + + if !e.nic.IsLoopback() { + if !e.protocol.options.AllowExternalLoopbackTraffic { + if header.IsV4LoopbackAddress(h.SourceAddress()) { + martianPacketLogger.Infof("Martian packet dropped with loopback source address. If your traffic is unexpectedly dropped, you may want to allow martian packets.") + stats.InvalidSourceAddressesReceived.Increment() + return + } + + if header.IsV4LoopbackAddress(h.DestinationAddress()) { + martianPacketLogger.Infof("Martian packet dropped with loopback destination address. If your traffic is unexpectedly dropped, you may want to allow martian packets.") + stats.InvalidDestinationAddressesReceived.Increment() + return + } + } + + if e.protocol.stack.HandleLocal() { + addressEndpoint := e.AcquireAssignedAddress(header.IPv4(pkt.NetworkHeader().Slice()).SourceAddress(), e.nic.Promiscuous(), stack.CanBePrimaryEndpoint, true /* readOnly */) + if addressEndpoint != nil { + // The source address is one of our own, so we never should have gotten + // a packet like this unless HandleLocal is false or our NIC is the + // loopback interface. + stats.InvalidSourceAddressesReceived.Increment() + return + } + } + + // Loopback traffic skips the prerouting chain. + inNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) + if ok := e.protocol.stack.IPTables().CheckPrerouting(pkt, e, inNicName); !ok { + // iptables is telling us to drop the packet. + stats.IPTablesPreroutingDropped.Increment() + return + } + } + // CheckPrerouting can modify the backing storage of the packet, so refresh + // the header. + h = header.IPv4(pkt.NetworkHeader().Slice()) + e.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) +} + +// handleLocalPacket is like HandlePacket except it does not perform the +// prerouting iptables hook or check for loopback traffic that originated from +// outside of the netstack (i.e. martian loopback packets). +func (e *endpoint) handleLocalPacket(pkt *stack.PacketBuffer, canSkipRXChecksum bool) { + stats := e.stats.ip + stats.PacketsReceived.Increment() + + pkt = pkt.CloneToInbound() + defer pkt.DecRef() + pkt.RXChecksumValidated = canSkipRXChecksum + + hView, ok := e.protocol.parseAndValidate(pkt) + if !ok { + stats.MalformedPacketsReceived.Increment() + return + } + h := header.IPv4(hView.AsSlice()) + defer hView.Release() + + e.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) +} + +func validateAddressesForForwarding(h header.IPv4) ip.ForwardingError { + srcAddr := h.SourceAddress() + + // As per RFC 5735 section 3, + // + // 0.0.0.0/8 - Addresses in this block refer to source hosts on "this" + // network. Address 0.0.0.0/32 may be used as a source address for this + // host on this network; other addresses within 0.0.0.0/8 may be used to + // refer to specified hosts on this network ([RFC1122], Section 3.2.1.3). + // + // And RFC 6890 section 2.2.2, + // + // +----------------------+----------------------------+ + // | Attribute | Value | + // +----------------------+----------------------------+ + // | Address Block | 0.0.0.0/8 | + // | Name | "This host on this network"| + // | RFC | [RFC1122], Section 3.2.1.3 | + // | Allocation Date | September 1981 | + // | Termination Date | N/A | + // | Source | True | + // | Destination | False | + // | Forwardable | False | + // | Global | False | + // | Reserved-by-Protocol | True | + // +----------------------+----------------------------+ + if header.IPv4CurrentNetworkSubnet.Contains(srcAddr) { + return &ip.ErrInitializingSourceAddress{} + } + + return nil +} + +// forwardMulticastPacket validates a multicast pkt and attempts to forward it. +// +// This method should be invoked for incoming multicast packets using the +// endpoint that received the packet. +func (e *endpoint) forwardMulticastPacket(h header.IPv4, pkt *stack.PacketBuffer) ip.ForwardingError { + if err := validateAddressesForForwarding(h); err != nil { + return err + } + + if opts := h.Options(); len(opts) != 0 { + // Check if the options are valid, but don't mutate them. This corresponds + // to step 3 of RFC 1812 section 5.2.1.1. + if _, _, optProblem := e.processIPOptions(pkt, opts, &optionUsageVerify{}); optProblem != nil { + // Per RFC 1812 section 4.3.2.7, an ICMP error message should not be + // sent for: + // + // A packet destined to an IP broadcast or IP multicast address. + // + // Note that protocol.returnError also enforces this requirement. + // However, we intentionally omit it here since this path is multicast + // only. + return &ip.ErrParameterProblem{} + } + } + + routeKey := stack.UnicastSourceAndMulticastDestination{ + Source: h.SourceAddress(), + Destination: h.DestinationAddress(), + } + + // The pkt has been validated. Consequently, if a route is not found, then + // the pkt can safely be queued. + result, hasBufferSpace := e.protocol.multicastRouteTable.GetRouteOrInsertPending(routeKey, pkt) + + if !hasBufferSpace { + // Unable to queue the pkt. Silently drop it. + return &ip.ErrNoMulticastPendingQueueBufferSpace{} + } + + switch result.GetRouteResultState { + case multicast.InstalledRouteFound: + // Attempt to forward the pkt using an existing route. + return e.forwardValidatedMulticastPacket(pkt, result.InstalledRoute) + case multicast.NoRouteFoundAndPendingInserted: + e.emitMulticastEvent(func(disp stack.MulticastForwardingEventDispatcher) { + disp.OnMissingRoute(stack.MulticastPacketContext{ + stack.UnicastSourceAndMulticastDestination{h.SourceAddress(), h.DestinationAddress()}, + e.nic.ID(), + }) + }) + case multicast.PacketQueuedInPendingRoute: + default: + panic(fmt.Sprintf("unexpected GetRouteResultState: %s", result.GetRouteResultState)) + } + return &ip.ErrHostUnreachable{} +} + +func (e *endpoint) updateOptionsForForwarding(pkt *stack.PacketBuffer) ip.ForwardingError { + h := header.IPv4(pkt.NetworkHeader().Slice()) + if opts := h.Options(); len(opts) != 0 { + newOpts, _, optProblem := e.processIPOptions(pkt, opts, &optionUsageForward{}) + if optProblem != nil { + if optProblem.NeedICMP { + // Note that this will not emit an ICMP error if the destination is + // multicast. + _ = e.protocol.returnError(&icmpReasonParamProblem{ + pointer: optProblem.Pointer, + }, pkt, false /* deliveredLocally */) + } + return &ip.ErrParameterProblem{} + } + copied := copy(opts, newOpts) + if copied != len(newOpts) { + panic(fmt.Sprintf("copied %d bytes of new options, expected %d bytes", copied, len(newOpts))) + } + // Since in forwarding we handle all options, including copying those we + // do not recognise, the options region should remain the same size which + // simplifies processing. As we MAY receive a packet with a lot of padded + // bytes after the "end of options list" byte, make sure we copy + // them as the legal padding value (0). + for i := copied; i < len(opts); i++ { + // Pad with 0 (EOL). RFC 791 page 23 says "The padding is zero". + opts[i] = byte(header.IPv4OptionListEndType) + } + } + return nil +} + +// forwardValidatedMulticastPacket attempts to forward the pkt using the +// provided installedRoute. +// +// This method should be invoked by the endpoint that received the pkt. +func (e *endpoint) forwardValidatedMulticastPacket(pkt *stack.PacketBuffer, installedRoute *multicast.InstalledRoute) ip.ForwardingError { + // Per RFC 1812 section 5.2.1.3, + // + // Based on the IP source and destination addresses found in the datagram + // header, the router determines whether the datagram has been received + // on the proper interface for forwarding. If not, the datagram is + // dropped silently. + if e.nic.ID() != installedRoute.ExpectedInputInterface { + h := header.IPv4(pkt.NetworkHeader().Slice()) + e.emitMulticastEvent(func(disp stack.MulticastForwardingEventDispatcher) { + disp.OnUnexpectedInputInterface(stack.MulticastPacketContext{ + stack.UnicastSourceAndMulticastDestination{h.SourceAddress(), h.DestinationAddress()}, + e.nic.ID(), + }, installedRoute.ExpectedInputInterface) + }) + return &ip.ErrUnexpectedMulticastInputInterface{} + } + + for _, outgoingInterface := range installedRoute.OutgoingInterfaces { + if err := e.forwardMulticastPacketForOutgoingInterface(pkt, outgoingInterface); err != nil { + e.handleForwardingError(err) + continue + } + // The pkt was successfully forwarded. Mark the route as used. + installedRoute.SetLastUsedTimestamp(e.protocol.stack.Clock().NowMonotonic()) + } + return nil +} + +// forwardMulticastPacketForOutgoingInterface attempts to forward the pkt out +// of the provided outgoingInterface. +// +// This method should be invoked by the endpoint that received the pkt. +func (e *endpoint) forwardMulticastPacketForOutgoingInterface(pkt *stack.PacketBuffer, outgoingInterface stack.MulticastRouteOutgoingInterface) ip.ForwardingError { + h := header.IPv4(pkt.NetworkHeader().Slice()) + + // Per RFC 1812 section 5.2.1.3, + // + // A copy of the multicast datagram is forwarded out each outgoing + // interface whose minimum TTL value is less than or equal to the TTL + // value in the datagram header. + // + // Copying of the packet is deferred to forwardPacketWithRoute since unicast + // and multicast both require a copy. + if outgoingInterface.MinTTL > h.TTL() { + return &ip.ErrTTLExceeded{} + } + + route := e.protocol.stack.NewRouteForMulticast(outgoingInterface.ID, h.DestinationAddress(), e.NetworkProtocolNumber()) + + if route == nil { + // Failed to convert to a stack.Route. This likely means that the outgoing + // endpoint no longer exists. + return &ip.ErrHostUnreachable{} + } + defer route.Release() + + return e.forwardPacketWithRoute(route, pkt, true /* updateOptions */) +} + +func (e *endpoint) handleValidatedPacket(h header.IPv4, pkt *stack.PacketBuffer, inNICName string) { + pkt.NICID = e.nic.ID() + + // Raw socket packets are delivered based solely on the transport protocol + // number. We only require that the packet be valid IPv4, and that they not + // be fragmented. + if !h.More() && h.FragmentOffset() == 0 { + e.dispatcher.DeliverRawPacket(h.TransportProtocol(), pkt) + } + + stats := e.stats + stats.ip.ValidPacketsReceived.Increment() + + srcAddr := h.SourceAddress() + dstAddr := h.DestinationAddress() + + // As per RFC 1122 section 3.2.1.3: + // When a host sends any datagram, the IP source address MUST + // be one of its own IP addresses (but not a broadcast or + // multicast address). + if srcAddr == header.IPv4Broadcast || header.IsV4MulticastAddress(srcAddr) { + stats.ip.InvalidSourceAddressesReceived.Increment() + return + } + // Make sure the source address is not a subnet-local broadcast address. + if addressEndpoint := e.AcquireAssignedAddress(srcAddr, false /* createTemp */, stack.NeverPrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + subnet := addressEndpoint.Subnet() + if subnet.IsBroadcast(srcAddr) { + stats.ip.InvalidSourceAddressesReceived.Increment() + return + } + } + + if header.IsV4MulticastAddress(dstAddr) { + // Handle all packets destined to a multicast address separately. Unlike + // unicast, these packets can be both delivered locally and forwarded. See + // RFC 1812 section 5.2.3 for details regarding the forwarding/local + // delivery decision. + + multicastForwarding := e.MulticastForwarding() && e.protocol.multicastForwarding() + + if multicastForwarding { + e.handleForwardingError(e.forwardMulticastPacket(h, pkt)) + } + + if e.IsInGroup(dstAddr) { + e.deliverPacketLocally(h, pkt, inNICName) + return + } + + if !multicastForwarding { + // Only consider the destination address invalid if we didn't attempt to + // forward the pkt and it was not delivered locally. + stats.ip.InvalidDestinationAddressesReceived.Increment() + } + return + } + + // Before we do any processing, check if the packet was received as some + // sort of broadcast. + // + // If the packet is destined for this device, then it should be delivered + // locally. Otherwise, if forwarding is enabled, it should be forwarded. + if addressEndpoint := e.AcquireAssignedAddress(dstAddr, e.nic.Promiscuous(), stack.CanBePrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + pkt.NetworkPacketInfo.LocalAddressTemporary = addressEndpoint.Temporary() + subnet := addressEndpoint.AddressWithPrefix().Subnet() + pkt.NetworkPacketInfo.LocalAddressBroadcast = subnet.IsBroadcast(dstAddr) || dstAddr == header.IPv4Broadcast + e.deliverPacketLocally(h, pkt, inNICName) + } else if e.Forwarding() { + e.handleForwardingError(e.forwardUnicastPacket(pkt)) + } else { + stats.ip.InvalidDestinationAddressesReceived.Increment() + } +} + +// handleForwardingError processes the provided err and increments any relevant +// counters. +func (e *endpoint) handleForwardingError(err ip.ForwardingError) { + stats := e.stats.ip + switch err := err.(type) { + case nil: + return + case *ip.ErrInitializingSourceAddress: + stats.Forwarding.InitializingSource.Increment() + case *ip.ErrLinkLocalSourceAddress: + stats.Forwarding.LinkLocalSource.Increment() + case *ip.ErrLinkLocalDestinationAddress: + stats.Forwarding.LinkLocalDestination.Increment() + case *ip.ErrTTLExceeded: + stats.Forwarding.ExhaustedTTL.Increment() + case *ip.ErrHostUnreachable: + stats.Forwarding.Unrouteable.Increment() + case *ip.ErrParameterProblem: + stats.MalformedPacketsReceived.Increment() + case *ip.ErrMessageTooLong: + stats.Forwarding.PacketTooBig.Increment() + case *ip.ErrNoMulticastPendingQueueBufferSpace: + stats.Forwarding.NoMulticastPendingQueueBufferSpace.Increment() + case *ip.ErrUnexpectedMulticastInputInterface: + stats.Forwarding.UnexpectedMulticastInputInterface.Increment() + case *ip.ErrUnknownOutputEndpoint: + stats.Forwarding.UnknownOutputEndpoint.Increment() + case *ip.ErrOutgoingDeviceNoBufferSpace: + stats.Forwarding.OutgoingDeviceNoBufferSpace.Increment() + case *ip.ErrOther: + switch err := err.Err.(type) { + case *tcpip.ErrClosedForSend: + stats.Forwarding.OutgoingDeviceClosedForSend.Increment() + default: + panic(fmt.Sprintf("unrecognized tcpip forwarding error: %s", err)) + } + default: + panic(fmt.Sprintf("unrecognized forwarding error: %s", err)) + } + stats.Forwarding.Errors.Increment() +} + +func (e *endpoint) deliverPacketLocally(h header.IPv4, pkt *stack.PacketBuffer, inNICName string) { + stats := e.stats + // iptables filtering. All packets that reach here are intended for + // this machine and will not be forwarded. + if ok := e.protocol.stack.IPTables().CheckInput(pkt, inNICName); !ok { + // iptables is telling us to drop the packet. + stats.ip.IPTablesInputDropped.Increment() + return + } + + if h.More() || h.FragmentOffset() != 0 { + if pkt.Data().Size()+len(pkt.TransportHeader().Slice()) == 0 { + // Drop the packet as it's marked as a fragment but has + // no payload. + stats.ip.MalformedPacketsReceived.Increment() + stats.ip.MalformedFragmentsReceived.Increment() + return + } + if opts := h.Options(); len(opts) != 0 { + // If there are options we need to check them before we do assembly + // or we could be assembling errant packets. However we do not change the + // options as that could lead to double processing later. + if _, _, optProblem := e.processIPOptions(pkt, opts, &optionUsageVerify{}); optProblem != nil { + if optProblem.NeedICMP { + _ = e.protocol.returnError(&icmpReasonParamProblem{ + pointer: optProblem.Pointer, + }, pkt, true /* deliveredLocally */) + e.stats.ip.MalformedPacketsReceived.Increment() + } + return + } + } + // The packet is a fragment, let's try to reassemble it. + start := h.FragmentOffset() + // Drop the fragment if the size of the reassembled payload would exceed the + // maximum payload size. + // + // Note that this addition doesn't overflow even on 32bit architecture + // because pkt.Data().Size() should not exceed 65535 (the max IP datagram + // size). Otherwise the packet would've been rejected as invalid before + // reaching here. + if int(start)+pkt.Data().Size() > header.IPv4MaximumPayloadSize { + stats.ip.MalformedPacketsReceived.Increment() + stats.ip.MalformedFragmentsReceived.Increment() + return + } + + proto := h.Protocol() + resPkt, transProtoNum, ready, err := e.protocol.fragmentation.Process( + // As per RFC 791 section 2.3, the identification value is unique + // for a source-destination pair and protocol. + fragmentation.FragmentID{ + Source: h.SourceAddress(), + Destination: h.DestinationAddress(), + ID: uint32(h.ID()), + Protocol: proto, + }, + start, + start+uint16(pkt.Data().Size())-1, + h.More(), + proto, + pkt, + ) + if err != nil { + stats.ip.MalformedPacketsReceived.Increment() + stats.ip.MalformedFragmentsReceived.Increment() + return + } + if !ready { + return + } + defer resPkt.DecRef() + pkt = resPkt + h = header.IPv4(pkt.NetworkHeader().Slice()) + + // The reassembler doesn't take care of fixing up the header, so we need + // to do it here. + h.SetTotalLength(uint16(pkt.Data().Size() + len(h))) + h.SetFlagsFragmentOffset(0, 0) + + e.protocol.parseTransport(pkt, tcpip.TransportProtocolNumber(transProtoNum)) + + // Now that the packet is reassembled, it can be sent to raw sockets. + e.dispatcher.DeliverRawPacket(h.TransportProtocol(), pkt) + } + stats.ip.PacketsDelivered.Increment() + + p := h.TransportProtocol() + if p == header.ICMPv4ProtocolNumber { + // TODO(gvisor.dev/issues/3810): when we sort out ICMP and transport + // headers, the setting of the transport number here should be + // unnecessary and removed. + pkt.TransportProtocolNumber = p + e.handleICMP(pkt) + return + } + // ICMP handles options itself but do it here for all remaining destinations. + var hasRouterAlertOption bool + if opts := h.Options(); len(opts) != 0 { + newOpts, processedOpts, optProblem := e.processIPOptions(pkt, opts, &optionUsageReceive{}) + if optProblem != nil { + if optProblem.NeedICMP { + _ = e.protocol.returnError(&icmpReasonParamProblem{ + pointer: optProblem.Pointer, + }, pkt, true /* deliveredLocally */) + stats.ip.MalformedPacketsReceived.Increment() + } + return + } + hasRouterAlertOption = processedOpts.routerAlert + copied := copy(opts, newOpts) + if copied != len(newOpts) { + panic(fmt.Sprintf("copied %d bytes of new options, expected %d bytes", copied, len(newOpts))) + } + for i := copied; i < len(opts); i++ { + // Pad with 0 (EOL). RFC 791 page 23 says "The padding is zero". + opts[i] = byte(header.IPv4OptionListEndType) + } + } + if p == header.IGMPProtocolNumber { + e.mu.Lock() + e.igmp.handleIGMP(pkt, hasRouterAlertOption) // +checklocksforce: e == e.igmp.ep. + e.mu.Unlock() + return + } + + switch res := e.dispatcher.DeliverTransportPacket(p, pkt); res { + case stack.TransportPacketHandled: + case stack.TransportPacketDestinationPortUnreachable: + // As per RFC: 1122 Section 3.2.2.1 A host SHOULD generate Destination + // Unreachable messages with code: + // 3 (Port Unreachable), when the designated transport protocol + // (e.g., UDP) is unable to demultiplex the datagram but has no + // protocol mechanism to inform the sender. + _ = e.protocol.returnError(&icmpReasonPortUnreachable{}, pkt, true /* deliveredLocally */) + case stack.TransportPacketProtocolUnreachable: + // As per RFC: 1122 Section 3.2.2.1 + // A host SHOULD generate Destination Unreachable messages with code: + // 2 (Protocol Unreachable), when the designated transport protocol + // is not supported + _ = e.protocol.returnError(&icmpReasonProtoUnreachable{}, pkt, true /* deliveredLocally */) + default: + panic(fmt.Sprintf("unrecognized result from DeliverTransportPacket = %d", res)) + } +} + +// Close cleans up resources associated with the endpoint. +func (e *endpoint) Close() { + e.mu.Lock() + e.disableLocked() + e.addressableEndpointState.Cleanup() + e.mu.Unlock() + + e.protocol.forgetEndpoint(e.nic.ID()) +} + +// AddAndAcquirePermanentAddress implements stack.AddressableEndpoint. +func (e *endpoint) AddAndAcquirePermanentAddress(addr tcpip.AddressWithPrefix, properties stack.AddressProperties) (stack.AddressEndpoint, tcpip.Error) { + e.mu.Lock() + defer e.mu.Unlock() + + ep, err := e.addressableEndpointState.AddAndAcquireAddress(addr, properties, stack.Permanent) + if err == nil { + e.sendQueuedReports() + } + return ep, err +} + +// sendQueuedReports sends queued igmp reports. +// +// +checklocks:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) sendQueuedReports() { + e.igmp.sendQueuedReports() +} + +// RemovePermanentAddress implements stack.AddressableEndpoint. +func (e *endpoint) RemovePermanentAddress(addr tcpip.Address) tcpip.Error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addressableEndpointState.RemovePermanentAddress(addr) +} + +// SetDeprecated implements stack.AddressableEndpoint. +func (e *endpoint) SetDeprecated(addr tcpip.Address, deprecated bool) tcpip.Error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addressableEndpointState.SetDeprecated(addr, deprecated) +} + +// SetLifetimes implements stack.AddressableEndpoint. +func (e *endpoint) SetLifetimes(addr tcpip.Address, lifetimes stack.AddressLifetimes) tcpip.Error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addressableEndpointState.SetLifetimes(addr, lifetimes) +} + +// MainAddress implements stack.AddressableEndpoint. +func (e *endpoint) MainAddress() tcpip.AddressWithPrefix { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addressableEndpointState.MainAddress() +} + +// AcquireAssignedAddress implements stack.AddressableEndpoint. +func (e *endpoint) AcquireAssignedAddress(localAddr tcpip.Address, allowTemp bool, tempPEB stack.PrimaryEndpointBehavior, readOnly bool) stack.AddressEndpoint { + e.mu.RLock() + defer e.mu.RUnlock() + + loopback := e.nic.IsLoopback() + return e.addressableEndpointState.AcquireAssignedAddressOrMatching(localAddr, func(addressEndpoint stack.AddressEndpoint) bool { + subnet := addressEndpoint.Subnet() + // IPv4 has a notion of a subnet broadcast address and considers the + // loopback interface bound to an address's whole subnet (on linux). + return subnet.IsBroadcast(localAddr) || (loopback && subnet.Contains(localAddr)) + }, allowTemp, tempPEB, readOnly) +} + +// AcquireOutgoingPrimaryAddress implements stack.AddressableEndpoint. +func (e *endpoint) AcquireOutgoingPrimaryAddress(remoteAddr, srcHint tcpip.Address, allowExpired bool) stack.AddressEndpoint { + e.mu.RLock() + defer e.mu.RUnlock() + return e.acquireOutgoingPrimaryAddressRLocked(remoteAddr, srcHint, allowExpired) +} + +// acquireOutgoingPrimaryAddressRLocked is like AcquireOutgoingPrimaryAddress +// but with locking requirements +// +// +checklocksread:e.mu +func (e *endpoint) acquireOutgoingPrimaryAddressRLocked(remoteAddr, srcHint tcpip.Address, allowExpired bool) stack.AddressEndpoint { + return e.addressableEndpointState.AcquireOutgoingPrimaryAddress(remoteAddr, srcHint, allowExpired) +} + +// PrimaryAddresses implements stack.AddressableEndpoint. +func (e *endpoint) PrimaryAddresses() []tcpip.AddressWithPrefix { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addressableEndpointState.PrimaryAddresses() +} + +// PermanentAddresses implements stack.AddressableEndpoint. +func (e *endpoint) PermanentAddresses() []tcpip.AddressWithPrefix { + e.mu.RLock() + defer e.mu.RUnlock() + return e.addressableEndpointState.PermanentAddresses() +} + +// JoinGroup implements stack.GroupAddressableEndpoint. +func (e *endpoint) JoinGroup(addr tcpip.Address) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + return e.joinGroupLocked(addr) +} + +// joinGroupLocked is like JoinGroup but with locking requirements. +// +// +checklocks:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) joinGroupLocked(addr tcpip.Address) tcpip.Error { + if !header.IsV4MulticastAddress(addr) { + return &tcpip.ErrBadAddress{} + } + + e.igmp.joinGroup(addr) + return nil +} + +// LeaveGroup implements stack.GroupAddressableEndpoint. +func (e *endpoint) LeaveGroup(addr tcpip.Address) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + return e.leaveGroupLocked(addr) +} + +// leaveGroupLocked is like LeaveGroup but with locking requirements. +// +// +checklocks:e.mu +// +checklocksalias:e.igmp.ep.mu=e.mu +func (e *endpoint) leaveGroupLocked(addr tcpip.Address) tcpip.Error { + return e.igmp.leaveGroup(addr) +} + +// IsInGroup implements stack.GroupAddressableEndpoint. +func (e *endpoint) IsInGroup(addr tcpip.Address) bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.igmp.isInGroup(addr) // +checklocksforce: e.mu==e.igmp.ep.mu. +} + +// Stats implements stack.NetworkEndpoint. +func (e *endpoint) Stats() stack.NetworkEndpointStats { + return &e.stats.localStats +} + +var ( + _ stack.NetworkProtocol = (*protocol)(nil) + _ stack.MulticastForwardingNetworkProtocol = (*protocol)(nil) + _ stack.RejectIPv4WithHandler = (*protocol)(nil) + _ fragmentation.TimeoutHandler = (*protocol)(nil) +) + +// +stateify savable +type protocol struct { + stack *stack.Stack + + // mu protects annotated fields below. + mu sync.RWMutex `state:"nosave"` + + // eps is keyed by NICID to allow protocol methods to retrieve an endpoint + // when handling a packet, by looking at which NIC handled the packet. + // +checklocks:mu + eps map[tcpip.NICID]*endpoint + + // ICMP types for which the stack's global rate limiting must apply. + // +checklocks:mu + icmpRateLimitedTypes map[header.ICMPv4Type]struct{} + + // defaultTTL is the current default TTL for the protocol. Only the + // uint8 portion of it is meaningful. + defaultTTL atomicbitops.Uint32 + + ids []atomicbitops.Uint32 + hashIV uint32 + // idTS is the unix timestamp in milliseconds 'ids' was last accessed. + idTS atomicbitops.Int64 + + fragmentation *fragmentation.Fragmentation + + options Options + + multicastRouteTable multicast.RouteTable + // multicastForwardingDisp is the multicast forwarding event dispatcher that + // an integrator can provide to receive multicast forwarding events. Note + // that multicast packets will only be forwarded if this is non-nil. + // +checklocks:mu + multicastForwardingDisp stack.MulticastForwardingEventDispatcher +} + +// Number returns the ipv4 protocol number. +func (p *protocol) Number() tcpip.NetworkProtocolNumber { + return ProtocolNumber +} + +// MinimumPacketSize returns the minimum valid ipv4 packet size. +func (p *protocol) MinimumPacketSize() int { + return header.IPv4MinimumSize +} + +// ParseAddresses implements stack.NetworkProtocol. +func (*protocol) ParseAddresses(v []byte) (src, dst tcpip.Address) { + h := header.IPv4(v) + return h.SourceAddress(), h.DestinationAddress() +} + +// SetOption implements stack.NetworkProtocol. +func (p *protocol) SetOption(option tcpip.SettableNetworkProtocolOption) tcpip.Error { + switch v := option.(type) { + case *tcpip.DefaultTTLOption: + p.SetDefaultTTL(uint8(*v)) + return nil + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// Option implements stack.NetworkProtocol. +func (p *protocol) Option(option tcpip.GettableNetworkProtocolOption) tcpip.Error { + switch v := option.(type) { + case *tcpip.DefaultTTLOption: + *v = tcpip.DefaultTTLOption(p.DefaultTTL()) + return nil + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// SetDefaultTTL sets the default TTL for endpoints created with this protocol. +func (p *protocol) SetDefaultTTL(ttl uint8) { + p.defaultTTL.Store(uint32(ttl)) +} + +// DefaultTTL returns the default TTL for endpoints created with this protocol. +func (p *protocol) DefaultTTL() uint8 { + return uint8(p.defaultTTL.Load()) +} + +// Close implements stack.TransportProtocol. +func (p *protocol) Close() { + p.fragmentation.Release() + p.multicastRouteTable.Close() +} + +// Wait implements stack.TransportProtocol. +func (*protocol) Wait() {} + +func (p *protocol) validateUnicastSourceAndMulticastDestination(addresses stack.UnicastSourceAndMulticastDestination) tcpip.Error { + if !p.isUnicastAddress(addresses.Source) { + return &tcpip.ErrBadAddress{} + } + + if !header.IsV4MulticastAddress(addresses.Destination) { + return &tcpip.ErrBadAddress{} + } + + return nil +} + +func (p *protocol) multicastForwarding() bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.multicastForwardingDisp != nil +} + +func (p *protocol) newInstalledRoute(route stack.MulticastRoute) (*multicast.InstalledRoute, tcpip.Error) { + if len(route.OutgoingInterfaces) == 0 { + return nil, &tcpip.ErrMissingRequiredFields{} + } + + if !p.stack.HasNIC(route.ExpectedInputInterface) { + return nil, &tcpip.ErrUnknownNICID{} + } + + for _, outgoingInterface := range route.OutgoingInterfaces { + if route.ExpectedInputInterface == outgoingInterface.ID { + return nil, &tcpip.ErrMulticastInputCannotBeOutput{} + } + + if !p.stack.HasNIC(outgoingInterface.ID) { + return nil, &tcpip.ErrUnknownNICID{} + } + } + return p.multicastRouteTable.NewInstalledRoute(route), nil +} + +// AddMulticastRoute implements stack.MulticastForwardingNetworkProtocol. +func (p *protocol) AddMulticastRoute(addresses stack.UnicastSourceAndMulticastDestination, route stack.MulticastRoute) tcpip.Error { + if !p.multicastForwarding() { + return &tcpip.ErrNotPermitted{} + } + + if err := p.validateUnicastSourceAndMulticastDestination(addresses); err != nil { + return err + } + + installedRoute, err := p.newInstalledRoute(route) + if err != nil { + return err + } + + pendingPackets := p.multicastRouteTable.AddInstalledRoute(addresses, installedRoute) + + for _, pkt := range pendingPackets { + p.forwardPendingMulticastPacket(pkt, installedRoute) + } + return nil +} + +// RemoveMulticastRoute implements +// stack.MulticastForwardingNetworkProtocol.RemoveMulticastRoute. +func (p *protocol) RemoveMulticastRoute(addresses stack.UnicastSourceAndMulticastDestination) tcpip.Error { + if err := p.validateUnicastSourceAndMulticastDestination(addresses); err != nil { + return err + } + + if removed := p.multicastRouteTable.RemoveInstalledRoute(addresses); !removed { + return &tcpip.ErrHostUnreachable{} + } + + return nil +} + +// EnableMulticastForwarding implements +// stack.MulticastForwardingNetworkProtocol.EnableMulticastForwarding. +func (p *protocol) EnableMulticastForwarding(disp stack.MulticastForwardingEventDispatcher) (bool, tcpip.Error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.multicastForwardingDisp != nil { + return true, nil + } + + if disp == nil { + return false, &tcpip.ErrInvalidOptionValue{} + } + + p.multicastForwardingDisp = disp + return false, nil +} + +// DisableMulticastForwarding implements +// stack.MulticastForwardingNetworkProtocol.DisableMulticastForwarding. +func (p *protocol) DisableMulticastForwarding() { + p.mu.Lock() + defer p.mu.Unlock() + + p.multicastForwardingDisp = nil + p.multicastRouteTable.RemoveAllInstalledRoutes() +} + +// MulticastRouteLastUsedTime implements +// stack.MulticastForwardingNetworkProtocol. +func (p *protocol) MulticastRouteLastUsedTime(addresses stack.UnicastSourceAndMulticastDestination) (tcpip.MonotonicTime, tcpip.Error) { + if err := p.validateUnicastSourceAndMulticastDestination(addresses); err != nil { + return tcpip.MonotonicTime{}, err + } + + timestamp, found := p.multicastRouteTable.GetLastUsedTimestamp(addresses) + + if !found { + return tcpip.MonotonicTime{}, &tcpip.ErrHostUnreachable{} + } + + return timestamp, nil +} + +func (p *protocol) forwardPendingMulticastPacket(pkt *stack.PacketBuffer, installedRoute *multicast.InstalledRoute) { + defer pkt.DecRef() + + // Attempt to forward the packet using the endpoint that it originally + // arrived on. This ensures that the packet is only forwarded if it + // matches the route's expected input interface (see 5a of RFC 1812 section + // 5.2.1.3). + ep, ok := p.getEndpointForNIC(pkt.NICID) + + if !ok { + // The endpoint that the packet arrived on no longer exists. Silently + // drop the pkt. + return + } + + if !ep.MulticastForwarding() { + return + } + + ep.handleForwardingError(ep.forwardValidatedMulticastPacket(pkt, installedRoute)) +} + +func (p *protocol) isUnicastAddress(addr tcpip.Address) bool { + if addr.BitLen() != header.IPv4AddressSizeBits { + return false + } + + if addr == header.IPv4Any || addr == header.IPv4Broadcast { + return false + } + + if p.isSubnetLocalBroadcastAddress(addr) { + return false + } + return !header.IsV4MulticastAddress(addr) +} + +func (p *protocol) isSubnetLocalBroadcastAddress(addr tcpip.Address) bool { + p.mu.RLock() + defer p.mu.RUnlock() + + for _, e := range p.eps { + if addressEndpoint := e.AcquireAssignedAddress(addr, false /* createTemp */, stack.NeverPrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + subnet := addressEndpoint.Subnet() + if subnet.IsBroadcast(addr) { + return true + } + } + } + return false +} + +// parseAndValidate parses the packet (including its transport layer header) and +// returns the parsed IP header. +// +// Returns true if the IP header was successfully parsed. +func (p *protocol) parseAndValidate(pkt *stack.PacketBuffer) (*buffer.View, bool) { + transProtoNum, hasTransportHdr, ok := p.Parse(pkt) + if !ok { + return nil, false + } + + h := header.IPv4(pkt.NetworkHeader().Slice()) + // Do not include the link header's size when calculating the size of the IP + // packet. + if !h.IsValid(pkt.Size() - len(pkt.LinkHeader().Slice())) { + return nil, false + } + + if !pkt.RXChecksumValidated && !h.IsChecksumValid() { + return nil, false + } + + if hasTransportHdr { + p.parseTransport(pkt, transProtoNum) + } + + return pkt.NetworkHeader().View(), true +} + +func (p *protocol) parseTransport(pkt *stack.PacketBuffer, transProtoNum tcpip.TransportProtocolNumber) { + if transProtoNum == header.ICMPv4ProtocolNumber { + // The transport layer will handle transport layer parsing errors. + _ = parse.ICMPv4(pkt) + return + } + + switch err := p.stack.ParsePacketBufferTransport(transProtoNum, pkt); err { + case stack.ParsedOK: + case stack.UnknownTransportProtocol, stack.TransportLayerParseError: + // The transport layer will handle unknown protocols and transport layer + // parsing errors. + default: + panic(fmt.Sprintf("unexpected error parsing transport header = %d", err)) + } +} + +// Parse implements stack.NetworkProtocol. +func (*protocol) Parse(pkt *stack.PacketBuffer) (proto tcpip.TransportProtocolNumber, hasTransportHdr bool, ok bool) { + if ok := parse.IPv4(pkt); !ok { + return 0, false, false + } + + ipHdr := header.IPv4(pkt.NetworkHeader().Slice()) + return ipHdr.TransportProtocol(), !ipHdr.More() && ipHdr.FragmentOffset() == 0, true +} + +// allowICMPReply reports whether an ICMP reply with provided type and code may +// be sent following the rate mask options and global ICMP rate limiter. +func (p *protocol) allowICMPReply(icmpType header.ICMPv4Type, code header.ICMPv4Code) bool { + // Mimic linux and never rate limit for PMTU discovery. + // https://github.com/torvalds/linux/blob/9e9fb7655ed585da8f468e29221f0ba194a5f613/net/ipv4/icmp.c#L288 + if icmpType == header.ICMPv4DstUnreachable && code == header.ICMPv4FragmentationNeeded { + return true + } + p.mu.RLock() + defer p.mu.RUnlock() + + if _, ok := p.icmpRateLimitedTypes[icmpType]; ok { + return p.stack.AllowICMPMessage() + } + return true +} + +// SendRejectionError implements stack.RejectIPv4WithHandler. +func (p *protocol) SendRejectionError(pkt *stack.PacketBuffer, rejectWith stack.RejectIPv4WithICMPType, inputHook bool) tcpip.Error { + switch rejectWith { + case stack.RejectIPv4WithICMPNetUnreachable: + return p.returnError(&icmpReasonNetworkUnreachable{}, pkt, inputHook) + case stack.RejectIPv4WithICMPHostUnreachable: + return p.returnError(&icmpReasonHostUnreachable{}, pkt, inputHook) + case stack.RejectIPv4WithICMPPortUnreachable: + return p.returnError(&icmpReasonPortUnreachable{}, pkt, inputHook) + case stack.RejectIPv4WithICMPNetProhibited: + return p.returnError(&icmpReasonNetworkProhibited{}, pkt, inputHook) + case stack.RejectIPv4WithICMPHostProhibited: + return p.returnError(&icmpReasonHostProhibited{}, pkt, inputHook) + case stack.RejectIPv4WithICMPAdminProhibited: + return p.returnError(&icmpReasonAdministrativelyProhibited{}, pkt, inputHook) + default: + panic(fmt.Sprintf("unhandled %[1]T = %[1]d", rejectWith)) + } +} + +// calculateNetworkMTU calculates the network-layer payload MTU based on the +// link-layer payload mtu. +func calculateNetworkMTU(linkMTU, networkHeaderSize uint32) (uint32, tcpip.Error) { + if linkMTU < header.IPv4MinimumMTU { + return 0, &tcpip.ErrInvalidEndpointState{} + } + + // As per RFC 791 section 3.1, an IPv4 header cannot exceed 60 bytes in + // length: + // The maximal internet header is 60 octets, and a typical internet header + // is 20 octets, allowing a margin for headers of higher level protocols. + if networkHeaderSize > header.IPv4MaximumHeaderSize { + return 0, &tcpip.ErrMalformedHeader{} + } + + networkMTU := linkMTU + if networkMTU > MaxTotalSize { + networkMTU = MaxTotalSize + } + + return networkMTU - networkHeaderSize, nil +} + +func packetMustBeFragmented(pkt *stack.PacketBuffer, networkMTU uint32) bool { + payload := len(pkt.TransportHeader().Slice()) + pkt.Data().Size() + return pkt.GSOOptions.Type == stack.GSONone && uint32(payload) > networkMTU +} + +// addressToUint32 translates an IPv4 address into its little endian uint32 +// representation. +// +// This function does the same thing as binary.LittleEndian.Uint32 but operates +// on a tcpip.Address (a string) without the need to convert it to a byte slice, +// which would cause an allocation. +func addressToUint32(addr tcpip.Address) uint32 { + addrBytes := addr.As4() + _ = addrBytes[3] // bounds check hint to compiler + return uint32(addrBytes[0]) | uint32(addrBytes[1])<<8 | uint32(addrBytes[2])<<16 | uint32(addrBytes[3])<<24 +} + +// hashRoute calculates a hash value for the given source/destination pair using +// the addresses, transport protocol number and a 32-bit number to generate the +// hash. +func hashRoute(srcAddr, dstAddr tcpip.Address, protocol tcpip.TransportProtocolNumber, hashIV uint32) uint32 { + a := addressToUint32(srcAddr) + b := addressToUint32(dstAddr) + return hash.Hash3Words(a, b, uint32(protocol), hashIV) +} + +// Options holds options to configure a new protocol. +// +// +stateify savable +type Options struct { + // IGMP holds options for IGMP. + IGMP IGMPOptions + + // AllowExternalLoopbackTraffic indicates that inbound loopback packets (i.e. + // martian loopback packets) should be accepted. + AllowExternalLoopbackTraffic bool +} + +// NewProtocolWithOptions returns an IPv4 network protocol. +func NewProtocolWithOptions(opts Options) stack.NetworkProtocolFactory { + ids := make([]atomicbitops.Uint32, buckets) + + // Randomly initialize hashIV and the ids. + r := hash.RandN32(1 + buckets) + for i := range ids { + ids[i] = atomicbitops.FromUint32(r[i]) + } + hashIV := r[buckets] + + return func(s *stack.Stack) stack.NetworkProtocol { + p := &protocol{ + stack: s, + ids: ids, + hashIV: hashIV, + defaultTTL: atomicbitops.FromUint32(DefaultTTL), + options: opts, + } + p.fragmentation = fragmentation.NewFragmentation(fragmentblockSize, fragmentation.HighFragThreshold, fragmentation.LowFragThreshold, ReassembleTimeout, s.Clock(), p) + p.eps = make(map[tcpip.NICID]*endpoint) + // Set ICMP rate limiting to Linux defaults. + // See https://man7.org/linux/man-pages/man7/icmp.7.html. + p.icmpRateLimitedTypes = map[header.ICMPv4Type]struct{}{ + header.ICMPv4DstUnreachable: {}, + header.ICMPv4SrcQuench: {}, + header.ICMPv4TimeExceeded: {}, + header.ICMPv4ParamProblem: {}, + } + if err := p.multicastRouteTable.Init(multicast.DefaultConfig(s.Clock())); err != nil { + panic(fmt.Sprintf("p.multicastRouteTable.Init(_): %s", err)) + } + return p + } +} + +// NewProtocol is equivalent to NewProtocolWithOptions with an empty Options. +func NewProtocol(s *stack.Stack) stack.NetworkProtocol { + return NewProtocolWithOptions(Options{})(s) +} + +func buildNextFragment(pf *fragmentation.PacketFragmenter, originalIPHeader header.IPv4) (*stack.PacketBuffer, bool) { + fragPkt, offset, copied, more := pf.BuildNextFragment() + fragPkt.NetworkProtocolNumber = ProtocolNumber + + originalIPHeaderLength := len(originalIPHeader) + nextFragIPHeader := header.IPv4(fragPkt.NetworkHeader().Push(originalIPHeaderLength)) + fragPkt.NetworkProtocolNumber = ProtocolNumber + + if copied := copy(nextFragIPHeader, originalIPHeader); copied != len(originalIPHeader) { + panic(fmt.Sprintf("wrong number of bytes copied into fragmentIPHeaders: got = %d, want = %d", copied, originalIPHeaderLength)) + } + + flags := originalIPHeader.Flags() + if more { + flags |= header.IPv4FlagMoreFragments + } + nextFragIPHeader.SetFlagsFragmentOffset(flags, uint16(offset)) + nextFragIPHeader.SetTotalLength(uint16(nextFragIPHeader.HeaderLength()) + uint16(copied)) + nextFragIPHeader.SetChecksum(0) + nextFragIPHeader.SetChecksum(^nextFragIPHeader.CalculateChecksum()) + + return fragPkt, more +} + +// optionAction describes possible actions that may be taken on an option +// while processing it. +type optionAction uint8 + +const ( + // optionRemove says that the option should not be in the output option set. + optionRemove optionAction = iota + + // optionProcess says that the option should be fully processed. + optionProcess + + // optionVerify says the option should be checked and passed unchanged. + optionVerify + + // optionPass says to pass the output set without checking. + optionPass +) + +// optionActions list what to do for each option in a given scenario. +type optionActions struct { + // timestamp controls what to do with a Timestamp option. + timestamp optionAction + + // recordRoute controls what to do with a Record Route option. + recordRoute optionAction + + // routerAlert controls what to do with a Router Alert option. + routerAlert optionAction + + // unknown controls what to do with an unknown option. + unknown optionAction +} + +// optionsUsage specifies the ways options may be operated upon for a given +// scenario during packet processing. +type optionsUsage interface { + actions() optionActions +} + +// optionUsageVerify implements optionsUsage for when we just want to check +// fragments. Don't change anything, just check and reject if bad. No +// replacement options are generated. +type optionUsageVerify struct{} + +// actions implements optionsUsage. +func (*optionUsageVerify) actions() optionActions { + return optionActions{ + timestamp: optionVerify, + recordRoute: optionVerify, + routerAlert: optionVerify, + unknown: optionRemove, + } +} + +// optionUsageReceive implements optionsUsage for packets we will pass +// to the transport layer (with the exception of Echo requests). +type optionUsageReceive struct{} + +// actions implements optionsUsage. +func (*optionUsageReceive) actions() optionActions { + return optionActions{ + timestamp: optionProcess, + recordRoute: optionProcess, + routerAlert: optionVerify, + unknown: optionPass, + } +} + +// optionUsageForward implements optionsUsage for packets about to be forwarded. +// All options are passed on regardless of whether we recognise them, however +// we do process the Timestamp and Record Route options. +type optionUsageForward struct{} + +// actions implements optionsUsage. +func (*optionUsageForward) actions() optionActions { + return optionActions{ + timestamp: optionProcess, + recordRoute: optionProcess, + routerAlert: optionVerify, + unknown: optionPass, + } +} + +// optionUsageEcho implements optionsUsage for echo packet processing. +// Only Timestamp and RecordRoute are processed and sent back. +type optionUsageEcho struct{} + +// actions implements optionsUsage. +func (*optionUsageEcho) actions() optionActions { + return optionActions{ + timestamp: optionProcess, + recordRoute: optionProcess, + routerAlert: optionVerify, + unknown: optionRemove, + } +} + +// handleTimestamp does any required processing on a Timestamp option +// in place. +func handleTimestamp(tsOpt header.IPv4OptionTimestamp, localAddress tcpip.Address, clock tcpip.Clock, usage optionsUsage) *header.IPv4OptParameterProblem { + flags := tsOpt.Flags() + var entrySize uint8 + switch flags { + case header.IPv4OptionTimestampOnlyFlag: + entrySize = header.IPv4OptionTimestampSize + case + header.IPv4OptionTimestampWithIPFlag, + header.IPv4OptionTimestampWithPredefinedIPFlag: + entrySize = header.IPv4OptionTimestampWithAddrSize + default: + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptTSOFLWAndFLGOffset, + NeedICMP: true, + } + } + + pointer := tsOpt.Pointer() + // RFC 791 page 22 states: "The smallest legal value is 5." + // Since the pointer is 1 based, and the header is 4 bytes long the + // pointer must point beyond the header therefore 4 or less is bad. + if pointer <= header.IPv4OptionTimestampHdrLength { + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptTSPointerOffset, + NeedICMP: true, + } + } + // To simplify processing below, base further work on the array of timestamps + // beyond the header, rather than on the whole option. Also to aid + // calculations set 'nextSlot' to be 0 based as in the packet it is 1 based. + nextSlot := pointer - (header.IPv4OptionTimestampHdrLength + 1) + optLen := tsOpt.Size() + dataLength := optLen - header.IPv4OptionTimestampHdrLength + + // In the section below, we verify the pointer, length and overflow counter + // fields of the option. The distinction is in which byte you return as being + // in error in the ICMP packet. Offsets 1 (length), 2 pointer) + // or 3 (overflowed counter). + // + // The following RFC sections cover this section: + // + // RFC 791 (page 22): + // If there is some room but not enough room for a full timestamp + // to be inserted, or the overflow count itself overflows, the + // original datagram is considered to be in error and is discarded. + // In either case an ICMP parameter problem message may be sent to + // the source host [3]. + // + // You can get this situation in two ways. Firstly if the data area is not + // a multiple of the entry size or secondly, if the pointer is not at a + // multiple of the entry size. The wording of the RFC suggests that + // this is not an error until you actually run out of space. + if pointer > optLen { + // RFC 791 (page 22) says we should switch to using the overflow count. + // If the timestamp data area is already full (the pointer exceeds + // the length) the datagram is forwarded without inserting the + // timestamp, but the overflow count is incremented by one. + if flags == header.IPv4OptionTimestampWithPredefinedIPFlag { + // By definition we have nothing to do. + return nil + } + + if tsOpt.IncOverflow() != 0 { + return nil + } + // The overflow count is also full. + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptTSOFLWAndFLGOffset, + NeedICMP: true, + } + } + if nextSlot+entrySize > dataLength { + // The data area isn't full but there isn't room for a new entry. + // Either Length or Pointer could be bad. + if false { + // We must select Pointer for Linux compatibility, even if + // only the length is bad. + // The Linux code is at (in October 2020) + // https://github.com/torvalds/linux/blob/bbf5c979011a099af5dc76498918ed7df445635b/net/ipv4/ip_options.c#L367-L370 + // if (optptr[2]+3 > optlen) { + // pp_ptr = optptr + 2; + // goto error; + // } + // which doesn't distinguish between which of optptr[2] or optlen + // is wrong, but just arbitrarily decides on optptr+2. + if dataLength%entrySize != 0 { + // The Data section size should be a multiple of the expected + // timestamp entry size. + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptionLengthOffset, + NeedICMP: false, + } + } + // If the size is OK, the pointer must be corrupted. + } + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptTSPointerOffset, + NeedICMP: true, + } + } + + if usage.actions().timestamp == optionProcess { + tsOpt.UpdateTimestamp(localAddress, clock) + } + return nil +} + +// handleRecordRoute checks and processes a Record route option. It is much +// like the timestamp type 1 option, but without timestamps. The passed in +// address is stored in the option in the correct spot if possible. +func handleRecordRoute(rrOpt header.IPv4OptionRecordRoute, localAddress tcpip.Address, usage optionsUsage) *header.IPv4OptParameterProblem { + optlen := rrOpt.Size() + + if optlen < header.IPv4AddressSize+header.IPv4OptionRecordRouteHdrLength { + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptionLengthOffset, + NeedICMP: true, + } + } + + pointer := rrOpt.Pointer() + // RFC 791 page 20 states: + // The pointer is relative to this option, and the + // smallest legal value for the pointer is 4. + // Since the pointer is 1 based, and the header is 3 bytes long the + // pointer must point beyond the header therefore 3 or less is bad. + if pointer <= header.IPv4OptionRecordRouteHdrLength { + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptRRPointerOffset, + NeedICMP: true, + } + } + + // RFC 791 page 21 says + // If the route data area is already full (the pointer exceeds the + // length) the datagram is forwarded without inserting the address + // into the recorded route. If there is some room but not enough + // room for a full address to be inserted, the original datagram is + // considered to be in error and is discarded. In either case an + // ICMP parameter problem message may be sent to the source + // host. + // The use of the words "In either case" suggests that a 'full' RR option + // could generate an ICMP at every hop after it fills up. We chose to not + // do this (as do most implementations). It is probable that the inclusion + // of these words is a copy/paste error from the timestamp option where + // there are two failure reasons given. + if pointer > optlen { + return nil + } + + // The data area isn't full but there isn't room for a new entry. + // Either Length or Pointer could be bad. We must select Pointer for Linux + // compatibility, even if only the length is bad. NB. pointer is 1 based. + if pointer+header.IPv4AddressSize > optlen+1 { + if false { + // This is what we would do if we were not being Linux compatible. + // Check for bad pointer or length value. Must be a multiple of 4 after + // accounting for the 3 byte header and not within that header. + // RFC 791, page 20 says: + // The pointer is relative to this option, and the + // smallest legal value for the pointer is 4. + // + // A recorded route is composed of a series of internet addresses. + // Each internet address is 32 bits or 4 octets. + // Linux skips this test so we must too. See Linux code at: + // https://github.com/torvalds/linux/blob/bbf5c979011a099af5dc76498918ed7df445635b/net/ipv4/ip_options.c#L338-L341 + // if (optptr[2]+3 > optlen) { + // pp_ptr = optptr + 2; + // goto error; + // } + if (optlen-header.IPv4OptionRecordRouteHdrLength)%header.IPv4AddressSize != 0 { + // Length is bad, not on integral number of slots. + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptionLengthOffset, + NeedICMP: true, + } + } + // If not length, the fault must be with the pointer. + } + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptRRPointerOffset, + NeedICMP: true, + } + } + if usage.actions().recordRoute == optionVerify { + return nil + } + rrOpt.StoreAddress(localAddress) + return nil +} + +// handleRouterAlert performs sanity checks on a Router Alert option. +func handleRouterAlert(raOpt header.IPv4OptionRouterAlert) *header.IPv4OptParameterProblem { + // Only the zero value is acceptable, as per RFC 2113, section 2.1: + // Value: A two octet code with the following values: + // 0 - Router shall examine packet + // 1-65535 - Reserved + if raOpt.Value() != header.IPv4OptionRouterAlertValue { + return &header.IPv4OptParameterProblem{ + Pointer: header.IPv4OptionRouterAlertValueOffset, + NeedICMP: true, + } + } + return nil +} + +type optionTracker struct { + timestamp bool + recordRoute bool + routerAlert bool +} + +// processIPOptions parses the IPv4 options and produces a new set of options +// suitable for use in the next step of packet processing as informed by usage. +// The original will not be touched. +// +// If there were no errors during parsing, the new set of options is returned as +// a new buffer. +func (e *endpoint) processIPOptions(pkt *stack.PacketBuffer, opts header.IPv4Options, usage optionsUsage) (header.IPv4Options, optionTracker, *header.IPv4OptParameterProblem) { + stats := e.stats.ip + optIter := opts.MakeIterator() + + // Except NOP, each option must only appear at most once (RFC 791 section 3.1, + // at the definition of every type). + // Keep track of each option we find to enable duplicate option detection. + var seenOptions [math.MaxUint8 + 1]bool + + // TODO(https://gvisor.dev/issue/4586): This will need tweaking when we start + // really forwarding packets as we may need to get two addresses, for rx and + // tx interfaces. We will also have to take usage into account. + localAddress := e.MainAddress().Address + if localAddress.BitLen() == 0 { + h := header.IPv4(pkt.NetworkHeader().Slice()) + dstAddr := h.DestinationAddress() + if pkt.NetworkPacketInfo.LocalAddressBroadcast || header.IsV4MulticastAddress(dstAddr) { + return nil, optionTracker{}, &header.IPv4OptParameterProblem{ + NeedICMP: false, + } + } + localAddress = dstAddr + } + + var optionsProcessed optionTracker + for { + option, done, optProblem := optIter.Next() + if done || optProblem != nil { + return optIter.Finalize(), optionsProcessed, optProblem + } + optType := option.Type() + if optType == header.IPv4OptionNOPType { + optIter.PushNOPOrEnd(optType) + continue + } + if optType == header.IPv4OptionListEndType { + optIter.PushNOPOrEnd(optType) + return optIter.Finalize(), optionsProcessed, nil + } + + // check for repeating options (multiple NOPs are OK) + if seenOptions[optType] { + return nil, optionTracker{}, &header.IPv4OptParameterProblem{ + Pointer: optIter.ErrCursor, + NeedICMP: true, + } + } + seenOptions[optType] = true + + optLen, optProblem := func() (int, *header.IPv4OptParameterProblem) { + switch option := option.(type) { + case *header.IPv4OptionTimestamp: + stats.OptionTimestampReceived.Increment() + optionsProcessed.timestamp = true + if usage.actions().timestamp != optionRemove { + clock := e.protocol.stack.Clock() + newBuffer := optIter.InitReplacement(option) + optProblem := handleTimestamp(header.IPv4OptionTimestamp(newBuffer), localAddress, clock, usage) + return len(newBuffer), optProblem + } + + case *header.IPv4OptionRecordRoute: + stats.OptionRecordRouteReceived.Increment() + optionsProcessed.recordRoute = true + if usage.actions().recordRoute != optionRemove { + newBuffer := optIter.InitReplacement(option) + optProblem := handleRecordRoute(header.IPv4OptionRecordRoute(newBuffer), localAddress, usage) + return len(newBuffer), optProblem + } + + case *header.IPv4OptionRouterAlert: + stats.OptionRouterAlertReceived.Increment() + optionsProcessed.routerAlert = true + if usage.actions().routerAlert != optionRemove { + newBuffer := optIter.InitReplacement(option) + optProblem := handleRouterAlert(header.IPv4OptionRouterAlert(newBuffer)) + return len(newBuffer), optProblem + } + + default: + stats.OptionUnknownReceived.Increment() + if usage.actions().unknown == optionPass { + return len(optIter.InitReplacement(option)), nil + } + } + return 0, nil + }() + + if optProblem != nil { + optProblem.Pointer += optIter.ErrCursor + return nil, optionTracker{}, optProblem + } + optIter.ConsumeBuffer(optLen) + } +} diff --git a/pkg/tcpip/network/ipv4/ipv4_export.go b/pkg/tcpip/network/ipv4/ipv4_export.go new file mode 100644 index 0000000..ca5c882 --- /dev/null +++ b/pkg/tcpip/network/ipv4/ipv4_export.go @@ -0,0 +1,14 @@ +package ipv4 + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +type ExportedEndpoint interface { + WritePacketDirect(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error +} + +func (e *endpoint) WritePacketDirect(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error { + return e.writePacket(r, pkt) +} diff --git a/pkg/tcpip/network/ipv4/ipv4_state_autogen.go b/pkg/tcpip/network/ipv4/ipv4_state_autogen.go new file mode 100644 index 0000000..6204211 --- /dev/null +++ b/pkg/tcpip/network/ipv4/ipv4_state_autogen.go @@ -0,0 +1,785 @@ +// automatically generated by stateify. + +package ipv4 + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (i *icmpv4DestinationUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4DestinationUnreachableSockError" +} + +func (i *icmpv4DestinationUnreachableSockError) StateFields() []string { + return []string{} +} + +func (i *icmpv4DestinationUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4DestinationUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() +} + +func (i *icmpv4DestinationUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4DestinationUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (i *icmpv4DestinationHostUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4DestinationHostUnreachableSockError" +} + +func (i *icmpv4DestinationHostUnreachableSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4DestinationHostUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4DestinationHostUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationHostUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4DestinationHostUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationNetUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4DestinationNetUnreachableSockError" +} + +func (i *icmpv4DestinationNetUnreachableSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4DestinationNetUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4DestinationNetUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationNetUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4DestinationNetUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationPortUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4DestinationPortUnreachableSockError" +} + +func (i *icmpv4DestinationPortUnreachableSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4DestinationPortUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4DestinationPortUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationPortUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4DestinationPortUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationProtoUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4DestinationProtoUnreachableSockError" +} + +func (i *icmpv4DestinationProtoUnreachableSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4DestinationProtoUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4DestinationProtoUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationProtoUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4DestinationProtoUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4SourceRouteFailedSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4SourceRouteFailedSockError" +} + +func (i *icmpv4SourceRouteFailedSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4SourceRouteFailedSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4SourceRouteFailedSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4SourceRouteFailedSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4SourceRouteFailedSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4SourceHostIsolatedSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4SourceHostIsolatedSockError" +} + +func (i *icmpv4SourceHostIsolatedSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4SourceHostIsolatedSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4SourceHostIsolatedSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4SourceHostIsolatedSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4SourceHostIsolatedSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationHostUnknownSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4DestinationHostUnknownSockError" +} + +func (i *icmpv4DestinationHostUnknownSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + } +} + +func (i *icmpv4DestinationHostUnknownSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv4DestinationHostUnknownSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (i *icmpv4DestinationHostUnknownSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv4DestinationHostUnknownSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv4DestinationUnreachableSockError) +} + +func (e *icmpv4FragmentationNeededSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv4.icmpv4FragmentationNeededSockError" +} + +func (e *icmpv4FragmentationNeededSockError) StateFields() []string { + return []string{ + "icmpv4DestinationUnreachableSockError", + "mtu", + } +} + +func (e *icmpv4FragmentationNeededSockError) beforeSave() {} + +// +checklocksignore +func (e *icmpv4FragmentationNeededSockError) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.icmpv4DestinationUnreachableSockError) + stateSinkObject.Save(1, &e.mtu) +} + +func (e *icmpv4FragmentationNeededSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (e *icmpv4FragmentationNeededSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.icmpv4DestinationUnreachableSockError) + stateSourceObject.Load(1, &e.mtu) +} + +func (i *IGMPOptions) StateTypeName() string { + return "pkg/tcpip/network/ipv4.IGMPOptions" +} + +func (i *IGMPOptions) StateFields() []string { + return []string{ + "Enabled", + } +} + +func (i *IGMPOptions) beforeSave() {} + +// +checklocksignore +func (i *IGMPOptions) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Enabled) +} + +func (i *IGMPOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IGMPOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Enabled) +} + +func (igmp *igmpState) StateTypeName() string { + return "pkg/tcpip/network/ipv4.igmpState" +} + +func (igmp *igmpState) StateFields() []string { + return []string{ + "ep", + "genericMulticastProtocol", + "mode", + "igmpV1Job", + } +} + +func (igmp *igmpState) beforeSave() {} + +// +checklocksignore +func (igmp *igmpState) StateSave(stateSinkObject state.Sink) { + igmp.beforeSave() + stateSinkObject.Save(0, &igmp.ep) + stateSinkObject.Save(1, &igmp.genericMulticastProtocol) + stateSinkObject.Save(2, &igmp.mode) + stateSinkObject.Save(3, &igmp.igmpV1Job) +} + +func (igmp *igmpState) afterLoad(context.Context) {} + +// +checklocksignore +func (igmp *igmpState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &igmp.ep) + stateSourceObject.Load(1, &igmp.genericMulticastProtocol) + stateSourceObject.Load(2, &igmp.mode) + stateSourceObject.Load(3, &igmp.igmpV1Job) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/network/ipv4.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "nic", + "dispatcher", + "protocol", + "stats", + "enabled", + "forwarding", + "multicastForwarding", + "addressableEndpointState", + "igmp", + } +} + +func (e *endpoint) beforeSave() {} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.nic) + stateSinkObject.Save(1, &e.dispatcher) + stateSinkObject.Save(2, &e.protocol) + stateSinkObject.Save(3, &e.stats) + stateSinkObject.Save(4, &e.enabled) + stateSinkObject.Save(5, &e.forwarding) + stateSinkObject.Save(6, &e.multicastForwarding) + stateSinkObject.Save(7, &e.addressableEndpointState) + stateSinkObject.Save(8, &e.igmp) +} + +func (e *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.nic) + stateSourceObject.Load(1, &e.dispatcher) + stateSourceObject.Load(2, &e.protocol) + stateSourceObject.Load(3, &e.stats) + stateSourceObject.Load(4, &e.enabled) + stateSourceObject.Load(5, &e.forwarding) + stateSourceObject.Load(6, &e.multicastForwarding) + stateSourceObject.Load(7, &e.addressableEndpointState) + stateSourceObject.Load(8, &e.igmp) +} + +func (p *protocol) StateTypeName() string { + return "pkg/tcpip/network/ipv4.protocol" +} + +func (p *protocol) StateFields() []string { + return []string{ + "stack", + "eps", + "icmpRateLimitedTypes", + "defaultTTL", + "ids", + "hashIV", + "idTS", + "fragmentation", + "options", + "multicastRouteTable", + "multicastForwardingDisp", + } +} + +func (p *protocol) beforeSave() {} + +// +checklocksignore +func (p *protocol) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.stack) + stateSinkObject.Save(1, &p.eps) + stateSinkObject.Save(2, &p.icmpRateLimitedTypes) + stateSinkObject.Save(3, &p.defaultTTL) + stateSinkObject.Save(4, &p.ids) + stateSinkObject.Save(5, &p.hashIV) + stateSinkObject.Save(6, &p.idTS) + stateSinkObject.Save(7, &p.fragmentation) + stateSinkObject.Save(8, &p.options) + stateSinkObject.Save(9, &p.multicastRouteTable) + stateSinkObject.Save(10, &p.multicastForwardingDisp) +} + +func (p *protocol) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.stack) + stateSourceObject.Load(1, &p.eps) + stateSourceObject.Load(2, &p.icmpRateLimitedTypes) + stateSourceObject.Load(3, &p.defaultTTL) + stateSourceObject.Load(4, &p.ids) + stateSourceObject.Load(5, &p.hashIV) + stateSourceObject.Load(6, &p.idTS) + stateSourceObject.Load(7, &p.fragmentation) + stateSourceObject.Load(8, &p.options) + stateSourceObject.Load(9, &p.multicastRouteTable) + stateSourceObject.Load(10, &p.multicastForwardingDisp) +} + +func (o *Options) StateTypeName() string { + return "pkg/tcpip/network/ipv4.Options" +} + +func (o *Options) StateFields() []string { + return []string{ + "IGMP", + "AllowExternalLoopbackTraffic", + } +} + +func (o *Options) beforeSave() {} + +// +checklocksignore +func (o *Options) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.IGMP) + stateSinkObject.Save(1, &o.AllowExternalLoopbackTraffic) +} + +func (o *Options) afterLoad(context.Context) {} + +// +checklocksignore +func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.IGMP) + stateSourceObject.Load(1, &o.AllowExternalLoopbackTraffic) +} + +func (s *Stats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.Stats" +} + +func (s *Stats) StateFields() []string { + return []string{ + "IP", + "IGMP", + "ICMP", + } +} + +func (s *Stats) beforeSave() {} + +// +checklocksignore +func (s *Stats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.IP) + stateSinkObject.Save(1, &s.IGMP) + stateSinkObject.Save(2, &s.ICMP) +} + +func (s *Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.IP) + stateSourceObject.Load(1, &s.IGMP) + stateSourceObject.Load(2, &s.ICMP) +} + +func (s *sharedStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.sharedStats" +} + +func (s *sharedStats) StateFields() []string { + return []string{ + "localStats", + "ip", + "icmp", + "igmp", + } +} + +func (s *sharedStats) beforeSave() {} + +// +checklocksignore +func (s *sharedStats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.localStats) + stateSinkObject.Save(1, &s.ip) + stateSinkObject.Save(2, &s.icmp) + stateSinkObject.Save(3, &s.igmp) +} + +func (s *sharedStats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *sharedStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.localStats) + stateSourceObject.Load(1, &s.ip) + stateSourceObject.Load(2, &s.icmp) + stateSourceObject.Load(3, &s.igmp) +} + +func (m *multiCounterICMPv4PacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterICMPv4PacketStats" +} + +func (m *multiCounterICMPv4PacketStats) StateFields() []string { + return []string{ + "echoRequest", + "echoReply", + "dstUnreachable", + "srcQuench", + "redirect", + "timeExceeded", + "paramProblem", + "timestamp", + "timestampReply", + "infoRequest", + "infoReply", + } +} + +func (m *multiCounterICMPv4PacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv4PacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.echoRequest) + stateSinkObject.Save(1, &m.echoReply) + stateSinkObject.Save(2, &m.dstUnreachable) + stateSinkObject.Save(3, &m.srcQuench) + stateSinkObject.Save(4, &m.redirect) + stateSinkObject.Save(5, &m.timeExceeded) + stateSinkObject.Save(6, &m.paramProblem) + stateSinkObject.Save(7, &m.timestamp) + stateSinkObject.Save(8, &m.timestampReply) + stateSinkObject.Save(9, &m.infoRequest) + stateSinkObject.Save(10, &m.infoReply) +} + +func (m *multiCounterICMPv4PacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv4PacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.echoRequest) + stateSourceObject.Load(1, &m.echoReply) + stateSourceObject.Load(2, &m.dstUnreachable) + stateSourceObject.Load(3, &m.srcQuench) + stateSourceObject.Load(4, &m.redirect) + stateSourceObject.Load(5, &m.timeExceeded) + stateSourceObject.Load(6, &m.paramProblem) + stateSourceObject.Load(7, &m.timestamp) + stateSourceObject.Load(8, &m.timestampReply) + stateSourceObject.Load(9, &m.infoRequest) + stateSourceObject.Load(10, &m.infoReply) +} + +func (m *multiCounterICMPv4SentPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterICMPv4SentPacketStats" +} + +func (m *multiCounterICMPv4SentPacketStats) StateFields() []string { + return []string{ + "multiCounterICMPv4PacketStats", + "dropped", + "rateLimited", + } +} + +func (m *multiCounterICMPv4SentPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv4SentPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.multiCounterICMPv4PacketStats) + stateSinkObject.Save(1, &m.dropped) + stateSinkObject.Save(2, &m.rateLimited) +} + +func (m *multiCounterICMPv4SentPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv4SentPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.multiCounterICMPv4PacketStats) + stateSourceObject.Load(1, &m.dropped) + stateSourceObject.Load(2, &m.rateLimited) +} + +func (m *multiCounterICMPv4ReceivedPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterICMPv4ReceivedPacketStats" +} + +func (m *multiCounterICMPv4ReceivedPacketStats) StateFields() []string { + return []string{ + "multiCounterICMPv4PacketStats", + "invalid", + } +} + +func (m *multiCounterICMPv4ReceivedPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv4ReceivedPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.multiCounterICMPv4PacketStats) + stateSinkObject.Save(1, &m.invalid) +} + +func (m *multiCounterICMPv4ReceivedPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv4ReceivedPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.multiCounterICMPv4PacketStats) + stateSourceObject.Load(1, &m.invalid) +} + +func (m *multiCounterICMPv4Stats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterICMPv4Stats" +} + +func (m *multiCounterICMPv4Stats) StateFields() []string { + return []string{ + "packetsSent", + "packetsReceived", + } +} + +func (m *multiCounterICMPv4Stats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv4Stats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.packetsSent) + stateSinkObject.Save(1, &m.packetsReceived) +} + +func (m *multiCounterICMPv4Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv4Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.packetsSent) + stateSourceObject.Load(1, &m.packetsReceived) +} + +func (m *multiCounterIGMPPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterIGMPPacketStats" +} + +func (m *multiCounterIGMPPacketStats) StateFields() []string { + return []string{ + "membershipQuery", + "v1MembershipReport", + "v2MembershipReport", + "v3MembershipReport", + "leaveGroup", + } +} + +func (m *multiCounterIGMPPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterIGMPPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.membershipQuery) + stateSinkObject.Save(1, &m.v1MembershipReport) + stateSinkObject.Save(2, &m.v2MembershipReport) + stateSinkObject.Save(3, &m.v3MembershipReport) + stateSinkObject.Save(4, &m.leaveGroup) +} + +func (m *multiCounterIGMPPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterIGMPPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.membershipQuery) + stateSourceObject.Load(1, &m.v1MembershipReport) + stateSourceObject.Load(2, &m.v2MembershipReport) + stateSourceObject.Load(3, &m.v3MembershipReport) + stateSourceObject.Load(4, &m.leaveGroup) +} + +func (m *multiCounterIGMPSentPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterIGMPSentPacketStats" +} + +func (m *multiCounterIGMPSentPacketStats) StateFields() []string { + return []string{ + "multiCounterIGMPPacketStats", + "dropped", + } +} + +func (m *multiCounterIGMPSentPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterIGMPSentPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.multiCounterIGMPPacketStats) + stateSinkObject.Save(1, &m.dropped) +} + +func (m *multiCounterIGMPSentPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterIGMPSentPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.multiCounterIGMPPacketStats) + stateSourceObject.Load(1, &m.dropped) +} + +func (m *multiCounterIGMPReceivedPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterIGMPReceivedPacketStats" +} + +func (m *multiCounterIGMPReceivedPacketStats) StateFields() []string { + return []string{ + "multiCounterIGMPPacketStats", + "invalid", + "checksumErrors", + "unrecognized", + } +} + +func (m *multiCounterIGMPReceivedPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterIGMPReceivedPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.multiCounterIGMPPacketStats) + stateSinkObject.Save(1, &m.invalid) + stateSinkObject.Save(2, &m.checksumErrors) + stateSinkObject.Save(3, &m.unrecognized) +} + +func (m *multiCounterIGMPReceivedPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterIGMPReceivedPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.multiCounterIGMPPacketStats) + stateSourceObject.Load(1, &m.invalid) + stateSourceObject.Load(2, &m.checksumErrors) + stateSourceObject.Load(3, &m.unrecognized) +} + +func (m *multiCounterIGMPStats) StateTypeName() string { + return "pkg/tcpip/network/ipv4.multiCounterIGMPStats" +} + +func (m *multiCounterIGMPStats) StateFields() []string { + return []string{ + "packetsSent", + "packetsReceived", + } +} + +func (m *multiCounterIGMPStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterIGMPStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.packetsSent) + stateSinkObject.Save(1, &m.packetsReceived) +} + +func (m *multiCounterIGMPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterIGMPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.packetsSent) + stateSourceObject.Load(1, &m.packetsReceived) +} + +func init() { + state.Register((*icmpv4DestinationUnreachableSockError)(nil)) + state.Register((*icmpv4DestinationHostUnreachableSockError)(nil)) + state.Register((*icmpv4DestinationNetUnreachableSockError)(nil)) + state.Register((*icmpv4DestinationPortUnreachableSockError)(nil)) + state.Register((*icmpv4DestinationProtoUnreachableSockError)(nil)) + state.Register((*icmpv4SourceRouteFailedSockError)(nil)) + state.Register((*icmpv4SourceHostIsolatedSockError)(nil)) + state.Register((*icmpv4DestinationHostUnknownSockError)(nil)) + state.Register((*icmpv4FragmentationNeededSockError)(nil)) + state.Register((*IGMPOptions)(nil)) + state.Register((*igmpState)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*protocol)(nil)) + state.Register((*Options)(nil)) + state.Register((*Stats)(nil)) + state.Register((*sharedStats)(nil)) + state.Register((*multiCounterICMPv4PacketStats)(nil)) + state.Register((*multiCounterICMPv4SentPacketStats)(nil)) + state.Register((*multiCounterICMPv4ReceivedPacketStats)(nil)) + state.Register((*multiCounterICMPv4Stats)(nil)) + state.Register((*multiCounterIGMPPacketStats)(nil)) + state.Register((*multiCounterIGMPSentPacketStats)(nil)) + state.Register((*multiCounterIGMPReceivedPacketStats)(nil)) + state.Register((*multiCounterIGMPStats)(nil)) +} diff --git a/pkg/tcpip/network/ipv4/stats.go b/pkg/tcpip/network/ipv4/stats.go new file mode 100644 index 0000000..b2d4298 --- /dev/null +++ b/pkg/tcpip/network/ipv4/stats.go @@ -0,0 +1,203 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv4 + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var _ stack.IPNetworkEndpointStats = (*Stats)(nil) + +// Stats holds statistics related to the IPv4 protocol family. +// +// +stateify savable +type Stats struct { + // IP holds IPv4 statistics. + IP tcpip.IPStats + + // IGMP holds IGMP statistics. + IGMP tcpip.IGMPStats + + // ICMP holds ICMPv4 statistics. + ICMP tcpip.ICMPv4Stats +} + +// IsNetworkEndpointStats implements stack.NetworkEndpointStats. +func (*Stats) IsNetworkEndpointStats() {} + +// IPStats implements stack.IPNetworkEndointStats +func (s *Stats) IPStats() *tcpip.IPStats { + return &s.IP +} + +// +stateify savable +type sharedStats struct { + localStats Stats + ip ip.MultiCounterIPStats + icmp multiCounterICMPv4Stats + igmp multiCounterIGMPStats +} + +// LINT.IfChange(multiCounterICMPv4PacketStats) + +// +stateify savable +type multiCounterICMPv4PacketStats struct { + echoRequest tcpip.MultiCounterStat + echoReply tcpip.MultiCounterStat + dstUnreachable tcpip.MultiCounterStat + srcQuench tcpip.MultiCounterStat + redirect tcpip.MultiCounterStat + timeExceeded tcpip.MultiCounterStat + paramProblem tcpip.MultiCounterStat + timestamp tcpip.MultiCounterStat + timestampReply tcpip.MultiCounterStat + infoRequest tcpip.MultiCounterStat + infoReply tcpip.MultiCounterStat +} + +func (m *multiCounterICMPv4PacketStats) init(a, b *tcpip.ICMPv4PacketStats) { + m.echoRequest.Init(a.EchoRequest, b.EchoRequest) + m.echoReply.Init(a.EchoReply, b.EchoReply) + m.dstUnreachable.Init(a.DstUnreachable, b.DstUnreachable) + m.srcQuench.Init(a.SrcQuench, b.SrcQuench) + m.redirect.Init(a.Redirect, b.Redirect) + m.timeExceeded.Init(a.TimeExceeded, b.TimeExceeded) + m.paramProblem.Init(a.ParamProblem, b.ParamProblem) + m.timestamp.Init(a.Timestamp, b.Timestamp) + m.timestampReply.Init(a.TimestampReply, b.TimestampReply) + m.infoRequest.Init(a.InfoRequest, b.InfoRequest) + m.infoReply.Init(a.InfoReply, b.InfoReply) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv4PacketStats) + +// LINT.IfChange(multiCounterICMPv4SentPacketStats) + +// +stateify savable +type multiCounterICMPv4SentPacketStats struct { + multiCounterICMPv4PacketStats + dropped tcpip.MultiCounterStat + rateLimited tcpip.MultiCounterStat +} + +func (m *multiCounterICMPv4SentPacketStats) init(a, b *tcpip.ICMPv4SentPacketStats) { + m.multiCounterICMPv4PacketStats.init(&a.ICMPv4PacketStats, &b.ICMPv4PacketStats) + m.dropped.Init(a.Dropped, b.Dropped) + m.rateLimited.Init(a.RateLimited, b.RateLimited) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv4SentPacketStats) + +// LINT.IfChange(multiCounterICMPv4ReceivedPacketStats) + +// +stateify savable +type multiCounterICMPv4ReceivedPacketStats struct { + multiCounterICMPv4PacketStats + invalid tcpip.MultiCounterStat +} + +func (m *multiCounterICMPv4ReceivedPacketStats) init(a, b *tcpip.ICMPv4ReceivedPacketStats) { + m.multiCounterICMPv4PacketStats.init(&a.ICMPv4PacketStats, &b.ICMPv4PacketStats) + m.invalid.Init(a.Invalid, b.Invalid) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv4ReceivedPacketStats) + +// LINT.IfChange(multiCounterICMPv4Stats) + +// +stateify savable +type multiCounterICMPv4Stats struct { + packetsSent multiCounterICMPv4SentPacketStats + packetsReceived multiCounterICMPv4ReceivedPacketStats +} + +func (m *multiCounterICMPv4Stats) init(a, b *tcpip.ICMPv4Stats) { + m.packetsSent.init(&a.PacketsSent, &b.PacketsSent) + m.packetsReceived.init(&a.PacketsReceived, &b.PacketsReceived) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv4Stats) + +// LINT.IfChange(multiCounterIGMPPacketStats) + +// +stateify savable +type multiCounterIGMPPacketStats struct { + membershipQuery tcpip.MultiCounterStat + v1MembershipReport tcpip.MultiCounterStat + v2MembershipReport tcpip.MultiCounterStat + v3MembershipReport tcpip.MultiCounterStat + leaveGroup tcpip.MultiCounterStat +} + +func (m *multiCounterIGMPPacketStats) init(a, b *tcpip.IGMPPacketStats) { + m.membershipQuery.Init(a.MembershipQuery, b.MembershipQuery) + m.v1MembershipReport.Init(a.V1MembershipReport, b.V1MembershipReport) + m.v2MembershipReport.Init(a.V2MembershipReport, b.V2MembershipReport) + m.v3MembershipReport.Init(a.V3MembershipReport, b.V3MembershipReport) + m.leaveGroup.Init(a.LeaveGroup, b.LeaveGroup) +} + +// LINT.ThenChange(../../tcpip.go:IGMPPacketStats) + +// LINT.IfChange(multiCounterIGMPSentPacketStats) + +// +stateify savable +type multiCounterIGMPSentPacketStats struct { + multiCounterIGMPPacketStats + dropped tcpip.MultiCounterStat +} + +func (m *multiCounterIGMPSentPacketStats) init(a, b *tcpip.IGMPSentPacketStats) { + m.multiCounterIGMPPacketStats.init(&a.IGMPPacketStats, &b.IGMPPacketStats) + m.dropped.Init(a.Dropped, b.Dropped) +} + +// LINT.ThenChange(../../tcpip.go:IGMPSentPacketStats) + +// LINT.IfChange(multiCounterIGMPReceivedPacketStats) + +// +stateify savable +type multiCounterIGMPReceivedPacketStats struct { + multiCounterIGMPPacketStats + invalid tcpip.MultiCounterStat + checksumErrors tcpip.MultiCounterStat + unrecognized tcpip.MultiCounterStat +} + +func (m *multiCounterIGMPReceivedPacketStats) init(a, b *tcpip.IGMPReceivedPacketStats) { + m.multiCounterIGMPPacketStats.init(&a.IGMPPacketStats, &b.IGMPPacketStats) + m.invalid.Init(a.Invalid, b.Invalid) + m.checksumErrors.Init(a.ChecksumErrors, b.ChecksumErrors) + m.unrecognized.Init(a.Unrecognized, b.Unrecognized) +} + +// LINT.ThenChange(../../tcpip.go:IGMPReceivedPacketStats) + +// LINT.IfChange(multiCounterIGMPStats) + +// +stateify savable +type multiCounterIGMPStats struct { + packetsSent multiCounterIGMPSentPacketStats + packetsReceived multiCounterIGMPReceivedPacketStats +} + +func (m *multiCounterIGMPStats) init(a, b *tcpip.IGMPStats) { + m.packetsSent.init(&a.PacketsSent, &b.PacketsSent) + m.packetsReceived.init(&a.PacketsReceived, &b.PacketsReceived) +} + +// LINT.ThenChange(../../tcpip.go:IGMPStats) diff --git a/pkg/tcpip/network/ipv6/dhcpv6configurationfromndpra_string.go b/pkg/tcpip/network/ipv6/dhcpv6configurationfromndpra_string.go new file mode 100644 index 0000000..09ba133 --- /dev/null +++ b/pkg/tcpip/network/ipv6/dhcpv6configurationfromndpra_string.go @@ -0,0 +1,40 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by "stringer -type DHCPv6ConfigurationFromNDPRA"; DO NOT EDIT. + +package ipv6 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[DHCPv6NoConfiguration-1] + _ = x[DHCPv6ManagedAddress-2] + _ = x[DHCPv6OtherConfigurations-3] +} + +const _DHCPv6ConfigurationFromNDPRA_name = "DHCPv6NoConfigurationDHCPv6ManagedAddressDHCPv6OtherConfigurations" + +var _DHCPv6ConfigurationFromNDPRA_index = [...]uint8{0, 21, 41, 66} + +func (i DHCPv6ConfigurationFromNDPRA) String() string { + i -= 1 + if i < 0 || i >= DHCPv6ConfigurationFromNDPRA(len(_DHCPv6ConfigurationFromNDPRA_index)-1) { + return "DHCPv6ConfigurationFromNDPRA(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _DHCPv6ConfigurationFromNDPRA_name[_DHCPv6ConfigurationFromNDPRA_index[i]:_DHCPv6ConfigurationFromNDPRA_index[i+1]] +} diff --git a/pkg/tcpip/network/ipv6/icmp.go b/pkg/tcpip/network/ipv6/icmp.go new file mode 100644 index 0000000..f6e8f33 --- /dev/null +++ b/pkg/tcpip/network/ipv6/icmp.go @@ -0,0 +1,1184 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv6 + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// icmpv6DestinationUnreachableSockError is a general ICMPv6 Destination +// Unreachable error. +// +// +stateify savable +type icmpv6DestinationUnreachableSockError struct{} + +// Origin implements tcpip.SockErrorCause. +func (*icmpv6DestinationUnreachableSockError) Origin() tcpip.SockErrOrigin { + return tcpip.SockExtErrorOriginICMP6 +} + +// Type implements tcpip.SockErrorCause. +func (*icmpv6DestinationUnreachableSockError) Type() uint8 { + return uint8(header.ICMPv6DstUnreachable) +} + +// Info implements tcpip.SockErrorCause. +func (*icmpv6DestinationUnreachableSockError) Info() uint32 { + return 0 +} + +var _ stack.TransportError = (*icmpv6DestinationNetworkUnreachableSockError)(nil) + +// icmpv6DestinationNetworkUnreachableSockError is an ICMPv6 Destination Network +// Unreachable error. +// +// It indicates that the destination network is unreachable. +// +// +stateify savable +type icmpv6DestinationNetworkUnreachableSockError struct { + icmpv6DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv6DestinationNetworkUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv6NetworkUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv6DestinationNetworkUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationNetworkUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv6DestinationPortUnreachableSockError)(nil) + +// icmpv6DestinationPortUnreachableSockError is an ICMPv6 Destination Port +// Unreachable error. +// +// It indicates that a packet reached the destination host, but the transport +// protocol was not active on the destination port. +// +// +stateify savable +type icmpv6DestinationPortUnreachableSockError struct { + icmpv6DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv6DestinationPortUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv6PortUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv6DestinationPortUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationPortUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv6DestinationAddressUnreachableSockError)(nil) + +// icmpv6DestinationAddressUnreachableSockError is an ICMPv6 Destination Address +// Unreachable error. +// +// It indicates that a packet was not able to reach the destination. +// +// +stateify savable +type icmpv6DestinationAddressUnreachableSockError struct { + icmpv6DestinationUnreachableSockError +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv6DestinationAddressUnreachableSockError) Code() uint8 { + return uint8(header.ICMPv6AddressUnreachable) +} + +// Kind implements stack.TransportError. +func (*icmpv6DestinationAddressUnreachableSockError) Kind() stack.TransportErrorKind { + return stack.DestinationHostUnreachableTransportError +} + +var _ stack.TransportError = (*icmpv6PacketTooBigSockError)(nil) + +// icmpv6PacketTooBigSockError is an ICMPv6 Packet Too Big error. +// +// It indicates that a link exists on the path to the destination with an MTU +// that is too small to carry the packet. +// +// +stateify savable +type icmpv6PacketTooBigSockError struct { + mtu uint32 +} + +// Origin implements tcpip.SockErrorCause. +func (*icmpv6PacketTooBigSockError) Origin() tcpip.SockErrOrigin { + return tcpip.SockExtErrorOriginICMP6 +} + +// Type implements tcpip.SockErrorCause. +func (*icmpv6PacketTooBigSockError) Type() uint8 { + return uint8(header.ICMPv6PacketTooBig) +} + +// Code implements tcpip.SockErrorCause. +func (*icmpv6PacketTooBigSockError) Code() uint8 { + return uint8(header.ICMPv6UnusedCode) +} + +// Info implements tcpip.SockErrorCause. +func (e *icmpv6PacketTooBigSockError) Info() uint32 { + return e.mtu +} + +// Kind implements stack.TransportError. +func (*icmpv6PacketTooBigSockError) Kind() stack.TransportErrorKind { + return stack.PacketTooBigTransportError +} + +func (e *endpoint) checkLocalAddress(addr tcpip.Address) bool { + if e.nic.Spoofing() { + return true + } + + if addressEndpoint := e.AcquireAssignedAddress(addr, false, stack.NeverPrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + return true + } + return false +} + +// handleControl handles the case when an ICMP packet contains the headers of +// the original packet that caused the ICMP one to be sent. This information is +// used to find out which transport endpoint must be notified about the ICMP +// packet. +func (e *endpoint) handleControl(transErr stack.TransportError, pkt *stack.PacketBuffer) { + h, ok := pkt.Data().PullUp(header.IPv6MinimumSize) + if !ok { + return + } + hdr := header.IPv6(h) + + // We don't use IsValid() here because ICMP only requires that up to + // 1280 bytes of the original packet be included. So it's likely that it + // is truncated, which would cause IsValid to return false. + // + // Drop packet if it doesn't have the basic IPv6 header or if the + // original source address doesn't match an address we own. + srcAddr := hdr.SourceAddress() + if !e.checkLocalAddress(srcAddr) { + return + } + + // Keep needed information before trimming header. + p := hdr.TransportProtocol() + dstAddr := hdr.DestinationAddress() + + // Skip the IP header, then handle the fragmentation header if there + // is one. + if _, ok := pkt.Data().Consume(header.IPv6MinimumSize); !ok { + panic("could not consume IPv6MinimumSize bytes") + } + if p == header.IPv6FragmentHeader { + f, ok := pkt.Data().PullUp(header.IPv6FragmentHeaderSize) + if !ok { + return + } + fragHdr := header.IPv6Fragment(f) + if !fragHdr.IsValid() || fragHdr.FragmentOffset() != 0 { + // We can't handle fragments that aren't at offset 0 + // because they don't have the transport headers. + return + } + p = fragHdr.TransportProtocol() + + // Skip fragmentation header and find out the actual protocol + // number. + if _, ok := pkt.Data().Consume(header.IPv6FragmentHeaderSize); !ok { + panic("could not consume IPv6FragmentHeaderSize bytes") + } + } + + e.dispatcher.DeliverTransportError(srcAddr, dstAddr, ProtocolNumber, p, transErr, pkt) +} + +// getLinkAddrOption searches NDP options for a given link address option using +// the provided getAddr function as a filter. Returns the link address if +// found; otherwise, returns the zero link address value. Also returns true if +// the options are valid as per the wire format, false otherwise. +func getLinkAddrOption(it header.NDPOptionIterator, getAddr func(header.NDPOption) tcpip.LinkAddress) (tcpip.LinkAddress, bool) { + var linkAddr tcpip.LinkAddress + for { + opt, done, err := it.Next() + if err != nil { + return "", false + } + if done { + break + } + if addr := getAddr(opt); len(addr) != 0 { + // No RFCs define what to do when an NDP message has multiple Link-Layer + // Address options. Since no interface can have multiple link-layer + // addresses, we consider such messages invalid. + if len(linkAddr) != 0 { + return "", false + } + linkAddr = addr + } + } + return linkAddr, true +} + +// getSourceLinkAddr searches NDP options for the source link address option. +// Returns the link address if found; otherwise, returns the zero link address +// value. Also returns true if the options are valid as per the wire format, +// false otherwise. +func getSourceLinkAddr(it header.NDPOptionIterator) (tcpip.LinkAddress, bool) { + return getLinkAddrOption(it, func(opt header.NDPOption) tcpip.LinkAddress { + if src, ok := opt.(header.NDPSourceLinkLayerAddressOption); ok { + return src.EthernetAddress() + } + return "" + }) +} + +// getTargetLinkAddr searches NDP options for the target link address option. +// Returns the link address if found; otherwise, returns the zero link address +// value. Also returns true if the options are valid as per the wire format, +// false otherwise. +func getTargetLinkAddr(it header.NDPOptionIterator) (tcpip.LinkAddress, bool) { + return getLinkAddrOption(it, func(opt header.NDPOption) tcpip.LinkAddress { + if dst, ok := opt.(header.NDPTargetLinkLayerAddressOption); ok { + return dst.EthernetAddress() + } + return "" + }) +} + +func isMLDValid(pkt *stack.PacketBuffer, iph header.IPv6, routerAlert *header.IPv6RouterAlertOption) bool { + // As per RFC 2710 section 3: + // All MLD messages described in this document are sent with a link-local + // IPv6 Source Address, an IPv6 Hop Limit of 1, and an IPv6 Router Alert + // option in a Hop-by-Hop Options header. + if routerAlert == nil || routerAlert.Value != header.IPv6RouterAlertMLD { + return false + } + if len(pkt.TransportHeader().Slice()) < header.ICMPv6HeaderSize+header.MLDMinimumSize { + return false + } + if iph.HopLimit() != header.MLDHopLimit { + return false + } + if !header.IsV6LinkLocalUnicastAddress(iph.SourceAddress()) { + return false + } + return true +} + +func (e *endpoint) handleICMP(pkt *stack.PacketBuffer, hasFragmentHeader bool, routerAlert *header.IPv6RouterAlertOption) { + sent := e.stats.icmp.packetsSent + received := e.stats.icmp.packetsReceived + h := header.ICMPv6(pkt.TransportHeader().Slice()) + if len(h) < header.ICMPv6MinimumSize { + received.invalid.Increment() + return + } + iph := header.IPv6(pkt.NetworkHeader().Slice()) + srcAddr := iph.SourceAddress() + dstAddr := iph.DestinationAddress() + + // Validate ICMPv6 checksum before processing the packet. + payload := pkt.Data() + if got, want := h.Checksum(), header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: h, + Src: srcAddr, + Dst: dstAddr, + PayloadCsum: payload.Checksum(), + PayloadLen: payload.Size(), + }); got != want { + received.invalid.Increment() + return + } + + isNDPValid := func() bool { + // As per RFC 4861 sections 4.1 - 4.5, 6.1.1, 6.1.2, 7.1.1, 7.1.2 and + // 8.1, nodes MUST silently drop NDP packets where the Hop Limit field + // in the IPv6 header is not set to 255, or the ICMPv6 Code field is not + // set to 0. + // + // As per RFC 6980 section 5, nodes MUST silently drop NDP messages if the + // packet includes a fragmentation header. + return !hasFragmentHeader && iph.HopLimit() == header.NDPHopLimit && h.Code() == 0 + } + + // TODO(b/112892170): Meaningfully handle all ICMP types. + switch icmpType := h.Type(); icmpType { + case header.ICMPv6PacketTooBig: + received.packetTooBig.Increment() + networkMTU, err := calculateNetworkMTU(h.MTU(), header.IPv6MinimumSize) + if err != nil { + networkMTU = 0 + } + e.handleControl(&icmpv6PacketTooBigSockError{mtu: networkMTU}, pkt) + + case header.ICMPv6DstUnreachable: + received.dstUnreachable.Increment() + switch h.Code() { + case header.ICMPv6NetworkUnreachable: + e.handleControl(&icmpv6DestinationNetworkUnreachableSockError{}, pkt) + case header.ICMPv6PortUnreachable: + e.handleControl(&icmpv6DestinationPortUnreachableSockError{}, pkt) + } + case header.ICMPv6NeighborSolicit: + received.neighborSolicit.Increment() + if !isNDPValid() || len(h) < header.ICMPv6NeighborSolicitMinimumSize { + received.invalid.Increment() + return + } + + ns := header.NDPNeighborSolicit(h.MessageBody()) + targetAddr := ns.TargetAddress() + + // As per RFC 4861 section 4.3, the Target Address MUST NOT be a multicast + // address. + if header.IsV6MulticastAddress(targetAddr) { + received.invalid.Increment() + return + } + + var it header.NDPOptionIterator + { + var err error + it, err = ns.Options().Iter(false /* check */) + if err != nil { + // Options are not valid as per the wire format, silently drop the + // packet. + received.invalid.Increment() + return + } + } + + if e.hasTentativeAddr(targetAddr) { + // If the target address is tentative and the source of the packet is a + // unicast (specified) address, then the source of the packet is + // attempting to perform address resolution on the target. In this case, + // the solicitation is silently ignored, as per RFC 4862 section 5.4.3. + // + // If the target address is tentative and the source of the packet is the + // unspecified address (::), then we know another node is also performing + // DAD for the same address (since the target address is tentative for us, + // we know we are also performing DAD on it). In this case we let the + // stack know so it can handle such a scenario and do nothing further with + // the NS. + if srcAddr == header.IPv6Any { + var nonce []byte + for { + opt, done, err := it.Next() + if err != nil { + received.invalid.Increment() + return + } + if done { + break + } + if n, ok := opt.(header.NDPNonceOption); ok { + nonce = n.Nonce() + break + } + } + + // Since this is a DAD message we know the sender does not actually hold + // the target address so there is no "holder". + var holderLinkAddress tcpip.LinkAddress + + // We would get an error if the address no longer exists or the address + // is no longer tentative (DAD resolved between the call to + // hasTentativeAddr and this point). Both of these are valid scenarios: + // 1) An address may be removed at any time. + // 2) As per RFC 4862 section 5.4, DAD is not a perfect: + // "Note that the method for detecting duplicates + // is not completely reliable, and it is possible that duplicate + // addresses will still exist" + // + // TODO(gvisor.dev/issue/4046): Handle the scenario when a duplicate + // address is detected for an assigned address. + switch err := e.dupTentativeAddrDetected(targetAddr, holderLinkAddress, nonce); err.(type) { + case nil, *tcpip.ErrBadAddress, *tcpip.ErrInvalidEndpointState: + default: + panic(fmt.Sprintf("unexpected error handling duplicate tentative address: %s", err)) + } + } + + // Do not handle neighbor solicitations targeted to an address that is + // tentative on the NIC any further. + return + } + + // At this point we know that the target address is not tentative on the NIC + // so the packet is processed as defined in RFC 4861, as per RFC 4862 + // section 5.4.3. + + // Is the NS targeting us? + if !e.checkLocalAddress(targetAddr) { + return + } + + sourceLinkAddr, ok := getSourceLinkAddr(it) + if !ok { + received.invalid.Increment() + return + } + + // As per RFC 4861 section 4.3, the Source Link-Layer Address Option MUST + // NOT be included when the source IP address is the unspecified address. + // Otherwise, on link layers that have addresses this option MUST be + // included in multicast solicitations and SHOULD be included in unicast + // solicitations. + unspecifiedSource := srcAddr == header.IPv6Any + if len(sourceLinkAddr) == 0 { + if header.IsV6MulticastAddress(dstAddr) && !unspecifiedSource { + received.invalid.Increment() + return + } + } else if unspecifiedSource { + received.invalid.Increment() + return + } else { + switch err := e.nic.HandleNeighborProbe(ProtocolNumber, srcAddr, sourceLinkAddr); err.(type) { + case nil: + case *tcpip.ErrNotSupported: + // The stack may support ICMPv6 but the NIC may not need link resolution. + default: + panic(fmt.Sprintf("unexpected error when informing NIC of neighbor probe message: %s", err)) + } + } + + // As per RFC 4861 section 7.1.1: + // A node MUST silently discard any received Neighbor Solicitation + // messages that do not satisfy all of the following validity checks: + // ... + // - If the IP source address is the unspecified address, the IP + // destination address is a solicited-node multicast address. + if unspecifiedSource && !header.IsSolicitedNodeAddr(dstAddr) { + received.invalid.Increment() + return + } + + // As per RFC 4861 section 7.2.4: + // + // If the source of the solicitation is the unspecified address, the node + // MUST [...] and multicast the advertisement to the all-nodes address. + // + remoteAddr := srcAddr + if unspecifiedSource { + remoteAddr = header.IPv6AllNodesMulticastAddress + } + + // Even if we were able to receive a packet from some remote, we may not + // have a route to it - the remote may be blocked via routing rules. We must + // always consult our routing table and find a route to the remote before + // sending any packet. + r, err := e.protocol.stack.FindRoute(e.nic.ID(), targetAddr, remoteAddr, ProtocolNumber, false /* multicastLoop */) + if err != nil { + // If we cannot find a route to the destination, silently drop the packet. + return + } + defer r.Release() + + // If the NS has a source link-layer option, resolve the route immediately + // to avoid querying the neighbor table when the neighbor entry was updated + // as probing the neighbor table for a link address will transition the + // entry's state from stale to delay. + // + // Note, if the source link address is unspecified and this is a unicast + // solicitation, we may need to perform neighbor discovery to send the + // neighbor advertisement response. This is expected as per RFC 4861 section + // 7.2.4: + // + // Because unicast Neighbor Solicitations are not required to include a + // Source Link-Layer Address, it is possible that a node sending a + // solicited Neighbor Advertisement does not have a corresponding link- + // layer address for its neighbor in its Neighbor Cache. In such + // situations, a node will first have to use Neighbor Discovery to + // determine the link-layer address of its neighbor (i.e., send out a + // multicast Neighbor Solicitation). + // + if len(sourceLinkAddr) != 0 { + r.ResolveWith(sourceLinkAddr) + } + + optsSerializer := header.NDPOptionsSerializer{ + header.NDPTargetLinkLayerAddressOption(e.nic.LinkAddress()), + } + neighborAdvertSize := header.ICMPv6NeighborAdvertMinimumSize + optsSerializer.Length() + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(r.MaxHeaderLength()) + neighborAdvertSize, + }) + defer pkt.DecRef() + pkt.TransportProtocolNumber = header.ICMPv6ProtocolNumber + packet := header.ICMPv6(pkt.TransportHeader().Push(neighborAdvertSize)) + packet.SetType(header.ICMPv6NeighborAdvert) + na := header.NDPNeighborAdvert(packet.MessageBody()) + + // As per RFC 4861 section 7.2.4: + // + // If the source of the solicitation is the unspecified address, the node + // MUST set the Solicited flag to zero and [..]. Otherwise, the node MUST + // set the Solicited flag to one and [..]. + // + na.SetSolicitedFlag(!unspecifiedSource) + na.SetOverrideFlag(true) + na.SetRouterFlag(e.Forwarding()) + na.SetTargetAddress(targetAddr) + na.Options().Serialize(optsSerializer) + packet.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: packet, + Src: r.LocalAddress(), + Dst: r.RemoteAddress(), + })) + + // RFC 4861 Neighbor Discovery for IP version 6 (IPv6) + // + // 7.1.2. Validation of Neighbor Advertisements + // + // The IP Hop Limit field has a value of 255, i.e., the packet + // could not possibly have been forwarded by a router. + if err := r.WritePacket(stack.NetworkHeaderParams{Protocol: header.ICMPv6ProtocolNumber, TTL: header.NDPHopLimit, TOS: stack.DefaultTOS}, pkt); err != nil { + sent.dropped.Increment() + return + } + sent.neighborAdvert.Increment() + + case header.ICMPv6NeighborAdvert: + received.neighborAdvert.Increment() + if !isNDPValid() || len(h) < header.ICMPv6NeighborAdvertMinimumSize { + received.invalid.Increment() + return + } + + na := header.NDPNeighborAdvert(h.MessageBody()) + + it, err := na.Options().Iter(false /* check */) + if err != nil { + // If we have a malformed NDP NA option, drop the packet. + received.invalid.Increment() + return + } + + targetLinkAddr, ok := getTargetLinkAddr(it) + if !ok { + received.invalid.Increment() + return + } + + targetAddr := na.TargetAddress() + + e.dad.mu.Lock() + e.dad.mu.dad.StopLocked(targetAddr, &stack.DADDupAddrDetected{HolderLinkAddress: targetLinkAddr}) + e.dad.mu.Unlock() + + if e.hasTentativeAddr(targetAddr) { + // We only send a nonce value in DAD messages to check for loopedback + // messages so we use the empty nonce value here. + var nonce []byte + + // We just got an NA from a node that owns an address we are performing + // DAD on, implying the address is not unique. In this case we let the + // stack know so it can handle such a scenario and do nothing further with + // the NDP NA. + // + // We would get an error if the address no longer exists or the address + // is no longer tentative (DAD resolved between the call to + // hasTentativeAddr and this point). Both of these are valid scenarios: + // 1) An address may be removed at any time. + // 2) As per RFC 4862 section 5.4, DAD is not a perfect: + // "Note that the method for detecting duplicates + // is not completely reliable, and it is possible that duplicate + // addresses will still exist" + // + // TODO(gvisor.dev/issue/4046): Handle the scenario when a duplicate + // address is detected for an assigned address. + switch err := e.dupTentativeAddrDetected(targetAddr, targetLinkAddr, nonce); err.(type) { + case nil, *tcpip.ErrBadAddress, *tcpip.ErrInvalidEndpointState: + return + default: + panic(fmt.Sprintf("unexpected error handling duplicate tentative address: %s", err)) + } + } + + // At this point we know that the target address is not tentative on the + // NIC. However, the target address may still be assigned to the NIC but not + // tentative (it could be permanent). Such a scenario is beyond the scope of + // RFC 4862. As such, we simply ignore such a scenario for now and proceed + // as normal. + // + // TODO(b/143147598): Handle the scenario described above. Also inform the + // netstack integration that a duplicate address was detected outside of + // DAD. + + // As per RFC 4861 section 7.1.2: + // A node MUST silently discard any received Neighbor Advertisement + // messages that do not satisfy all of the following validity checks: + // ... + // - If the IP Destination Address is a multicast address the + // Solicited flag is zero. + if header.IsV6MulticastAddress(dstAddr) && na.SolicitedFlag() { + received.invalid.Increment() + return + } + + // If the NA message has the target link layer option, update the link + // address cache with the link address for the target of the message. + switch err := e.nic.HandleNeighborConfirmation(ProtocolNumber, targetAddr, targetLinkAddr, stack.ReachabilityConfirmationFlags{ + Solicited: na.SolicitedFlag(), + Override: na.OverrideFlag(), + IsRouter: na.RouterFlag(), + }); err.(type) { + case nil: + case *tcpip.ErrNotSupported: + // The stack may support ICMPv6 but the NIC may not need link resolution. + default: + panic(fmt.Sprintf("unexpected error when informing NIC of neighbor confirmation message: %s", err)) + } + + case header.ICMPv6EchoRequest: + received.echoRequest.Increment() + if len(h) < header.ICMPv6EchoMinimumSize { + received.invalid.Increment() + return + } + e.dispatcher.DeliverTransportPacket(header.ICMPv6ProtocolNumber, pkt) + case header.ICMPv6EchoReply: + received.echoReply.Increment() + if len(h) < header.ICMPv6EchoMinimumSize { + received.invalid.Increment() + return + } + e.dispatcher.DeliverTransportPacket(header.ICMPv6ProtocolNumber, pkt) + case header.ICMPv6TimeExceeded: + received.timeExceeded.Increment() + + case header.ICMPv6ParamProblem: + received.paramProblem.Increment() + + case header.ICMPv6RouterSolicit: + received.routerSolicit.Increment() + + // + // Validate the RS as per RFC 4861 section 6.1.1. + // + + // Is the NDP payload of sufficient size to hold a Router Solictation? + if !isNDPValid() || len(h)-header.ICMPv6HeaderSize < header.NDPRSMinimumSize { + received.invalid.Increment() + return + } + + if !e.Forwarding() { + received.routerOnlyPacketsDroppedByHost.Increment() + return + } + + rs := header.NDPRouterSolicit(h.MessageBody()) + it, err := rs.Options().Iter(false /* check */) + if err != nil { + // Options are not valid as per the wire format, silently drop the packet. + received.invalid.Increment() + return + } + + sourceLinkAddr, ok := getSourceLinkAddr(it) + if !ok { + received.invalid.Increment() + return + } + + // If the RS message has the source link layer option, update the link + // address cache with the link address for the source of the message. + if len(sourceLinkAddr) != 0 { + // As per RFC 4861 section 4.1, the Source Link-Layer Address Option MUST + // NOT be included when the source IP address is the unspecified address. + // Otherwise, it SHOULD be included on link layers that have addresses. + if srcAddr == header.IPv6Any { + received.invalid.Increment() + return + } + + // A RS with a specified source IP address modifies the neighbor table + // in the same way a regular probe would. + switch err := e.nic.HandleNeighborProbe(ProtocolNumber, srcAddr, sourceLinkAddr); err.(type) { + case nil: + case *tcpip.ErrNotSupported: + // The stack may support ICMPv6 but the NIC may not need link resolution. + default: + panic(fmt.Sprintf("unexpected error when informing NIC of neighbor probe message: %s", err)) + } + } + + case header.ICMPv6RouterAdvert: + received.routerAdvert.Increment() + + // + // Validate the RA as per RFC 4861 section 6.1.2. + // + + // Is the NDP payload of sufficient size to hold a Router Advertisement? + if !isNDPValid() || len(h)-header.ICMPv6HeaderSize < header.NDPRAMinimumSize { + received.invalid.Increment() + return + } + + routerAddr := srcAddr + + // Is the IP Source Address a link-local address? + if !header.IsV6LinkLocalUnicastAddress(routerAddr) { + // ...No, silently drop the packet. + received.invalid.Increment() + return + } + + ra := header.NDPRouterAdvert(h.MessageBody()) + it, err := ra.Options().Iter(false /* check */) + if err != nil { + // Options are not valid as per the wire format, silently drop the packet. + received.invalid.Increment() + return + } + + sourceLinkAddr, ok := getSourceLinkAddr(it) + if !ok { + received.invalid.Increment() + return + } + + // + // At this point, we have a valid Router Advertisement, as far + // as RFC 4861 section 6.1.2 is concerned. + // + + // If the RA has the source link layer option, update the link address + // cache with the link address for the advertised router. + if len(sourceLinkAddr) != 0 { + switch err := e.nic.HandleNeighborProbe(ProtocolNumber, routerAddr, sourceLinkAddr); err.(type) { + case nil: + case *tcpip.ErrNotSupported: + // The stack may support ICMPv6 but the NIC may not need link resolution. + default: + panic(fmt.Sprintf("unexpected error when informing NIC of neighbor probe message: %s", err)) + } + } + + e.mu.Lock() + e.mu.ndp.handleRA(routerAddr, ra) + e.mu.Unlock() + + case header.ICMPv6RedirectMsg: + // TODO(gvisor.dev/issue/2285): Call `e.nud.HandleProbe` after validating + // this redirect message, as per RFC 4871 section 7.3.3: + // + // "A Neighbor Cache entry enters the STALE state when created as a + // result of receiving packets other than solicited Neighbor + // Advertisements (i.e., Router Solicitations, Router Advertisements, + // Redirects, and Neighbor Solicitations). These packets contain the + // link-layer address of either the sender or, in the case of Redirect, + // the redirection target. However, receipt of these link-layer + // addresses does not confirm reachability of the forward-direction path + // to that node. Placing a newly created Neighbor Cache entry for which + // the link-layer address is known in the STALE state provides assurance + // that path failures are detected quickly. In addition, should a cached + // link-layer address be modified due to receiving one of the above + // messages, the state SHOULD also be set to STALE to provide prompt + // verification that the path to the new link-layer address is working." + received.redirectMsg.Increment() + if !isNDPValid() { + received.invalid.Increment() + return + } + + case header.ICMPv6MulticastListenerQuery, + header.ICMPv6MulticastListenerReport, + header.ICMPv6MulticastListenerV2Report, + header.ICMPv6MulticastListenerDone: + icmpBody := h.MessageBody() + switch icmpType { + case header.ICMPv6MulticastListenerQuery: + received.multicastListenerQuery.Increment() + case header.ICMPv6MulticastListenerReport: + received.multicastListenerReport.Increment() + case header.ICMPv6MulticastListenerV2Report: + received.multicastListenerReportV2.Increment() + case header.ICMPv6MulticastListenerDone: + received.multicastListenerDone.Increment() + default: + panic(fmt.Sprintf("unrecognized MLD message = %d", icmpType)) + } + + if !isMLDValid(pkt, iph, routerAlert) { + received.invalid.Increment() + return + } + + switch icmpType { + case header.ICMPv6MulticastListenerQuery: + e.mu.Lock() + if len(icmpBody) >= header.MLDv2QueryMinimumSize { + e.mu.mld.handleMulticastListenerQueryV2(header.MLDv2Query(icmpBody)) + } else { + e.mu.mld.handleMulticastListenerQuery(header.MLD(icmpBody)) + } + e.mu.Unlock() + case header.ICMPv6MulticastListenerReport: + e.mu.Lock() + e.mu.mld.handleMulticastListenerReport(header.MLD(icmpBody)) + e.mu.Unlock() + case header.ICMPv6MulticastListenerDone, header.ICMPv6MulticastListenerV2Report: + default: + panic(fmt.Sprintf("unrecognized MLD message = %d", icmpType)) + } + + default: + received.unrecognized.Increment() + } +} + +// LinkAddressProtocol implements stack.LinkAddressResolver. +func (*endpoint) LinkAddressProtocol() tcpip.NetworkProtocolNumber { + return header.IPv6ProtocolNumber +} + +// LinkAddressRequest implements stack.LinkAddressResolver. +func (e *endpoint) LinkAddressRequest(targetAddr, localAddr tcpip.Address, remoteLinkAddr tcpip.LinkAddress) tcpip.Error { + remoteAddr := targetAddr + if len(remoteLinkAddr) == 0 { + remoteAddr = header.SolicitedNodeAddr(targetAddr) + remoteLinkAddr = header.EthernetAddressFromMulticastIPv6Address(remoteAddr) + } + + if localAddr.BitLen() == 0 { + // Find an address that we can use as our source address. + addressEndpoint := e.AcquireOutgoingPrimaryAddress(remoteAddr, tcpip.Address{} /* srcHint */, false /* allowExpired */) + if addressEndpoint == nil { + return &tcpip.ErrNetworkUnreachable{} + } + + localAddr = addressEndpoint.AddressWithPrefix().Address + addressEndpoint.DecRef() + } else if !e.checkLocalAddress(localAddr) { + // The provided local address is not assigned to us. + return &tcpip.ErrBadLocalAddress{} + } + + return e.sendNDPNS(localAddr, remoteAddr, targetAddr, remoteLinkAddr, header.NDPOptionsSerializer{ + header.NDPSourceLinkLayerAddressOption(e.nic.LinkAddress()), + }) +} + +// ResolveStaticAddress implements stack.LinkAddressResolver. +func (*endpoint) ResolveStaticAddress(addr tcpip.Address) (tcpip.LinkAddress, bool) { + if header.IsV6MulticastAddress(addr) { + return header.EthernetAddressFromMulticastIPv6Address(addr), true + } + return tcpip.LinkAddress([]byte(nil)), false +} + +// ======= ICMP Error packet generation ========= + +// icmpReason is a marker interface for IPv6 specific ICMP errors. +type icmpReason interface { + isICMPReason() + // respondToMulticast indicates whether this error falls under the exception + // outlined by RFC 4443 section 2.4 point e.3 exception 2: + // + // (e.3) A packet destined to an IPv6 multicast address. (There are two + // exceptions to this rule: (1) the Packet Too Big Message (Section 3.2) to + // allow Path MTU discovery to work for IPv6 multicast, and (2) the Parameter + // Problem Message, Code 2 (Section 3.4) reporting an unrecognized IPv6 + // option (see Section 4.2 of [IPv6]) that has the Option Type highest- + // order two bits set to 10). + respondsToMulticast() bool +} + +// icmpReasonParameterProblem is an error during processing of extension headers +// or the fixed header defined in RFC 4443 section 3.4. +type icmpReasonParameterProblem struct { + code header.ICMPv6Code + + // pointer is defined in the RFC 4443 section 3.4 which reads: + // + // Pointer Identifies the octet offset within the invoking packet + // where the error was detected. + // + // The pointer will point beyond the end of the ICMPv6 + // packet if the field in error is beyond what can fit + // in the maximum size of an ICMPv6 error message. + pointer uint32 + + respondToMulticast bool +} + +func (*icmpReasonParameterProblem) isICMPReason() {} + +func (p *icmpReasonParameterProblem) respondsToMulticast() bool { + return p.respondToMulticast +} + +// icmpReasonAdministrativelyProhibited is an error where the destination is +// administratively prohibited. +type icmpReasonAdministrativelyProhibited struct{} + +func (*icmpReasonAdministrativelyProhibited) isICMPReason() {} + +func (*icmpReasonAdministrativelyProhibited) respondsToMulticast() bool { + return false +} + +// icmpReasonPortUnreachable is an error where the transport protocol has no +// listener and no alternative means to inform the sender. +type icmpReasonPortUnreachable struct{} + +func (*icmpReasonPortUnreachable) isICMPReason() {} + +func (*icmpReasonPortUnreachable) respondsToMulticast() bool { + return false +} + +// icmpReasonNetUnreachable is an error where no route can be found to the +// network of the final destination. +type icmpReasonNetUnreachable struct{} + +func (*icmpReasonNetUnreachable) isICMPReason() {} + +func (*icmpReasonNetUnreachable) respondsToMulticast() bool { + return false +} + +// icmpReasonHostUnreachable is an error in which the host specified in the +// internet destination field of the datagram is unreachable. +type icmpReasonHostUnreachable struct{} + +func (*icmpReasonHostUnreachable) isICMPReason() {} + +func (*icmpReasonHostUnreachable) respondsToMulticast() bool { + return false +} + +// icmpReasonFragmentationNeeded is an error where a packet is to big to be sent +// out through the outgoing MTU, as per RFC 4443 page 9, Packet Too Big Message. +type icmpReasonPacketTooBig struct{} + +func (*icmpReasonPacketTooBig) isICMPReason() {} + +func (*icmpReasonPacketTooBig) respondsToMulticast() bool { + return true +} + +// icmpReasonHopLimitExceeded is an error where a packet's hop limit exceeded in +// transit to its final destination, as per RFC 4443 section 3.3. +type icmpReasonHopLimitExceeded struct{} + +func (*icmpReasonHopLimitExceeded) isICMPReason() {} + +func (*icmpReasonHopLimitExceeded) respondsToMulticast() bool { + return false +} + +// icmpReasonReassemblyTimeout is an error where insufficient fragments are +// received to complete reassembly of a packet within a configured time after +// the reception of the first-arriving fragment of that packet. +type icmpReasonReassemblyTimeout struct{} + +func (*icmpReasonReassemblyTimeout) isICMPReason() {} + +func (*icmpReasonReassemblyTimeout) respondsToMulticast() bool { + return false +} + +// returnError takes an error descriptor and generates the appropriate ICMP +// error packet for IPv6 and sends it. +func (p *protocol) returnError(reason icmpReason, pkt *stack.PacketBuffer, deliveredLocally bool) tcpip.Error { + origIPHdr := header.IPv6(pkt.NetworkHeader().Slice()) + origIPHdrSrc := origIPHdr.SourceAddress() + origIPHdrDst := origIPHdr.DestinationAddress() + + // Only send ICMP error if the address is not a multicast v6 + // address and the source is not the unspecified address. + // + // There are exceptions to this rule. + // See: point e.3) RFC 4443 section-2.4 + // + // (e) An ICMPv6 error message MUST NOT be originated as a result of + // receiving the following: + // + // (e.1) An ICMPv6 error message. + // + // (e.2) An ICMPv6 redirect message [IPv6-DISC]. + // + // (e.3) A packet destined to an IPv6 multicast address. (There are + // two exceptions to this rule: (1) the Packet Too Big Message + // (Section 3.2) to allow Path MTU discovery to work for IPv6 + // multicast, and (2) the Parameter Problem Message, Code 2 + // (Section 3.4) reporting an unrecognized IPv6 option (see + // Section 4.2 of [IPv6]) that has the Option Type highest- + // order two bits set to 10). + // + allowResponseToMulticast := reason.respondsToMulticast() + isOrigDstMulticast := header.IsV6MulticastAddress(origIPHdrDst) + if (!allowResponseToMulticast && isOrigDstMulticast) || origIPHdrSrc == header.IPv6Any { + return nil + } + + // If the packet wasn't delivered locally, do not use the packet's destination + // address as the response's source address as we should not own the + // destination address of a packet we are forwarding. + // + // If the packet was originally destined to a multicast address, then do not + // use the packet's destination address as the source for the response ICMP + // packet as "multicast addresses must not be used as source addresses in IPv6 + // packets", as per RFC 4291 section 2.7. + localAddr := origIPHdrDst + if !deliveredLocally || isOrigDstMulticast { + localAddr = tcpip.Address{} + } + // Even if we were able to receive a packet from some remote, we may not have + // a route to it - the remote may be blocked via routing rules. We must always + // consult our routing table and find a route to the remote before sending any + // packet. + route, err := p.stack.FindRoute(pkt.NICID, localAddr, origIPHdrSrc, ProtocolNumber, false /* multicastLoop */) + if err != nil { + return err + } + defer route.Release() + + p.mu.Lock() + // We retrieve an endpoint using the newly constructed route's NICID rather + // than the packet's NICID. The packet's NICID corresponds to the NIC on + // which it arrived, which isn't necessarily the same as the NIC on which it + // will be transmitted. On the other hand, the route's NIC *is* guaranteed + // to be the NIC on which the packet will be transmitted. + netEP, ok := p.mu.eps[route.NICID()] + p.mu.Unlock() + if !ok { + return &tcpip.ErrNotConnected{} + } + + if pkt.TransportProtocolNumber == header.ICMPv6ProtocolNumber { + if typ := header.ICMPv6(pkt.TransportHeader().Slice()).Type(); typ.IsErrorType() || typ == header.ICMPv6RedirectMsg { + return nil + } + } + + sent := netEP.stats.icmp.packetsSent + icmpType, icmpCode, counter, typeSpecific := func() (header.ICMPv6Type, header.ICMPv6Code, tcpip.MultiCounterStat, uint32) { + switch reason := reason.(type) { + case *icmpReasonParameterProblem: + return header.ICMPv6ParamProblem, reason.code, sent.paramProblem, reason.pointer + case *icmpReasonAdministrativelyProhibited: + return header.ICMPv6DstUnreachable, header.ICMPv6Prohibited, sent.dstUnreachable, 0 + case *icmpReasonPortUnreachable: + return header.ICMPv6DstUnreachable, header.ICMPv6PortUnreachable, sent.dstUnreachable, 0 + case *icmpReasonNetUnreachable: + return header.ICMPv6DstUnreachable, header.ICMPv6NetworkUnreachable, sent.dstUnreachable, 0 + case *icmpReasonHostUnreachable: + return header.ICMPv6DstUnreachable, header.ICMPv6AddressUnreachable, sent.dstUnreachable, 0 + case *icmpReasonPacketTooBig: + return header.ICMPv6PacketTooBig, header.ICMPv6UnusedCode, sent.packetTooBig, 0 + case *icmpReasonHopLimitExceeded: + return header.ICMPv6TimeExceeded, header.ICMPv6HopLimitExceeded, sent.timeExceeded, 0 + case *icmpReasonReassemblyTimeout: + return header.ICMPv6TimeExceeded, header.ICMPv6ReassemblyTimeout, sent.timeExceeded, 0 + default: + panic(fmt.Sprintf("unsupported ICMP type %T", reason)) + } + }() + + if !p.allowICMPReply(icmpType) { + sent.rateLimited.Increment() + return nil + } + + network, transport := pkt.NetworkHeader().View(), pkt.TransportHeader().View() + + // As per RFC 4443 section 2.4 + // + // (c) Every ICMPv6 error message (type < 128) MUST include + // as much of the IPv6 offending (invoking) packet (the + // packet that caused the error) as possible without making + // the error message packet exceed the minimum IPv6 MTU + // [IPv6]. + mtu := int(route.MTU()) + const maxIPv6Data = header.IPv6MinimumMTU - header.IPv6FixedHeaderSize + if mtu > maxIPv6Data { + mtu = maxIPv6Data + } + available := mtu - header.ICMPv6ErrorHeaderSize + if available < header.IPv6MinimumSize { + return nil + } + payloadLen := network.Size() + transport.Size() + pkt.Data().Size() + if payloadLen > available { + payloadLen = available + } + payload := buffer.MakeWithView(network) + payload.Append(transport) + dataBuf := pkt.Data().ToBuffer() + payload.Merge(&dataBuf) + payload.Truncate(int64(payloadLen)) + + newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(route.MaxHeaderLength()) + header.ICMPv6ErrorHeaderSize, + Payload: payload, + }) + defer newPkt.DecRef() + newPkt.TransportProtocolNumber = header.ICMPv6ProtocolNumber + + icmpHdr := header.ICMPv6(newPkt.TransportHeader().Push(header.ICMPv6DstUnreachableMinimumSize)) + icmpHdr.SetType(icmpType) + icmpHdr.SetCode(icmpCode) + icmpHdr.SetTypeSpecific(typeSpecific) + + pktData := newPkt.Data() + icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmpHdr, + Src: route.LocalAddress(), + Dst: route.RemoteAddress(), + PayloadCsum: pktData.Checksum(), + PayloadLen: pktData.Size(), + })) + if err := route.WritePacket( + stack.NetworkHeaderParams{ + Protocol: header.ICMPv6ProtocolNumber, + TTL: route.DefaultTTL(), + TOS: stack.DefaultTOS, + }, + newPkt, + ); err != nil { + sent.dropped.Increment() + return err + } + counter.Increment() + return nil +} + +// OnReassemblyTimeout implements fragmentation.TimeoutHandler. +func (p *protocol) OnReassemblyTimeout(pkt *stack.PacketBuffer) { + // OnReassemblyTimeout sends a Time Exceeded Message as per RFC 2460 Section + // 4.5: + // + // If the first fragment (i.e., the one with a Fragment Offset of zero) has + // been received, an ICMP Time Exceeded -- Fragment Reassembly Time Exceeded + // message should be sent to the source of that fragment. + if pkt != nil { + p.returnError(&icmpReasonReassemblyTimeout{}, pkt, true /* deliveredLocally */) + } +} diff --git a/pkg/tcpip/network/ipv6/ipv6.go b/pkg/tcpip/network/ipv6/ipv6.go new file mode 100644 index 0000000..bfc28a5 --- /dev/null +++ b/pkg/tcpip/network/ipv6/ipv6.go @@ -0,0 +1,2875 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ipv6 contains the implementation of the ipv6 network protocol. +package ipv6 + +import ( + "fmt" + "math" + "reflect" + "sort" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/fragmentation" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/multicast" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // ReassembleTimeout controls how long a fragment will be held. + // As per RFC 8200 section 4.5: + // + // If insufficient fragments are received to complete reassembly of a packet + // within 60 seconds of the reception of the first-arriving fragment of that + // packet, reassembly of that packet must be abandoned. + // + // Linux also uses 60 seconds for reassembly timeout: + // https://github.com/torvalds/linux/blob/47ec5303d73ea344e84f46660fff693c57641386/include/net/ipv6.h#L456 + ReassembleTimeout = 60 * time.Second + + // ProtocolNumber is the ipv6 protocol number. + ProtocolNumber = header.IPv6ProtocolNumber + + // maxPayloadSize is the maximum size that can be encoded in the 16-bit + // PayloadLength field of the ipv6 header. + maxPayloadSize = 0xffff + + // DefaultTTL is the default hop limit for IPv6 Packets egressed by + // Netstack. + DefaultTTL = 64 + + // buckets for fragment identifiers + buckets = 2048 +) + +const ( + forwardingDisabled = 0 + forwardingEnabled = 1 +) + +// policyTable is the default policy table defined in RFC 6724 section 2.1. +// +// A more human-readable version: +// +// Prefix Precedence Label +// ::1/128 50 0 +// ::/0 40 1 +// ::ffff:0:0/96 35 4 +// 2002::/16 30 2 +// 2001::/32 5 5 +// fc00::/7 3 13 +// ::/96 1 3 +// fec0::/10 1 11 +// 3ffe::/16 1 12 +// +// The table is sorted by prefix length so longest-prefix match can be easily +// achieved. +// +// We willingly left out ::/96, fec0::/10 and 3ffe::/16 since those prefix +// assignments are deprecated. +// +// As per RFC 4291 section 2.5.5.1 (for ::/96), +// +// The "IPv4-Compatible IPv6 address" is now deprecated because the +// current IPv6 transition mechanisms no longer use these addresses. +// New or updated implementations are not required to support this +// address type. +// +// As per RFC 3879 section 4 (for fec0::/10), +// +// This document formally deprecates the IPv6 site-local unicast prefix +// defined in [RFC3513], i.e., 1111111011 binary or FEC0::/10. +// +// As per RFC 3701 section 1 (for 3ffe::/16), +// +// As clearly stated in [TEST-NEW], the addresses for the 6bone are +// temporary and will be reclaimed in the future. It further states +// that all users of these addresses (within the 3FFE::/16 prefix) will +// be required to renumber at some time in the future. +// +// and section 2, +// +// Thus after the pTLA allocation cutoff date January 1, 2004, it is +// REQUIRED that no new 6bone 3FFE pTLAs be allocated. +// +// MUST NOT BE MODIFIED. +var policyTable = [...]struct { + subnet tcpip.Subnet + + label uint8 +}{ + // ::1/128 + { + subnet: header.IPv6Loopback.WithPrefix().Subnet(), + label: 0, + }, + // ::ffff:0:0/96 + { + subnet: header.IPv4MappedIPv6Subnet, + label: 4, + }, + // 2001::/32 (Teredo prefix as per RFC 4380 section 2.6). + { + subnet: tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16([16]byte{0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), + PrefixLen: 32, + }.Subnet(), + label: 5, + }, + // 2002::/16 (6to4 prefix as per RFC 3056 section 2). + { + subnet: tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16([16]byte{0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), + PrefixLen: 16, + }.Subnet(), + label: 2, + }, + // fc00::/7 (Unique local addresses as per RFC 4193 section 3.1). + { + subnet: tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16([16]byte{0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), + PrefixLen: 7, + }.Subnet(), + label: 13, + }, + // ::/0 + { + subnet: header.IPv6EmptySubnet, + label: 1, + }, +} + +func getLabel(addr tcpip.Address) uint8 { + for _, p := range policyTable { + if p.subnet.Contains(addr) { + return p.label + } + } + + panic(fmt.Sprintf("should have a label for address = %s", addr)) +} + +var ( + _ stack.DuplicateAddressDetector = (*endpoint)(nil) + _ stack.LinkAddressResolver = (*endpoint)(nil) + _ stack.LinkResolvableNetworkEndpoint = (*endpoint)(nil) + _ stack.ForwardingNetworkEndpoint = (*endpoint)(nil) + _ stack.MulticastForwardingNetworkEndpoint = (*endpoint)(nil) + _ stack.GroupAddressableEndpoint = (*endpoint)(nil) + _ stack.AddressableEndpoint = (*endpoint)(nil) + _ stack.NetworkEndpoint = (*endpoint)(nil) + _ stack.NDPEndpoint = (*endpoint)(nil) + _ MLDEndpoint = (*endpoint)(nil) + _ NDPEndpoint = (*endpoint)(nil) +) + +// +stateify savable +type endpointMu struct { + sync.RWMutex `state:"nosave"` + + addressableEndpointState stack.AddressableEndpointState + ndp ndpState + mld mldState +} + +// +stateify savable +type dadMu struct { + sync.Mutex `state:"nosave"` + + dad ip.DAD +} + +// +stateify savable +type endpointDAD struct { + mu dadMu +} + +// +stateify savable +type endpoint struct { + nic stack.NetworkInterface + dispatcher stack.TransportDispatcher + protocol *protocol + stats sharedStats + + // enabled is set to 1 when the endpoint is enabled and 0 when it is + // disabled. + enabled atomicbitops.Uint32 + + // forwarding is set to forwardingEnabled when the endpoint has forwarding + // enabled and forwardingDisabled when it is disabled. + forwarding atomicbitops.Uint32 + + // multicastForwarding is set to forwardingEnabled when the endpoint has + // forwarding enabled and forwardingDisabled when it is disabled. + multicastForwarding atomicbitops.Uint32 + + mu endpointMu + + // dad is used to check if an arbitrary address is already assigned to some + // neighbor. + // + // Note: this is different from mu.ndp.dad which is used to perform DAD for + // addresses that are assigned to the interface. Removing an address aborts + // DAD; if we had used the same state, handlers for a removed address would + // not be called with the actual DAD result. + // + // LOCK ORDERING: mu > dad.mu. + dad endpointDAD +} + +// NICNameFromID is a function that returns a stable name for the specified NIC, +// even if different NIC IDs are used to refer to the same NIC in different +// program runs. It is used when generating opaque interface identifiers (IIDs). +// If the NIC was created with a name, it is passed to NICNameFromID. +// +// NICNameFromID SHOULD return unique NIC names so unique opaque IIDs are +// generated for the same prefix on different NICs. +type NICNameFromID func(tcpip.NICID, string) string + +// OpaqueInterfaceIdentifierOptions holds the options related to the generation +// of opaque interface identifiers (IIDs) as defined by RFC 7217. +// +// +stateify savable +type OpaqueInterfaceIdentifierOptions struct { + // NICNameFromID is a function that returns a stable name for a specified NIC, + // even if the NIC ID changes over time. + // + // Must be specified to generate the opaque IID. + NICNameFromID NICNameFromID `state:"nosave"` + + // SecretKey is a pseudo-random number used as the secret key when generating + // opaque IIDs as defined by RFC 7217. The key SHOULD be at least + // header.OpaqueIIDSecretKeyMinBytes bytes and MUST follow minimum randomness + // requirements for security as outlined by RFC 4086. SecretKey MUST NOT + // change between program runs, unless explicitly changed. + // + // OpaqueInterfaceIdentifierOptions takes ownership of SecretKey. SecretKey + // MUST NOT be modified after Stack is created. + // + // May be nil, but a nil value is highly discouraged to maintain + // some level of randomness between nodes. + SecretKey []byte +} + +// CheckDuplicateAddress implements stack.DuplicateAddressDetector. +func (e *endpoint) CheckDuplicateAddress(addr tcpip.Address, h stack.DADCompletionHandler) stack.DADCheckAddressDisposition { + e.dad.mu.Lock() + defer e.dad.mu.Unlock() + return e.dad.mu.dad.CheckDuplicateAddressLocked(addr, h) +} + +// SetDADConfigurations implements stack.DuplicateAddressDetector. +func (e *endpoint) SetDADConfigurations(c stack.DADConfigurations) { + e.mu.Lock() + defer e.mu.Unlock() + e.dad.mu.Lock() + defer e.dad.mu.Unlock() + + e.mu.ndp.dad.SetConfigsLocked(c) + e.dad.mu.dad.SetConfigsLocked(c) +} + +// DuplicateAddressProtocol implements stack.DuplicateAddressDetector. +func (*endpoint) DuplicateAddressProtocol() tcpip.NetworkProtocolNumber { + return ProtocolNumber +} + +// HandleLinkResolutionFailure implements stack.LinkResolvableNetworkEndpoint. +func (e *endpoint) HandleLinkResolutionFailure(pkt *stack.PacketBuffer) { + // If we are operating as a router, we should return an ICMP error to the + // original packet's sender. + if pkt.NetworkPacketInfo.IsForwardedPacket { + // TODO(gvisor.dev/issue/6005): Propagate asynchronously generated ICMP + // errors to local endpoints. + e.protocol.returnError(&icmpReasonHostUnreachable{}, pkt, false /* deliveredLocally */) + e.stats.ip.Forwarding.Errors.Increment() + e.stats.ip.Forwarding.HostUnreachable.Increment() + return + } + // handleControl expects the entire offending packet to be in the packet + // buffer's data field. + pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: pkt.ToBuffer(), + }) + defer pkt.DecRef() + pkt.NICID = e.nic.ID() + pkt.NetworkProtocolNumber = ProtocolNumber + e.handleControl(&icmpv6DestinationAddressUnreachableSockError{}, pkt) +} + +// onAddressAssignedLocked handles an address being assigned. +// +// Precondition: e.mu must be exclusively locked. +func (e *endpoint) onAddressAssignedLocked(addr tcpip.Address) { + // As per RFC 2710 section 3, + // + // All MLD messages described in this document are sent with a link-local + // IPv6 Source Address, ... + // + // If we just completed DAD for a link-local address, then attempt to send any + // queued MLD reports. Note, we may have sent reports already for some of the + // groups before we had a valid link-local address to use as the source for + // the MLD messages, but that was only so that MLD snooping switches are aware + // of our membership to groups - routers would not have handled those reports. + // + // As per RFC 3590 section 4, + // + // MLD Report and Done messages are sent with a link-local address as + // the IPv6 source address, if a valid address is available on the + // interface. If a valid link-local address is not available (e.g., one + // has not been configured), the message is sent with the unspecified + // address (::) as the IPv6 source address. + // + // Once a valid link-local address is available, a node SHOULD generate + // new MLD Report messages for all multicast addresses joined on the + // interface. + // + // Routers receiving an MLD Report or Done message with the unspecified + // address as the IPv6 source address MUST silently discard the packet + // without taking any action on the packets contents. + // + // Snooping switches MUST manage multicast forwarding state based on MLD + // Report and Done messages sent with the unspecified address as the + // IPv6 source address. + if header.IsV6LinkLocalUnicastAddress(addr) { + e.mu.mld.sendQueuedReports() + } +} + +// InvalidateDefaultRouter implements stack.NDPEndpoint. +func (e *endpoint) InvalidateDefaultRouter(rtr tcpip.Address) { + e.mu.Lock() + defer e.mu.Unlock() + + // We represent default routers with a default (off-link) route through the + // router. + e.mu.ndp.invalidateOffLinkRoute(offLinkRoute{dest: header.IPv6EmptySubnet, router: rtr}) +} + +// SetMLDVersion implements MLDEndpoint. +func (e *endpoint) SetMLDVersion(v MLDVersion) MLDVersion { + e.mu.Lock() + defer e.mu.Unlock() + return e.mu.mld.setVersion(v) +} + +// GetMLDVersion implements MLDEndpoint. +func (e *endpoint) GetMLDVersion() MLDVersion { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.mld.getVersion() +} + +// SetNDPConfigurations implements NDPEndpoint. +func (e *endpoint) SetNDPConfigurations(c NDPConfigurations) { + c.validate() + e.mu.Lock() + defer e.mu.Unlock() + e.mu.ndp.configs = c +} + +// NDPConfigurations implements NDPEndpoint. +func (e *endpoint) NDPConfigurations() NDPConfigurations { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.ndp.configs +} + +// hasTentativeAddr returns true if addr is tentative on e. +func (e *endpoint) hasTentativeAddr(addr tcpip.Address) bool { + e.mu.RLock() + addressEndpoint := e.getAddressRLocked(addr) + e.mu.RUnlock() + return addressEndpoint != nil && addressEndpoint.GetKind() == stack.PermanentTentative +} + +// dupTentativeAddrDetected attempts to inform e that a tentative addr is a +// duplicate on a link. +// +// dupTentativeAddrDetected removes the tentative address if it exists. If the +// address was generated via SLAAC, an attempt is made to generate a new +// address. +func (e *endpoint) dupTentativeAddrDetected(addr tcpip.Address, holderLinkAddr tcpip.LinkAddress, nonce []byte) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + addressEndpoint := e.getAddressRLocked(addr) + if addressEndpoint == nil { + return &tcpip.ErrBadAddress{} + } + + if addressEndpoint.GetKind() != stack.PermanentTentative { + return &tcpip.ErrInvalidEndpointState{} + } + + switch result := e.mu.ndp.dad.ExtendIfNonceEqualLocked(addr, nonce); result { + case ip.Extended: + // The nonce we got back was the same we sent so we know the message + // indicating a duplicate address was likely ours so do not consider + // the address duplicate here. + return nil + case ip.AlreadyExtended: + // See Extended. + // + // Our DAD message was looped back already. + return nil + case ip.NoDADStateFound: + panic(fmt.Sprintf("expected DAD state for tentative address %s", addr)) + case ip.NonceDisabled: + // If nonce is disabled then we have no way to know if the packet was + // looped-back so we have to assume it indicates a duplicate address. + fallthrough + case ip.NonceNotEqual: + // If the address is a SLAAC address, do not invalidate its SLAAC prefix as an + // attempt will be made to generate a new address for it. + if err := e.removePermanentEndpointLocked(addressEndpoint, false /* allowSLAACInvalidation */, stack.AddressRemovalDADFailed, &stack.DADDupAddrDetected{HolderLinkAddress: holderLinkAddr}); err != nil { + return err + } + + prefix := addressEndpoint.Subnet() + + switch t := addressEndpoint.ConfigType(); t { + case stack.AddressConfigStatic: + case stack.AddressConfigSlaac: + if addressEndpoint.Temporary() { + // Do not reset the generation attempts counter for the prefix as the + // temporary address is being regenerated in response to a DAD conflict. + e.mu.ndp.regenerateTempSLAACAddr(prefix, false /* resetGenAttempts */) + } else { + e.mu.ndp.regenerateSLAACAddr(prefix) + } + default: + panic(fmt.Sprintf("unrecognized address config type = %d", t)) + } + + return nil + default: + panic(fmt.Sprintf("unhandled result = %d", result)) + } +} + +// Forwarding implements stack.ForwardingNetworkEndpoint. +func (e *endpoint) Forwarding() bool { + return e.forwarding.Load() == forwardingEnabled +} + +// setForwarding sets the forwarding status for the endpoint. +// +// Returns the previous forwarding status. +func (e *endpoint) setForwarding(v bool) bool { + forwarding := uint32(forwardingDisabled) + if v { + forwarding = forwardingEnabled + } + + return e.forwarding.Swap(forwarding) != forwardingDisabled +} + +// SetForwarding implements stack.ForwardingNetworkEndpoint. +func (e *endpoint) SetForwarding(forwarding bool) bool { + e.mu.Lock() + defer e.mu.Unlock() + + prevForwarding := e.setForwarding(forwarding) + if prevForwarding == forwarding { + return prevForwarding + } + + allRoutersGroups := [...]tcpip.Address{ + header.IPv6AllRoutersInterfaceLocalMulticastAddress, + header.IPv6AllRoutersLinkLocalMulticastAddress, + header.IPv6AllRoutersSiteLocalMulticastAddress, + } + + if forwarding { + // As per RFC 4291 section 2.8: + // + // A router is required to recognize all addresses that a host is + // required to recognize, plus the following addresses as identifying + // itself: + // + // o The All-Routers multicast addresses defined in Section 2.7.1. + // + // As per RFC 4291 section 2.7.1, + // + // All Routers Addresses: FF01:0:0:0:0:0:0:2 + // FF02:0:0:0:0:0:0:2 + // FF05:0:0:0:0:0:0:2 + // + // The above multicast addresses identify the group of all IPv6 routers, + // within scope 1 (interface-local), 2 (link-local), or 5 (site-local). + for _, g := range allRoutersGroups { + if err := e.joinGroupLocked(g); err != nil { + // joinGroupLocked only returns an error if the group address is not a + // valid IPv6 multicast address. + panic(fmt.Sprintf("e.joinGroupLocked(%s): %s", g, err)) + } + } + } else { + for _, g := range allRoutersGroups { + switch err := e.leaveGroupLocked(g).(type) { + case nil: + case *tcpip.ErrBadLocalAddress: + // The endpoint may have already left the multicast group. + default: + panic(fmt.Sprintf("e.leaveGroupLocked(%s): %s", g, err)) + } + } + } + + e.mu.ndp.forwardingChanged(forwarding) + return prevForwarding +} + +// MulticastForwarding implements stack.MulticastForwardingNetworkEndpoint. +func (e *endpoint) MulticastForwarding() bool { + return e.multicastForwarding.Load() == forwardingEnabled +} + +// SetMulticastForwarding implements stack.MulticastForwardingNetworkEndpoint. +func (e *endpoint) SetMulticastForwarding(forwarding bool) bool { + updatedForwarding := uint32(forwardingDisabled) + if forwarding { + updatedForwarding = forwardingEnabled + } + + return e.multicastForwarding.Swap(updatedForwarding) != forwardingDisabled +} + +// Enable implements stack.NetworkEndpoint. +func (e *endpoint) Enable() tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + // If the NIC is not enabled, the endpoint can't do anything meaningful so + // don't enable the endpoint. + if !e.nic.Enabled() { + return &tcpip.ErrNotPermitted{} + } + + // If the endpoint is already enabled, there is nothing for it to do. + if !e.setEnabled(true) { + return nil + } + + // Perform DAD on the all the unicast IPv6 endpoints that are in the permanent + // state. + // + // Addresses may have already completed DAD but in the time since the endpoint + // was last enabled, other devices may have acquired the same addresses. + var err tcpip.Error + e.mu.addressableEndpointState.ForEachEndpoint(func(addressEndpoint stack.AddressEndpoint) bool { + addr := addressEndpoint.AddressWithPrefix().Address + if !header.IsV6UnicastAddress(addr) { + return true + } + + switch kind := addressEndpoint.GetKind(); kind { + case stack.Permanent: + addressEndpoint.SetKind(stack.PermanentTentative) + fallthrough + case stack.PermanentTentative: + err = e.mu.ndp.startDuplicateAddressDetection(addr, addressEndpoint) + return err == nil + case stack.Temporary, stack.PermanentExpired: + return true + default: + panic(fmt.Sprintf("address %s has unknown kind %d", addressEndpoint.AddressWithPrefix(), kind)) + } + }) + // It is important to enable after starting DAD on all the addresses so that + // if DAD is disabled, the Tentative state is not observed. + // + // Must be called after Enabled has been set. + e.mu.addressableEndpointState.OnNetworkEndpointEnabledChanged() + if err != nil { + return err + } + + // Groups may have been joined when the endpoint was disabled, or the + // endpoint may have left groups from the perspective of MLD when the + // endpoint was disabled. Either way, we need to let routers know to + // send us multicast traffic. + e.mu.mld.initializeAll() + + // Join the IPv6 All-Nodes Multicast group if the stack is configured to + // use IPv6. This is required to ensure that this node properly receives + // and responds to the various NDP messages that are destined to the + // all-nodes multicast address. An example is the Neighbor Advertisement + // when we perform Duplicate Address Detection, or Router Advertisement + // when we do Router Discovery. See RFC 4862, section 5.4.2 and RFC 4861 + // section 4.2 for more information. + // + // Also auto-generate an IPv6 link-local address based on the endpoint's + // link address if it is configured to do so. Note, each interface is + // required to have IPv6 link-local unicast address, as per RFC 4291 + // section 2.1. + + // Join the All-Nodes multicast group before starting DAD as responses to DAD + // (NDP NS) messages may be sent to the All-Nodes multicast group if the + // source address of the NDP NS is the unspecified address, as per RFC 4861 + // section 7.2.4. + if err := e.joinGroupLocked(header.IPv6AllNodesMulticastAddress); err != nil { + // joinGroupLocked only returns an error if the group address is not a valid + // IPv6 multicast address. + panic(fmt.Sprintf("e.joinGroupLocked(%s): %s", header.IPv6AllNodesMulticastAddress, err)) + } + + // Do not auto-generate an IPv6 link-local address for loopback devices. + if e.protocol.options.AutoGenLinkLocal && !e.nic.IsLoopback() { + // The valid and preferred lifetime is infinite for the auto-generated + // link-local address. + e.mu.ndp.doSLAAC(header.IPv6LinkLocalPrefix.Subnet(), header.NDPInfiniteLifetime, header.NDPInfiniteLifetime) + } + + e.mu.ndp.startSolicitingRouters() + return nil +} + +// Enabled implements stack.NetworkEndpoint. +func (e *endpoint) Enabled() bool { + return e.nic.Enabled() && e.isEnabled() +} + +// isEnabled returns true if the endpoint is enabled, regardless of the +// enabled status of the NIC. +func (e *endpoint) isEnabled() bool { + return e.enabled.Load() == 1 +} + +// setEnabled sets the enabled status for the endpoint. +// +// Returns true if the enabled status was updated. +func (e *endpoint) setEnabled(v bool) bool { + if v { + return e.enabled.Swap(1) == 0 + } + return e.enabled.Swap(0) == 1 +} + +// Disable implements stack.NetworkEndpoint. +func (e *endpoint) Disable() { + e.mu.Lock() + defer e.mu.Unlock() + e.disableLocked() +} + +func (e *endpoint) disableLocked() { + if !e.Enabled() { + return + } + + e.mu.ndp.stopSolicitingRouters() + e.mu.ndp.cleanupState() + + // The endpoint may have already left the multicast group. + switch err := e.leaveGroupLocked(header.IPv6AllNodesMulticastAddress).(type) { + case nil, *tcpip.ErrBadLocalAddress: + default: + panic(fmt.Sprintf("unexpected error when leaving group = %s: %s", header.IPv6AllNodesMulticastAddress, err)) + } + + // Leave groups from the perspective of MLD so that routers know that + // we are no longer interested in the group. + e.mu.mld.softLeaveAll() + + // Stop DAD for all the tentative unicast addresses. + e.mu.addressableEndpointState.ForEachEndpoint(func(addressEndpoint stack.AddressEndpoint) bool { + addrWithPrefix := addressEndpoint.AddressWithPrefix() + switch kind := addressEndpoint.GetKind(); kind { + case stack.Permanent, stack.PermanentTentative: + if header.IsV6UnicastAddress(addrWithPrefix.Address) { + e.mu.ndp.stopDuplicateAddressDetection(addrWithPrefix.Address, &stack.DADAborted{}) + } + case stack.Temporary, stack.PermanentExpired: + default: + panic(fmt.Sprintf("address %s has unknown address kind %d", addrWithPrefix, kind)) + } + return true + }) + + if !e.setEnabled(false) { + panic("should have only done work to disable the endpoint if it was enabled") + } + + // Must be called after Enabled has been set. + e.mu.addressableEndpointState.OnNetworkEndpointEnabledChanged() +} + +// DefaultTTL is the default hop limit for this endpoint. +func (e *endpoint) DefaultTTL() uint8 { + return e.protocol.DefaultTTL() +} + +// MTU implements stack.NetworkEndpoint. It returns the link-layer MTU minus the +// network layer max header length. +func (e *endpoint) MTU() uint32 { + networkMTU, err := calculateNetworkMTU(e.nic.MTU(), header.IPv6MinimumSize) + if err != nil { + return 0 + } + return networkMTU +} + +// MaxHeaderLength returns the maximum length needed by ipv6 headers (and +// underlying protocols). +func (e *endpoint) MaxHeaderLength() uint16 { + // TODO(gvisor.dev/issues/5035): The maximum header length returned here does + // not open the possibility for the caller to know about size required for + // extension headers. + return e.nic.MaxHeaderLength() + header.IPv6MinimumSize +} + +func addIPHeader(srcAddr, dstAddr tcpip.Address, pkt *stack.PacketBuffer, params stack.NetworkHeaderParams, extensionHeaders header.IPv6ExtHdrSerializer) tcpip.Error { + if params.ExperimentOptionValue != 0 { + extensionHeaders = append(extensionHeaders, &header.IPv6ExperimentExtHdr{Value: params.ExperimentOptionValue}) + } + extHdrsLen := extensionHeaders.Length() + length := pkt.Size() + extensionHeaders.Length() + if length > math.MaxUint16 { + return &tcpip.ErrMessageTooLong{} + } + header.IPv6(pkt.NetworkHeader().Push(header.IPv6MinimumSize + extHdrsLen)).Encode(&header.IPv6Fields{ + PayloadLength: uint16(length), + TransportProtocol: params.Protocol, + HopLimit: params.TTL, + TrafficClass: params.TOS, + SrcAddr: srcAddr, + DstAddr: dstAddr, + ExtensionHeaders: extensionHeaders, + }) + pkt.NetworkProtocolNumber = ProtocolNumber + return nil +} + +func packetMustBeFragmented(pkt *stack.PacketBuffer, networkMTU uint32) bool { + payload := len(pkt.TransportHeader().Slice()) + pkt.Data().Size() + return pkt.GSOOptions.Type == stack.GSONone && uint32(payload) > networkMTU +} + +// handleFragments fragments pkt and calls the handler function on each +// fragment. It returns the number of fragments handled and the number of +// fragments left to be processed. The IP header must already be present in the +// original packet. The transport header protocol number is required to avoid +// parsing the IPv6 extension headers. +func (e *endpoint) handleFragments(r *stack.Route, networkMTU uint32, pkt *stack.PacketBuffer, transProto tcpip.TransportProtocolNumber, handler func(*stack.PacketBuffer) tcpip.Error) (int, int, tcpip.Error) { + networkHeader := header.IPv6(pkt.NetworkHeader().Slice()) + + // TODO(gvisor.dev/issue/3912): Once the Authentication or ESP Headers are + // supported for outbound packets, their length should not affect the fragment + // maximum payload length because they should only be transmitted once. + fragmentPayloadLen := (networkMTU - header.IPv6FragmentHeaderSize) &^ 7 + if fragmentPayloadLen < header.IPv6FragmentExtHdrFragmentOffsetBytesPerUnit { + // We need at least 8 bytes of space left for the fragmentable part because + // the fragment payload must obviously be non-zero and must be a multiple + // of 8 as per RFC 8200 section 4.5: + // Each complete fragment, except possibly the last ("rightmost") one, is + // an integer multiple of 8 octets long. + return 0, 1, &tcpip.ErrMessageTooLong{} + } + + if fragmentPayloadLen < uint32(len(pkt.TransportHeader().Slice())) { + // As per RFC 8200 Section 4.5, the Transport Header is expected to be small + // enough to fit in the first fragment. + return 0, 1, &tcpip.ErrMessageTooLong{} + } + + pf := fragmentation.MakePacketFragmenter(pkt, fragmentPayloadLen, calculateFragmentReserve(pkt)) + defer pf.Release() + id := e.getFragmentID() + + var n int + for { + fragPkt, more := buildNextFragment(&pf, networkHeader, transProto, id) + err := handler(fragPkt) + fragPkt.DecRef() + if err != nil { + return n, pf.RemainingFragmentCount() + 1, err + } + n++ + if !more { + return n, pf.RemainingFragmentCount(), nil + } + } +} + +// WritePacket writes a packet to the given destination address and protocol. +func (e *endpoint) WritePacket(r *stack.Route, params stack.NetworkHeaderParams, pkt *stack.PacketBuffer) tcpip.Error { + dstAddr := r.RemoteAddress() + if err := addIPHeader(r.LocalAddress(), dstAddr, pkt, params, nil /* extensionHeaders */); err != nil { + return err + } + + // iptables filtering. All packets that reach here are locally + // generated. + outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) + if ok := e.protocol.stack.IPTables().CheckOutput(pkt, r, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesOutputDropped.Increment() + return nil + } + + // If the packet is manipulated as per DNAT Output rules, handle packet + // based on destination address and do not send the packet to link + // layer. + // + // We should do this for every packet, rather than only DNATted packets, but + // removing this check short circuits broadcasts before they are sent out to + // other hosts. + if netHeader := header.IPv6(pkt.NetworkHeader().Slice()); dstAddr != netHeader.DestinationAddress() { + if ep := e.protocol.findEndpointWithAddress(netHeader.DestinationAddress()); ep != nil { + // Since we rewrote the packet but it is being routed back to us, we + // can safely assume the checksum is valid. + ep.handleLocalPacket(pkt, true /* canSkipRXChecksum */) + return nil + } + } + + return e.writePacket(r, pkt, params.Protocol, false /* headerIncluded */) +} + +func (e *endpoint) writePacket(r *stack.Route, pkt *stack.PacketBuffer, protocol tcpip.TransportProtocolNumber, headerIncluded bool) tcpip.Error { + if r.Loop()&stack.PacketLoop != 0 { + // If the packet was generated by the stack (not a raw/packet endpoint + // where a packet may be written with the header included), then we can + // safely assume the checksum is valid. + e.handleLocalPacket(pkt, !headerIncluded /* canSkipRXChecksum */) + } + if r.Loop()&stack.PacketOut == 0 { + return nil + } + + // Postrouting NAT can only change the source address, and does not alter the + // route or outgoing interface of the packet. + outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) + if ok := e.protocol.stack.IPTables().CheckPostrouting(pkt, r, e, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesPostroutingDropped.Increment() + return nil + } + + stats := e.stats.ip + networkMTU, err := calculateNetworkMTU(e.nic.MTU(), uint32(len(pkt.NetworkHeader().Slice()))) + if err != nil { + stats.OutgoingPacketErrors.Increment() + return err + } + + if packetMustBeFragmented(pkt, networkMTU) { + if pkt.NetworkPacketInfo.IsForwardedPacket { + // As per RFC 2460, section 4.5: + // Unlike IPv4, fragmentation in IPv6 is performed only by source nodes, + // not by routers along a packet's delivery path. + return &tcpip.ErrMessageTooLong{} + } + sent, remain, err := e.handleFragments(r, networkMTU, pkt, protocol, func(fragPkt *stack.PacketBuffer) tcpip.Error { + // TODO(gvisor.dev/issue/3884): Evaluate whether we want to send each + // fragment one by one using WritePacket() (current strategy) or if we + // want to create a PacketBufferList from the fragments and feed it to + // WritePackets(). It'll be faster but cost more memory. + return e.nic.WritePacket(r, fragPkt) + }) + stats.PacketsSent.IncrementBy(uint64(sent)) + stats.OutgoingPacketErrors.IncrementBy(uint64(remain)) + return err + } + + if err := e.nic.WritePacket(r, pkt); err != nil { + stats.OutgoingPacketErrors.Increment() + return err + } + + stats.PacketsSent.Increment() + return nil +} + +// WriteHeaderIncludedPacket implements stack.NetworkEndpoint. +func (e *endpoint) WriteHeaderIncludedPacket(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error { + // The packet already has an IP header, but there are a few required checks. + h, ok := pkt.Data().PullUp(header.IPv6MinimumSize) + if !ok { + return &tcpip.ErrMalformedHeader{} + } + ipH := header.IPv6(h) + + // Always set the payload length. + pktSize := pkt.Data().Size() + ipH.SetPayloadLength(uint16(pktSize - header.IPv6MinimumSize)) + + // Set the source address when zero. + if ipH.SourceAddress() == header.IPv6Any { + ipH.SetSourceAddress(r.LocalAddress()) + } + + // Populate the packet buffer's network header and don't allow an invalid + // packet to be sent. + // + // Note that parsing only makes sure that the packet is well formed as per the + // wire format. We also want to check if the header's fields are valid before + // sending the packet. + proto, _, _, _, ok := parse.IPv6(pkt) + if !ok || !header.IPv6(pkt.NetworkHeader().Slice()).IsValid(pktSize) { + return &tcpip.ErrMalformedHeader{} + } + + return e.writePacket(r, pkt, proto, true /* headerIncluded */) +} + +func validateAddressesForForwarding(h header.IPv6) ip.ForwardingError { + srcAddr := h.SourceAddress() + + // As per RFC 4291 section 2.5.2, + // + // The address 0:0:0:0:0:0:0:0 is called the unspecified address. It + // must never be assigned to any node. It indicates the absence of an + // address. One example of its use is in the Source Address field of + // any IPv6 packets sent by an initializing host before it has learned + // its own address. + // + // The unspecified address must not be used as the destination address + // of IPv6 packets or in IPv6 Routing headers. An IPv6 packet with a + // source address of unspecified must never be forwarded by an IPv6 + // router. + if srcAddr.Unspecified() { + return &ip.ErrInitializingSourceAddress{} + } + + // As per RFC 4291 section 2.5.6, + // + // Routers must not forward any packets with Link-Local source or + // destination addresses to other links. + if header.IsV6LinkLocalUnicastAddress(srcAddr) { + return &ip.ErrLinkLocalSourceAddress{} + } + + if dstAddr := h.DestinationAddress(); header.IsV6LinkLocalUnicastAddress(dstAddr) || header.IsV6LinkLocalMulticastAddress(dstAddr) { + return &ip.ErrLinkLocalDestinationAddress{} + } + return nil +} + +// forwardUnicastPacket attempts to forward a unicast packet to its final +// destination. +func (e *endpoint) forwardUnicastPacket(pkt *stack.PacketBuffer) ip.ForwardingError { + h := header.IPv6(pkt.NetworkHeader().Slice()) + + if err := validateAddressesForForwarding(h); err != nil { + return err + } + + hopLimit := h.HopLimit() + if hopLimit <= 1 { + // As per RFC 4443 section 3.3, + // + // If a router receives a packet with a Hop Limit of zero, or if a + // router decrements a packet's Hop Limit to zero, it MUST discard the + // packet and originate an ICMPv6 Time Exceeded message with Code 0 to + // the source of the packet. This indicates either a routing loop or + // too small an initial Hop Limit value. + // + // We return the original error rather than the result of returning + // the ICMP packet because the original error is more relevant to + // the caller. + _ = e.protocol.returnError(&icmpReasonHopLimitExceeded{}, pkt, false /* deliveredLocally */) + return &ip.ErrTTLExceeded{} + } + + stk := e.protocol.stack + + dstAddr := h.DestinationAddress() + + // Check if the destination is owned by the stack. + if ep := e.protocol.findEndpointWithAddress(dstAddr); ep != nil { + inNicName := stk.FindNICNameFromID(e.nic.ID()) + outNicName := stk.FindNICNameFromID(ep.nic.ID()) + if ok := stk.IPTables().CheckForward(pkt, inNicName, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesForwardDropped.Increment() + return nil + } + + // The packet originally arrived on e so provide its NIC as the input NIC. + ep.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) + return nil + } + + // Check extension headers for any errors requiring action during forwarding. + if err := e.processExtensionHeaders(h, pkt, true /* forwarding */); err != nil { + return &ip.ErrParameterProblem{} + } + + r, err := stk.FindRoute(0, tcpip.Address{}, dstAddr, ProtocolNumber, false /* multicastLoop */) + switch err.(type) { + case nil: + case *tcpip.ErrNetworkUnreachable: + // We return the original error rather than the result of returning the + // ICMP packet because the original error is more relevant to the caller. + _ = e.protocol.returnError(&icmpReasonNetUnreachable{}, pkt, false /* deliveredLocally */) + return &ip.ErrHostUnreachable{} + default: + return &ip.ErrOther{Err: err} + } + defer r.Release() + + return e.forwardPacketWithRoute(r, pkt) +} + +// forwardPacketWithRoute emits the pkt using the provided route. +// +// This method should be invoked by the endpoint that received the pkt. +func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketBuffer) ip.ForwardingError { + h := header.IPv6(pkt.NetworkHeader().Slice()) + stk := e.protocol.stack + + inNicName := stk.FindNICNameFromID(e.nic.ID()) + outNicName := stk.FindNICNameFromID(route.NICID()) + if ok := stk.IPTables().CheckForward(pkt, inNicName, outNicName); !ok { + // iptables is telling us to drop the packet. + e.stats.ip.IPTablesForwardDropped.Increment() + return nil + } + + hopLimit := h.HopLimit() + + // We need to do a deep copy of the IP packet because + // WriteHeaderIncludedPacket takes ownership of the packet buffer, but we do + // not own it. + newPkt := pkt.DeepCopyForForwarding(int(route.MaxHeaderLength())) + defer newPkt.DecRef() + newHdr := header.IPv6(newPkt.NetworkHeader().Slice()) + + // As per RFC 8200 section 3, + // + // Hop Limit 8-bit unsigned integer. Decremented by 1 by + // each node that forwards the packet. + newHdr.SetHopLimit(hopLimit - 1) + + forwardToEp, ok := e.protocol.getEndpointForNIC(route.NICID()) + if !ok { + // The interface was removed after we obtained the route. + return &ip.ErrUnknownOutputEndpoint{} + } + + switch err := forwardToEp.writePacket(route, newPkt, newPkt.TransportProtocolNumber, true /* headerIncluded */); err.(type) { + case nil: + return nil + case *tcpip.ErrMessageTooLong: + // As per RFC 4443, section 3.2: + // A Packet Too Big MUST be sent by a router in response to a packet that + // it cannot forward because the packet is larger than the MTU of the + // outgoing link. + _ = e.protocol.returnError(&icmpReasonPacketTooBig{}, pkt, false /* deliveredLocally */) + return &ip.ErrMessageTooLong{} + case *tcpip.ErrNoBufferSpace: + return &ip.ErrOutgoingDeviceNoBufferSpace{} + default: + return &ip.ErrOther{Err: err} + } +} + +// HandlePacket is called by the link layer when new ipv6 packets arrive for +// this endpoint. +func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { + stats := e.stats.ip + + stats.PacketsReceived.Increment() + + if !e.isEnabled() { + stats.DisabledPacketsReceived.Increment() + return + } + + hView, ok := e.protocol.parseAndValidate(pkt) + if !ok { + stats.MalformedPacketsReceived.Increment() + return + } + defer hView.Release() + h := header.IPv6(hView.AsSlice()) + + if !checkV4Mapped(h, stats) { + return + } + + if !e.nic.IsLoopback() { + if !e.protocol.options.AllowExternalLoopbackTraffic { + if header.IsV6LoopbackAddress(h.SourceAddress()) { + stats.InvalidSourceAddressesReceived.Increment() + return + } + + if header.IsV6LoopbackAddress(h.DestinationAddress()) { + stats.InvalidDestinationAddressesReceived.Increment() + return + } + } + + if e.protocol.stack.HandleLocal() { + addressEndpoint := e.AcquireAssignedAddress(header.IPv6(pkt.NetworkHeader().Slice()).SourceAddress(), e.nic.Promiscuous(), stack.CanBePrimaryEndpoint, true /* readOnly */) + if addressEndpoint != nil { + // The source address is one of our own, so we never should have gotten + // a packet like this unless HandleLocal is false or our NIC is the + // loopback interface. + stats.InvalidSourceAddressesReceived.Increment() + return + } + } + + // Loopback traffic skips the prerouting chain. + inNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) + if ok := e.protocol.stack.IPTables().CheckPrerouting(pkt, e, inNicName); !ok { + // iptables is telling us to drop the packet. + stats.IPTablesPreroutingDropped.Increment() + return + } + } + + // CheckPrerouting can modify the backing storage of the packet, so refresh + // the header. + h = header.IPv6(pkt.NetworkHeader().Slice()) + e.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) +} + +// handleLocalPacket is like HandlePacket except it does not perform the +// prerouting iptables hook or check for loopback traffic that originated from +// outside of the netstack (i.e. martian loopback packets). +func (e *endpoint) handleLocalPacket(pkt *stack.PacketBuffer, canSkipRXChecksum bool) { + stats := e.stats.ip + stats.PacketsReceived.Increment() + + pkt = pkt.CloneToInbound() + defer pkt.DecRef() + pkt.RXChecksumValidated = canSkipRXChecksum + + hView, ok := e.protocol.parseAndValidate(pkt) + if !ok { + stats.MalformedPacketsReceived.Increment() + return + } + defer hView.Release() + h := header.IPv6(hView.AsSlice()) + + if !checkV4Mapped(h, stats) { + return + } + + e.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) +} + +// forwardMulticastPacket validates a multicast pkt and attempts to forward it. +// +// This method should be invoked for incoming multicast packets using the +// endpoint that received the packet. +func (e *endpoint) forwardMulticastPacket(h header.IPv6, pkt *stack.PacketBuffer) ip.ForwardingError { + if err := validateAddressesForForwarding(h); err != nil { + return err + } + + // Check extension headers for any errors. + if err := e.processExtensionHeaders(h, pkt, true /* forwarding */); err != nil { + return &ip.ErrParameterProblem{} + } + + routeKey := stack.UnicastSourceAndMulticastDestination{ + Source: h.SourceAddress(), + Destination: h.DestinationAddress(), + } + + // The pkt has been validated. Consequently, if a route is not found, then + // the pkt can safely be queued. + result, hasBufferSpace := e.protocol.multicastRouteTable.GetRouteOrInsertPending(routeKey, pkt) + + if !hasBufferSpace { + // Unable to queue the pkt. Silently drop it. + return &ip.ErrNoMulticastPendingQueueBufferSpace{} + } + + switch result.GetRouteResultState { + case multicast.InstalledRouteFound: + // Attempt to forward the pkt using an existing route. + return e.forwardValidatedMulticastPacket(pkt, result.InstalledRoute) + case multicast.NoRouteFoundAndPendingInserted: + e.emitMulticastEvent(func(disp stack.MulticastForwardingEventDispatcher) { + disp.OnMissingRoute(stack.MulticastPacketContext{ + stack.UnicastSourceAndMulticastDestination{h.SourceAddress(), h.DestinationAddress()}, + e.nic.ID(), + }) + }) + case multicast.PacketQueuedInPendingRoute: + default: + panic(fmt.Sprintf("unexpected GetRouteResultState: %s", result.GetRouteResultState)) + } + return &ip.ErrHostUnreachable{} +} + +// forwardValidatedMulticastPacket attempts to forward the pkt using the +// provided installedRoute. +// +// This method should be invoked by the endpoint that received the pkt. +func (e *endpoint) forwardValidatedMulticastPacket(pkt *stack.PacketBuffer, installedRoute *multicast.InstalledRoute) ip.ForwardingError { + // Per RFC 1812 section 5.2.1.3, + // + // Based on the IP source and destination addresses found in the datagram + // header, the router determines whether the datagram has been received + // on the proper interface for forwarding. If not, the datagram is + // dropped silently. + if e.nic.ID() != installedRoute.ExpectedInputInterface { + h := header.IPv6(pkt.NetworkHeader().Slice()) + e.emitMulticastEvent(func(disp stack.MulticastForwardingEventDispatcher) { + disp.OnUnexpectedInputInterface(stack.MulticastPacketContext{ + stack.UnicastSourceAndMulticastDestination{h.SourceAddress(), h.DestinationAddress()}, + e.nic.ID(), + }, installedRoute.ExpectedInputInterface) + }) + return &ip.ErrUnexpectedMulticastInputInterface{} + } + + for _, outgoingInterface := range installedRoute.OutgoingInterfaces { + if err := e.forwardMulticastPacketForOutgoingInterface(pkt, outgoingInterface); err != nil { + e.handleForwardingError(err) + continue + } + // The pkt was successfully forwarded. Mark the route as used. + installedRoute.SetLastUsedTimestamp(e.protocol.stack.Clock().NowMonotonic()) + } + return nil +} + +// forwardMulticastPacketForOutgoingInterface attempts to forward the pkt out +// of the provided outgoing interface. +// +// This method should be invoked by the endpoint that received the pkt. +func (e *endpoint) forwardMulticastPacketForOutgoingInterface(pkt *stack.PacketBuffer, outgoingInterface stack.MulticastRouteOutgoingInterface) ip.ForwardingError { + h := header.IPv6(pkt.NetworkHeader().Slice()) + + // Per RFC 1812 section 5.2.1.3, + // + // A copy of the multicast datagram is forwarded out each outgoing + // interface whose minimum TTL value is less than or equal to the TTL + // value in the datagram header. + // + // Copying of the packet is deferred to forwardPacketWithRoute since unicast + // and multicast both require a copy. + if outgoingInterface.MinTTL > h.HopLimit() { + return &ip.ErrTTLExceeded{} + } + + route := e.protocol.stack.NewRouteForMulticast(outgoingInterface.ID, h.DestinationAddress(), e.NetworkProtocolNumber()) + + if route == nil { + // Failed to convert to a stack.Route. This likely means that the outgoing + // endpoint no longer exists. + return &ip.ErrHostUnreachable{} + } + defer route.Release() + return e.forwardPacketWithRoute(route, pkt) +} + +// handleForwardingError processes the provided err and increments any relevant +// counters. +func (e *endpoint) handleForwardingError(err ip.ForwardingError) { + stats := e.stats.ip + switch err := err.(type) { + case nil: + return + case *ip.ErrInitializingSourceAddress: + stats.Forwarding.InitializingSource.Increment() + case *ip.ErrLinkLocalSourceAddress: + stats.Forwarding.LinkLocalSource.Increment() + case *ip.ErrLinkLocalDestinationAddress: + stats.Forwarding.LinkLocalDestination.Increment() + case *ip.ErrTTLExceeded: + stats.Forwarding.ExhaustedTTL.Increment() + case *ip.ErrHostUnreachable: + stats.Forwarding.Unrouteable.Increment() + case *ip.ErrParameterProblem: + stats.Forwarding.ExtensionHeaderProblem.Increment() + case *ip.ErrMessageTooLong: + stats.Forwarding.PacketTooBig.Increment() + case *ip.ErrNoMulticastPendingQueueBufferSpace: + stats.Forwarding.NoMulticastPendingQueueBufferSpace.Increment() + case *ip.ErrUnexpectedMulticastInputInterface: + stats.Forwarding.UnexpectedMulticastInputInterface.Increment() + case *ip.ErrUnknownOutputEndpoint: + stats.Forwarding.UnknownOutputEndpoint.Increment() + case *ip.ErrOutgoingDeviceNoBufferSpace: + stats.Forwarding.OutgoingDeviceNoBufferSpace.Increment() + case *ip.ErrOther: + switch err := err.Err.(type) { + case *tcpip.ErrClosedForSend: + stats.Forwarding.OutgoingDeviceClosedForSend.Increment() + default: + panic(fmt.Sprintf("unrecognized tcpip forwarding error: %s", err)) + } + default: + panic(fmt.Sprintf("unrecognized forwarding error: %s", err)) + } + stats.Forwarding.Errors.Increment() +} + +func (e *endpoint) handleValidatedPacket(h header.IPv6, pkt *stack.PacketBuffer, inNICName string) { + pkt.NICID = e.nic.ID() + + // Raw socket packets are delivered based solely on the transport protocol + // number. We only require that the packet be valid IPv6. + e.dispatcher.DeliverRawPacket(h.TransportProtocol(), pkt) + + stats := e.stats.ip + stats.ValidPacketsReceived.Increment() + + srcAddr := h.SourceAddress() + dstAddr := h.DestinationAddress() + + // As per RFC 4291 section 2.7: + // Multicast addresses must not be used as source addresses in IPv6 + // packets or appear in any Routing header. + if header.IsV6MulticastAddress(srcAddr) { + stats.InvalidSourceAddressesReceived.Increment() + return + } + + if header.IsV6MulticastAddress(dstAddr) { + // Handle all packets destined to a multicast address separately. Unlike + // unicast, these packets can be both delivered locally and forwarded. See + // RFC 1812 section 5.2.3 for details regarding the forwarding/local + // delivery decision. + + multicastForwading := e.MulticastForwarding() && e.protocol.multicastForwarding() + + if multicastForwading { + e.handleForwardingError(e.forwardMulticastPacket(h, pkt)) + } + + if e.IsInGroup(dstAddr) { + e.deliverPacketLocally(h, pkt, inNICName) + return + } + + if !multicastForwading { + // Only consider the destination address invalid if we didn't attempt to + // forward the pkt and it was not delivered locally. + stats.InvalidDestinationAddressesReceived.Increment() + } + + return + } + + // The destination address should be an address we own for us to receive the + // packet. Otherwise, attempt to forward the packet. + if addressEndpoint := e.AcquireAssignedAddress(dstAddr, e.nic.Promiscuous(), stack.CanBePrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + e.deliverPacketLocally(h, pkt, inNICName) + } else if e.Forwarding() { + e.handleForwardingError(e.forwardUnicastPacket(pkt)) + } else { + stats.InvalidDestinationAddressesReceived.Increment() + } +} + +func (e *endpoint) deliverPacketLocally(h header.IPv6, pkt *stack.PacketBuffer, inNICName string) { + stats := e.stats.ip + + // iptables filtering. All packets that reach here are intended for + // this machine and need not be forwarded. + if ok := e.protocol.stack.IPTables().CheckInput(pkt, inNICName); !ok { + // iptables is telling us to drop the packet. + stats.IPTablesInputDropped.Increment() + return + } + + // Any returned error is only useful for terminating execution early, but + // we have nothing left to do, so we can drop it. + _ = e.processExtensionHeaders(h, pkt, false /* forwarding */) +} + +func (e *endpoint) processExtensionHeader(it *header.IPv6PayloadIterator, pkt **stack.PacketBuffer, h header.IPv6, routerAlert **header.IPv6RouterAlertOption, hasFragmentHeader *bool, forwarding bool) (bool, error) { + stats := e.stats.ip + dstAddr := h.DestinationAddress() + // Keep track of the start of the previous header so we can report the + // special case of a Hop by Hop at a location other than at the start. + previousHeaderStart := it.HeaderOffset() + extHdr, done, err := it.Next() + if err != nil { + stats.MalformedPacketsReceived.Increment() + return true, err + } + if done { + return true, nil + } + defer extHdr.Release() + + // As per RFC 8200, section 4: + // + // Extension headers (except for the Hop-by-Hop Options header) are + // not processed, inserted, or deleted by any node along a packet's + // delivery path until the packet reaches the node identified in the + // Destination Address field of the IPv6 header. + // + // Furthermore, as per RFC 8200 section 4.1, the Hop By Hop extension + // header is restricted to appear first in the list of extension headers. + // + // Therefore, we can immediately return once we hit any header other + // than the Hop-by-Hop header while forwarding a packet. + if forwarding { + if _, ok := extHdr.(header.IPv6HopByHopOptionsExtHdr); !ok { + return true, nil + } + } + + switch extHdr := extHdr.(type) { + case header.IPv6HopByHopOptionsExtHdr: + if err := e.processIPv6HopByHopOptionsExtHdr(&extHdr, it, *pkt, dstAddr, routerAlert, previousHeaderStart, forwarding); err != nil { + return true, err + } + case header.IPv6RoutingExtHdr: + if err := e.processIPv6RoutingExtHeader(&extHdr, it, *pkt); err != nil { + return true, err + } + case header.IPv6FragmentExtHdr: + *hasFragmentHeader = true + if extHdr.IsAtomic() { + // This fragment extension header indicates that this packet is an + // atomic fragment. An atomic fragment is a fragment that contains + // all the data required to reassemble a full packet. As per RFC 6946, + // atomic fragments must not interfere with "normal" fragmented traffic + // so we skip processing the fragment instead of feeding it through the + // reassembly process below. + return false, nil + } + + if err := e.processFragmentExtHdr(&extHdr, it, pkt, h); err != nil { + return true, err + } + case header.IPv6DestinationOptionsExtHdr: + if err := e.processIPv6DestinationOptionsExtHdr(&extHdr, it, *pkt, dstAddr); err != nil { + return true, err + } + case header.IPv6RawPayloadHeader: + if err := e.processIPv6RawPayloadHeader(&extHdr, it, *pkt, *routerAlert, previousHeaderStart, *hasFragmentHeader); err != nil { + return true, err + } + case header.IPv6ExperimentExtHdr: + default: + // Since the iterator returns IPv6RawPayloadHeader for unknown Extension + // Header IDs this should never happen unless we missed a supported type + // here. + panic(fmt.Sprintf("unrecognized type from it.Next() = %T", extHdr)) + } + return false, nil +} + +// processExtensionHeaders processes the extension headers in the given packet. +// Returns an error if the processing of a header failed or if the packet should +// be discarded. +func (e *endpoint) processExtensionHeaders(h header.IPv6, pkt *stack.PacketBuffer, forwarding bool) error { + // Create a VV to parse the packet. We don't plan to modify anything here. + // vv consists of: + // - Any IPv6 header bytes after the first 40 (i.e. extensions). + // - The transport header, if present. + // - Any other payload data. + v := pkt.NetworkHeader().View() + if v != nil { + v.TrimFront(header.IPv6MinimumSize) + } + buf := buffer.MakeWithView(v) + buf.Append(pkt.TransportHeader().View()) + dataBuf := pkt.Data().ToBuffer() + buf.Merge(&dataBuf) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(h.NextHeader()), buf) + + // Clone the pkt here because fragment header processing can replace this + // packet with a new one that has an extra reference. Cloning here + // keeps it so this method owns the packet either way so the packet can always + // be DecRef'd after processing is done. + processingPkt := pkt.Clone() + defer func() { + processingPkt.DecRef() + it.Release() + }() + + var ( + hasFragmentHeader bool + routerAlert *header.IPv6RouterAlertOption + ) + for { + h := header.IPv6(pkt.NetworkHeader().Slice()) + if done, err := e.processExtensionHeader(&it, &processingPkt, h, &routerAlert, &hasFragmentHeader, forwarding); err != nil || done { + return err + } + } +} + +func (e *endpoint) processIPv6RawPayloadHeader(extHdr *header.IPv6RawPayloadHeader, it *header.IPv6PayloadIterator, pkt *stack.PacketBuffer, routerAlert *header.IPv6RouterAlertOption, previousHeaderStart uint32, hasFragmentHeader bool) error { + stats := e.stats.ip + // If the last header in the payload isn't a known IPv6 extension header, + // handle it as if it is transport layer data.å + + // Calculate the number of octets parsed from data. We want to consume all + // the data except the unparsed portion located at the end, whose size is + // extHdr.Buf.Size(). + trim := pkt.Data().Size() - int(extHdr.Buf.Size()) + + // For unfragmented packets, extHdr still contains the transport header. + // Consume that too. + // + // For reassembled fragments, pkt.TransportHeader is unset, so this is a + // no-op and pkt.Data begins with the transport header. + trim += len(pkt.TransportHeader().Slice()) + + if _, ok := pkt.Data().Consume(trim); !ok { + stats.MalformedPacketsReceived.Increment() + return fmt.Errorf("could not consume %d bytes", trim) + } + + proto := tcpip.TransportProtocolNumber(extHdr.Identifier) + // If the packet was reassembled from a fragment, it will not have a + // transport header set yet. + if len(pkt.TransportHeader().Slice()) == 0 { + e.protocol.parseTransport(pkt, proto) + } + + stats.PacketsDelivered.Increment() + if proto == header.ICMPv6ProtocolNumber { + e.handleICMP(pkt, hasFragmentHeader, routerAlert) + return nil + } + switch res := e.dispatcher.DeliverTransportPacket(proto, pkt); res { + case stack.TransportPacketHandled: + return nil + case stack.TransportPacketDestinationPortUnreachable: + // As per RFC 4443 section 3.1: + // A destination node SHOULD originate a Destination Unreachable + // message with Code 4 in response to a packet for which the + // transport protocol (e.g., UDP) has no listener, if that transport + // protocol has no alternative means to inform the sender. + _ = e.protocol.returnError(&icmpReasonPortUnreachable{}, pkt, true /* deliveredLocally */) + return fmt.Errorf("destination port unreachable") + case stack.TransportPacketProtocolUnreachable: + // As per RFC 8200 section 4. (page 7): + // Extension headers are numbered from IANA IP Protocol Numbers + // [IANA-PN], the same values used for IPv4 and IPv6. When + // processing a sequence of Next Header values in a packet, the + // first one that is not an extension header [IANA-EH] indicates + // that the next item in the packet is the corresponding upper-layer + // header. + // With more related information on page 8: + // If, as a result of processing a header, the destination node is + // required to proceed to the next header but the Next Header value + // in the current header is unrecognized by the node, it should + // discard the packet and send an ICMP Parameter Problem message to + // the source of the packet, with an ICMP Code value of 1 + // ("unrecognized Next Header type encountered") and the ICMP + // Pointer field containing the offset of the unrecognized value + // within the original packet. + // + // Which when taken together indicate that an unknown protocol should + // be treated as an unrecognized next header value. + // The location of the Next Header field is in a different place in + // the initial IPv6 header than it is in the extension headers so + // treat it specially. + prevHdrIDOffset := uint32(header.IPv6NextHeaderOffset) + if previousHeaderStart != 0 { + prevHdrIDOffset = previousHeaderStart + } + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6UnknownHeader, + pointer: prevHdrIDOffset, + }, pkt, true /* deliveredLocally */) + return fmt.Errorf("transport protocol unreachable") + default: + panic(fmt.Sprintf("unrecognized result from DeliverTransportPacket = %d", res)) + } +} + +func (e *endpoint) processIPv6RoutingExtHeader(extHdr *header.IPv6RoutingExtHdr, it *header.IPv6PayloadIterator, pkt *stack.PacketBuffer) error { + // As per RFC 8200 section 4.4, if a node encounters a routing header with + // an unrecognized routing type value, with a non-zero Segments Left + // value, the node must discard the packet and send an ICMP Parameter + // Problem, Code 0 to the packet's Source Address, pointing to the + // unrecognized Routing Type. + // + // If the Segments Left is 0, the node must ignore the Routing extension + // header and process the next header in the packet. + // + // Note, the stack does not yet handle any type of routing extension + // header, so we just make sure Segments Left is zero before processing + // the next extension header. + if extHdr.SegmentsLeft() == 0 { + return nil + } + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6ErroneousHeader, + pointer: it.ParseOffset(), + }, pkt, true /* deliveredLocally */) + return fmt.Errorf("found unrecognized routing type with non-zero segments left in header = %#v", extHdr) +} + +func (e *endpoint) processIPv6DestinationOptionsExtHdr(extHdr *header.IPv6DestinationOptionsExtHdr, it *header.IPv6PayloadIterator, pkt *stack.PacketBuffer, dstAddr tcpip.Address) error { + stats := e.stats.ip + optsIt := extHdr.Iter() + var uopt *header.IPv6UnknownExtHdrOption + defer func() { + if uopt != nil { + uopt.Data.Release() + } + }() + + for { + opt, done, err := optsIt.Next() + if err != nil { + stats.MalformedPacketsReceived.Increment() + return err + } + if uo, ok := opt.(*header.IPv6UnknownExtHdrOption); ok { + uopt = uo + } + if done { + break + } + + // We currently do not support any IPv6 Destination extension header + // options. + switch opt.UnknownAction() { + case header.IPv6OptionUnknownActionSkip: + case header.IPv6OptionUnknownActionDiscard: + return fmt.Errorf("found unknown destination header option = %#v with discard action", opt) + case header.IPv6OptionUnknownActionDiscardSendICMPNoMulticastDest: + if header.IsV6MulticastAddress(dstAddr) { + if uo, ok := opt.(*header.IPv6UnknownExtHdrOption); ok { + uopt = uo + } + return fmt.Errorf("found unknown destination header option %#v with discard action", opt) + } + fallthrough + case header.IPv6OptionUnknownActionDiscardSendICMP: + // This case satisfies a requirement of RFC 8200 section 4.2 + // which states that an unknown option starting with bits [10] should: + // + // discard the packet and, regardless of whether or not the + // packet's Destination Address was a multicast address, send an + // ICMP Parameter Problem, Code 2, message to the packet's + // Source Address, pointing to the unrecognized Option Type. + // + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6UnknownOption, + pointer: it.ParseOffset() + optsIt.OptionOffset(), + respondToMulticast: true, + }, pkt, true /* deliveredLocally */) + return fmt.Errorf("found unknown destination header option %#v with discard action", opt) + default: + panic(fmt.Sprintf("unrecognized action for an unrecognized Destination extension header option = %#v", opt)) + } + if uopt != nil { + uopt.Data.Release() + uopt = nil + } + } + return nil +} + +func (e *endpoint) processIPv6HopByHopOptionsExtHdr(extHdr *header.IPv6HopByHopOptionsExtHdr, it *header.IPv6PayloadIterator, pkt *stack.PacketBuffer, dstAddr tcpip.Address, routerAlert **header.IPv6RouterAlertOption, previousHeaderStart uint32, forwarding bool) error { + stats := e.stats.ip + // As per RFC 8200 section 4.1, the Hop By Hop extension header is + // restricted to appear immediately after an IPv6 fixed header. + if previousHeaderStart != 0 { + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6UnknownHeader, + pointer: previousHeaderStart, + }, pkt, !forwarding /* deliveredLocally */) + return fmt.Errorf("found Hop-by-Hop header = %#v with non-zero previous header offset = %d", extHdr, previousHeaderStart) + } + + optsIt := extHdr.Iter() + var uopt *header.IPv6UnknownExtHdrOption + defer func() { + if uopt != nil { + uopt.Data.Release() + } + }() + + for { + opt, done, err := optsIt.Next() + if err != nil { + stats.MalformedPacketsReceived.Increment() + return err + } + if uo, ok := opt.(*header.IPv6UnknownExtHdrOption); ok { + uopt = uo + } + if done { + break + } + + switch opt := opt.(type) { + case *header.IPv6RouterAlertOption: + if *routerAlert != nil { + // As per RFC 2711 section 3, there should be at most one Router + // Alert option per packet. + // + // There MUST only be one option of this type, regardless of + // value, per Hop-by-Hop header. + stats.MalformedPacketsReceived.Increment() + return fmt.Errorf("found multiple Router Alert options (%#v, %#v)", opt, *routerAlert) + } + *routerAlert = opt + stats.OptionRouterAlertReceived.Increment() + default: + switch opt.UnknownAction() { + case header.IPv6OptionUnknownActionSkip: + case header.IPv6OptionUnknownActionDiscard: + return fmt.Errorf("found unknown Hop-by-Hop header option = %#v with discard action", opt) + case header.IPv6OptionUnknownActionDiscardSendICMPNoMulticastDest: + if header.IsV6MulticastAddress(dstAddr) { + return fmt.Errorf("found unknown hop-by-hop header option = %#v with discard action", opt) + } + fallthrough + case header.IPv6OptionUnknownActionDiscardSendICMP: + // This case satisfies a requirement of RFC 8200 section 4.2 which + // states that an unknown option starting with bits [10] should: + // + // discard the packet and, regardless of whether or not the + // packet's Destination Address was a multicast address, send an + // ICMP Parameter Problem, Code 2, message to the packet's + // Source Address, pointing to the unrecognized Option Type. + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6UnknownOption, + pointer: it.ParseOffset() + optsIt.OptionOffset(), + respondToMulticast: true, + }, pkt, !forwarding /* deliveredLocally */) + return fmt.Errorf("found unknown hop-by-hop header option = %#v with discard action", opt) + default: + panic(fmt.Sprintf("unrecognized action for an unrecognized Hop By Hop extension header option = %#v", opt)) + } + } + if uopt != nil { + uopt.Data.Release() + uopt = nil + } + } + return nil +} + +func (e *endpoint) processFragmentExtHdr(extHdr *header.IPv6FragmentExtHdr, it *header.IPv6PayloadIterator, pkt **stack.PacketBuffer, h header.IPv6) error { + stats := e.stats.ip + fragmentFieldOffset := it.ParseOffset() + + // Don't consume the iterator if we have the first fragment because we + // will use it to validate that the first fragment holds the upper layer + // header. + rawPayload := it.AsRawHeader(extHdr.FragmentOffset() != 0 /* consume */) + defer rawPayload.Release() + + if extHdr.FragmentOffset() == 0 { + // Check that the iterator ends with a raw payload as the first fragment + // should include all headers up to and including any upper layer + // headers, as per RFC 8200 section 4.5; only upper layer data + // (non-headers) should follow the fragment extension header. + var lastHdr header.IPv6PayloadHeader + + for { + it, done, err := it.Next() + if err != nil { + stats.MalformedPacketsReceived.Increment() + stats.MalformedFragmentsReceived.Increment() + return err + } + if done { + break + } + it.Release() + + lastHdr = it + } + + // If the last header is a raw header, then the last portion of the IPv6 + // payload is not a known IPv6 extension header. Note, this does not + // mean that the last portion is an upper layer header or not an + // extension header because: + // 1) we do not yet support all extension headers + // 2) we do not validate the upper layer header before reassembling. + // + // This check makes sure that a known IPv6 extension header is not + // present after the Fragment extension header in a non-initial + // fragment. + // + // TODO(#2196): Support IPv6 Authentication and Encapsulated + // Security Payload extension headers. + // TODO(#2333): Validate that the upper layer header is valid. + switch lastHdr.(type) { + case header.IPv6RawPayloadHeader: + default: + stats.MalformedPacketsReceived.Increment() + stats.MalformedFragmentsReceived.Increment() + return fmt.Errorf("known extension header = %#v present after fragment header in a non-initial fragment", lastHdr) + } + } + + fragmentPayloadLen := rawPayload.Buf.Size() + if fragmentPayloadLen == 0 { + // Drop the packet as it's marked as a fragment but has no payload. + stats.MalformedPacketsReceived.Increment() + stats.MalformedFragmentsReceived.Increment() + return fmt.Errorf("fragment has no payload") + } + + // As per RFC 2460 Section 4.5: + // + // If the length of a fragment, as derived from the fragment packet's + // Payload Length field, is not a multiple of 8 octets and the M flag + // of that fragment is 1, then that fragment must be discarded and an + // ICMP Parameter Problem, Code 0, message should be sent to the source + // of the fragment, pointing to the Payload Length field of the + // fragment packet. + if extHdr.More() && fragmentPayloadLen%header.IPv6FragmentExtHdrFragmentOffsetBytesPerUnit != 0 { + stats.MalformedPacketsReceived.Increment() + stats.MalformedFragmentsReceived.Increment() + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6ErroneousHeader, + pointer: header.IPv6PayloadLenOffset, + }, *pkt, true /* deliveredLocally */) + return fmt.Errorf("found fragment length = %d that is not a multiple of 8 octets", fragmentPayloadLen) + } + + // The packet is a fragment, let's try to reassemble it. + start := extHdr.FragmentOffset() * header.IPv6FragmentExtHdrFragmentOffsetBytesPerUnit + + // As per RFC 2460 Section 4.5: + // + // If the length and offset of a fragment are such that the Payload + // Length of the packet reassembled from that fragment would exceed + // 65,535 octets, then that fragment must be discarded and an ICMP + // Parameter Problem, Code 0, message should be sent to the source of + // the fragment, pointing to the Fragment Offset field of the fragment + // packet. + lengthAfterReassembly := int(start) + int(fragmentPayloadLen) + if lengthAfterReassembly > header.IPv6MaximumPayloadSize { + stats.MalformedPacketsReceived.Increment() + stats.MalformedFragmentsReceived.Increment() + _ = e.protocol.returnError(&icmpReasonParameterProblem{ + code: header.ICMPv6ErroneousHeader, + pointer: fragmentFieldOffset, + }, *pkt, true /* deliveredLocally */) + return fmt.Errorf("determined that reassembled packet length = %d would exceed allowed length = %d", lengthAfterReassembly, header.IPv6MaximumPayloadSize) + } + + // Note that pkt doesn't have its transport header set after reassembly, + // and won't until DeliverNetworkPacket sets it. + resPkt, proto, ready, err := e.protocol.fragmentation.Process( + // IPv6 ignores the Protocol field since the ID only needs to be unique + // across source-destination pairs, as per RFC 8200 section 4.5. + fragmentation.FragmentID{ + Source: h.SourceAddress(), + Destination: h.DestinationAddress(), + ID: extHdr.ID(), + }, + start, + start+uint16(fragmentPayloadLen)-1, + extHdr.More(), + uint8(rawPayload.Identifier), + *pkt, + ) + if err != nil { + stats.MalformedPacketsReceived.Increment() + stats.MalformedFragmentsReceived.Increment() + return err + } + + if ready { + // We create a new iterator with the reassembled packet because we could + // have more extension headers in the reassembled payload, as per RFC + // 8200 section 4.5. We also use the NextHeader value from the first + // fragment. + it.Release() + *it = header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(proto), resPkt.Data().ToBuffer()) + (*pkt).DecRef() + *pkt = resPkt + } + return nil +} + +// Close cleans up resources associated with the endpoint. +func (e *endpoint) Close() { + e.mu.Lock() + e.disableLocked() + e.mu.addressableEndpointState.Cleanup() + e.mu.Unlock() + + e.protocol.forgetEndpoint(e.nic.ID()) +} + +// NetworkProtocolNumber implements stack.NetworkEndpoint. +func (e *endpoint) NetworkProtocolNumber() tcpip.NetworkProtocolNumber { + return e.protocol.Number() +} + +// AddAndAcquirePermanentAddress implements stack.AddressableEndpoint. +func (e *endpoint) AddAndAcquirePermanentAddress(addr tcpip.AddressWithPrefix, properties stack.AddressProperties) (stack.AddressEndpoint, tcpip.Error) { + // TODO(b/169350103): add checks here after making sure we no longer receive + // an empty address. + e.mu.Lock() + defer e.mu.Unlock() + + // The dance of registering the dispatcher after adding the address makes it + // so that the tentative state is skipped if DAD is disabled. + addrDisp := properties.Disp + properties.Disp = nil + addressEndpoint, err := e.addAndAcquirePermanentAddressLocked(addr, properties) + if addrDisp != nil && err == nil { + addressEndpoint.RegisterDispatcher(addrDisp) + } + return addressEndpoint, err +} + +// addAndAcquirePermanentAddressLocked is like AddAndAcquirePermanentAddress but +// with locking requirements. +// +// addAndAcquirePermanentAddressLocked also joins the passed address's +// solicited-node multicast group and start duplicate address detection. +// +// Precondition: e.mu must be write locked. +func (e *endpoint) addAndAcquirePermanentAddressLocked(addr tcpip.AddressWithPrefix, properties stack.AddressProperties) (stack.AddressEndpoint, tcpip.Error) { + addressEndpoint, err := e.mu.addressableEndpointState.AddAndAcquireAddress(addr, properties, stack.PermanentTentative) + if err != nil { + return nil, err + } + + if !header.IsV6UnicastAddress(addr.Address) { + return addressEndpoint, nil + } + + if e.Enabled() { + if err := e.mu.ndp.startDuplicateAddressDetection(addr.Address, addressEndpoint); err != nil { + return nil, err + } + } + + snmc := header.SolicitedNodeAddr(addr.Address) + if err := e.joinGroupLocked(snmc); err != nil { + // joinGroupLocked only returns an error if the group address is not a valid + // IPv6 multicast address. + panic(fmt.Sprintf("e.joinGroupLocked(%s): %s", snmc, err)) + } + + return addressEndpoint, nil +} + +// RemovePermanentAddress implements stack.AddressableEndpoint. +func (e *endpoint) RemovePermanentAddress(addr tcpip.Address) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + addressEndpoint := e.getAddressRLocked(addr) + if addressEndpoint == nil || !addressEndpoint.GetKind().IsPermanent() { + return &tcpip.ErrBadLocalAddress{} + } + + return e.removePermanentEndpointLocked(addressEndpoint, true /* allowSLAACInvalidation */, stack.AddressRemovalManualAction, &stack.DADAborted{}) +} + +// removePermanentEndpointLocked is like removePermanentAddressLocked except +// it works with a stack.AddressEndpoint. +// +// Precondition: e.mu must be write locked. +func (e *endpoint) removePermanentEndpointLocked(addressEndpoint stack.AddressEndpoint, allowSLAACInvalidation bool, reason stack.AddressRemovalReason, dadResult stack.DADResult) tcpip.Error { + addr := addressEndpoint.AddressWithPrefix() + // If we are removing an address generated via SLAAC, cleanup + // its SLAAC resources and notify the integrator. + if addressEndpoint.ConfigType() == stack.AddressConfigSlaac { + if addressEndpoint.Temporary() { + e.mu.ndp.cleanupTempSLAACAddrResourcesAndNotify(addr) + } else { + e.mu.ndp.cleanupSLAACAddrResourcesAndNotify(addr, allowSLAACInvalidation) + } + } + + return e.removePermanentEndpointInnerLocked(addressEndpoint, reason, dadResult) +} + +// removePermanentEndpointInnerLocked is like removePermanentEndpointLocked +// except it does not cleanup SLAAC address state. +// +// Precondition: e.mu must be write locked. +func (e *endpoint) removePermanentEndpointInnerLocked(addressEndpoint stack.AddressEndpoint, reason stack.AddressRemovalReason, dadResult stack.DADResult) tcpip.Error { + addr := addressEndpoint.AddressWithPrefix() + e.mu.ndp.stopDuplicateAddressDetection(addr.Address, dadResult) + + if err := e.mu.addressableEndpointState.RemovePermanentEndpoint(addressEndpoint, reason); err != nil { + return err + } + + snmc := header.SolicitedNodeAddr(addr.Address) + err := e.leaveGroupLocked(snmc) + // The endpoint may have already left the multicast group. + if _, ok := err.(*tcpip.ErrBadLocalAddress); ok { + err = nil + } + return err +} + +// hasPermanentAddressLocked returns true if the endpoint has a permanent +// address equal to the passed address. +// +// Precondition: e.mu must be read or write locked. +func (e *endpoint) hasPermanentAddressRLocked(addr tcpip.Address) bool { + addressEndpoint := e.getAddressRLocked(addr) + if addressEndpoint == nil { + return false + } + return addressEndpoint.GetKind().IsPermanent() +} + +// getAddressRLocked returns the endpoint for the passed address. +// +// Precondition: e.mu must be read or write locked. +func (e *endpoint) getAddressRLocked(localAddr tcpip.Address) stack.AddressEndpoint { + return e.mu.addressableEndpointState.GetAddress(localAddr) +} + +// SetDeprecated implements stack.AddressableEndpoint. +func (e *endpoint) SetDeprecated(addr tcpip.Address, deprecated bool) tcpip.Error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.addressableEndpointState.SetDeprecated(addr, deprecated) +} + +// SetLifetimes implements stack.AddressableEndpoint. +func (e *endpoint) SetLifetimes(addr tcpip.Address, lifetimes stack.AddressLifetimes) tcpip.Error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.addressableEndpointState.SetLifetimes(addr, lifetimes) +} + +// MainAddress implements stack.AddressableEndpoint. +func (e *endpoint) MainAddress() tcpip.AddressWithPrefix { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.addressableEndpointState.MainAddress() +} + +// AcquireAssignedAddress implements stack.AddressableEndpoint. +func (e *endpoint) AcquireAssignedAddress(localAddr tcpip.Address, allowTemp bool, tempPEB stack.PrimaryEndpointBehavior, readOnly bool) stack.AddressEndpoint { + e.mu.RLock() + defer e.mu.RUnlock() + return e.acquireAddressOrCreateTempLocked(localAddr, allowTemp, tempPEB, readOnly) +} + +// acquireAddressOrCreateTempLocked is like AcquireAssignedAddress but with +// locking requirements. +// +// Precondition: e.mu must be write locked. +func (e *endpoint) acquireAddressOrCreateTempLocked(localAddr tcpip.Address, allowTemp bool, tempPEB stack.PrimaryEndpointBehavior, readOnly bool) stack.AddressEndpoint { + return e.mu.addressableEndpointState.AcquireAssignedAddress(localAddr, allowTemp, tempPEB, readOnly) +} + +// AcquireOutgoingPrimaryAddress implements stack.AddressableEndpoint. +func (e *endpoint) AcquireOutgoingPrimaryAddress(remoteAddr, srcHint tcpip.Address, allowExpired bool) stack.AddressEndpoint { + e.mu.RLock() + defer e.mu.RUnlock() + return e.acquireOutgoingPrimaryAddressRLocked(remoteAddr, srcHint, allowExpired) +} + +// getLinkLocalAddressRLocked returns a link-local address from the primary list +// of addresses, if one is available. +// +// See stack.PrimaryEndpointBehavior for more details about the primary list. +// +// Precondition: e.mu must be read locked. +func (e *endpoint) getLinkLocalAddressRLocked() tcpip.Address { + var linkLocalAddr tcpip.Address + e.mu.addressableEndpointState.ForEachPrimaryEndpoint(func(addressEndpoint stack.AddressEndpoint) bool { + if addressEndpoint.IsAssigned(false /* allowExpired */) { + if addr := addressEndpoint.AddressWithPrefix().Address; header.IsV6LinkLocalUnicastAddress(addr) { + linkLocalAddr = addr + return false + } + } + return true + }) + return linkLocalAddr +} + +// acquireOutgoingPrimaryAddressRLocked is like AcquireOutgoingPrimaryAddress +// but with locking requirements. +// +// Precondition: e.mu must be read locked. +func (e *endpoint) acquireOutgoingPrimaryAddressRLocked(remoteAddr, srcHint tcpip.Address, allowExpired bool) stack.AddressEndpoint { + // TODO(b/309216156): Support IPv6 hints. + + // addrCandidate is a candidate for Source Address Selection, as per + // RFC 6724 section 5. + type addrCandidate struct { + addressEndpoint stack.AddressEndpoint + addr tcpip.Address + scope header.IPv6AddressScope + + label uint8 + matchingPrefix uint8 + } + + if remoteAddr.BitLen() == 0 { + return e.mu.addressableEndpointState.AcquireOutgoingPrimaryAddress(remoteAddr, srcHint, allowExpired) + } + + // Create a candidate set of available addresses we can potentially use as a + // source address. + var cs []addrCandidate + e.mu.addressableEndpointState.ForEachPrimaryEndpoint(func(addressEndpoint stack.AddressEndpoint) bool { + // If r is not valid for outgoing connections, it is not a valid endpoint. + if !addressEndpoint.IsAssigned(allowExpired) { + return true + } + + addr := addressEndpoint.AddressWithPrefix().Address + scope, err := header.ScopeForIPv6Address(addr) + if err != nil { + // Should never happen as we got r from the primary IPv6 endpoint list and + // ScopeForIPv6Address only returns an error if addr is not an IPv6 + // address. + panic(fmt.Sprintf("header.ScopeForIPv6Address(%s): %s", addr, err)) + } + + cs = append(cs, addrCandidate{ + addressEndpoint: addressEndpoint, + addr: addr, + scope: scope, + label: getLabel(addr), + matchingPrefix: remoteAddr.MatchingPrefix(addr), + }) + + return true + }) + + remoteScope, err := header.ScopeForIPv6Address(remoteAddr) + if err != nil { + // primaryIPv6Endpoint should never be called with an invalid IPv6 address. + panic(fmt.Sprintf("header.ScopeForIPv6Address(%s): %s", remoteAddr, err)) + } + + remoteLabel := getLabel(remoteAddr) + + // Sort the addresses as per RFC 6724 section 5 rules 1-3. + // + // TODO(b/146021396): Implement rules 4, 5 of RFC 6724 section 5. + sort.Slice(cs, func(i, j int) bool { + sa := cs[i] + sb := cs[j] + + // Prefer same address as per RFC 6724 section 5 rule 1. + if sa.addr == remoteAddr { + return true + } + if sb.addr == remoteAddr { + return false + } + + // Prefer appropriate scope as per RFC 6724 section 5 rule 2. + if sa.scope < sb.scope { + return sa.scope >= remoteScope + } else if sb.scope < sa.scope { + return sb.scope < remoteScope + } + + // Avoid deprecated addresses as per RFC 6724 section 5 rule 3. + if saDep, sbDep := sa.addressEndpoint.Deprecated(), sb.addressEndpoint.Deprecated(); saDep != sbDep { + // If sa is not deprecated, it is preferred over sb. + return sbDep + } + + // Prefer matching label as per RFC 6724 section 5 rule 6. + if sa, sb := sa.label == remoteLabel, sb.label == remoteLabel; sa != sb { + if sa { + return true + } + if sb { + return false + } + } + + // Prefer temporary addresses as per RFC 6724 section 5 rule 7. + if saTemp, sbTemp := sa.addressEndpoint.Temporary(), sb.addressEndpoint.Temporary(); saTemp != sbTemp { + return saTemp + } + + // Use longest matching prefix as per RFC 6724 section 5 rule 8. + if sa.matchingPrefix > sb.matchingPrefix { + return true + } + if sb.matchingPrefix > sa.matchingPrefix { + return false + } + + // sa and sb are equal, return the endpoint that is closest to the front of + // the primary endpoint list. + return i < j + }) + + // Return the most preferred address that can have its reference count + // incremented. + for _, c := range cs { + if c.addressEndpoint.TryIncRef() { + return c.addressEndpoint + } + } + + return nil +} + +// PrimaryAddresses implements stack.AddressableEndpoint. +func (e *endpoint) PrimaryAddresses() []tcpip.AddressWithPrefix { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.addressableEndpointState.PrimaryAddresses() +} + +// PermanentAddresses implements stack.AddressableEndpoint. +func (e *endpoint) PermanentAddresses() []tcpip.AddressWithPrefix { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.addressableEndpointState.PermanentAddresses() +} + +// JoinGroup implements stack.GroupAddressableEndpoint. +func (e *endpoint) JoinGroup(addr tcpip.Address) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + return e.joinGroupLocked(addr) +} + +// joinGroupLocked is like JoinGroup but with locking requirements. +// +// Precondition: e.mu must be locked. +func (e *endpoint) joinGroupLocked(addr tcpip.Address) tcpip.Error { + if !header.IsV6MulticastAddress(addr) { + return &tcpip.ErrBadAddress{} + } + + e.mu.mld.joinGroup(addr) + return nil +} + +// LeaveGroup implements stack.GroupAddressableEndpoint. +func (e *endpoint) LeaveGroup(addr tcpip.Address) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + return e.leaveGroupLocked(addr) +} + +// leaveGroupLocked is like LeaveGroup but with locking requirements. +// +// Precondition: e.mu must be locked. +func (e *endpoint) leaveGroupLocked(addr tcpip.Address) tcpip.Error { + return e.mu.mld.leaveGroup(addr) +} + +// IsInGroup implements stack.GroupAddressableEndpoint. +func (e *endpoint) IsInGroup(addr tcpip.Address) bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.mu.mld.isInGroup(addr) +} + +// Stats implements stack.NetworkEndpoint. +func (e *endpoint) Stats() stack.NetworkEndpointStats { + return &e.stats.localStats +} + +var ( + _ stack.NetworkProtocol = (*protocol)(nil) + _ stack.MulticastForwardingNetworkProtocol = (*protocol)(nil) + _ stack.RejectIPv6WithHandler = (*protocol)(nil) + _ fragmentation.TimeoutHandler = (*protocol)(nil) +) + +// +stateify savable +type protocolMu struct { + sync.RWMutex `state:"nosave"` + + // eps is keyed by NICID to allow protocol methods to retrieve an endpoint + // when handling a packet, by looking at which NIC handled the packet. + eps map[tcpip.NICID]*endpoint + + // ICMP types for which the stack's global rate limiting must apply. + icmpRateLimitedTypes map[header.ICMPv6Type]struct{} + + // multicastForwardingDisp is the multicast forwarding event dispatcher that + // an integrator can provide to receive multicast forwarding events. Note + // that multicast packets will only be forwarded if this is non-nil. + multicastForwardingDisp stack.MulticastForwardingEventDispatcher +} + +// +stateify savable +type protocol struct { + stack *stack.Stack + options Options + + mu protocolMu + + // defaultTTL is the current default TTL for the protocol. Only the + // uint8 portion of it is meaningful. + defaultTTL atomicbitops.Uint32 + + fragmentation *fragmentation.Fragmentation + icmpRateLimiter *stack.ICMPRateLimiter + + multicastRouteTable multicast.RouteTable +} + +// Number returns the ipv6 protocol number. +func (p *protocol) Number() tcpip.NetworkProtocolNumber { + return ProtocolNumber +} + +// MinimumPacketSize returns the minimum valid ipv6 packet size. +func (p *protocol) MinimumPacketSize() int { + return header.IPv6MinimumSize +} + +// ParseAddresses implements stack.NetworkProtocol. +func (*protocol) ParseAddresses(b []byte) (src, dst tcpip.Address) { + h := header.IPv6(b) + return h.SourceAddress(), h.DestinationAddress() +} + +// NewEndpoint creates a new ipv6 endpoint. +func (p *protocol) NewEndpoint(nic stack.NetworkInterface, dispatcher stack.TransportDispatcher) stack.NetworkEndpoint { + e := &endpoint{ + nic: nic, + dispatcher: dispatcher, + protocol: p, + } + + // NDP options must be 8 octet aligned and the first 2 bytes are used for + // the type and length fields leaving 6 octets as the minimum size for a + // nonce option without padding. + const nonceSize = 6 + + // As per RFC 7527 section 4.1, + // + // If any probe is looped back within RetransTimer milliseconds after + // having sent DupAddrDetectTransmits NS(DAD) messages, the interface + // continues with another MAX_MULTICAST_SOLICIT number of NS(DAD) + // messages transmitted RetransTimer milliseconds apart. + // + // Value taken from RFC 4861 section 10. + const maxMulticastSolicit = 3 + dadOptions := ip.DADOptions{ + Clock: p.stack.Clock(), + SecureRNG: p.stack.SecureRNG().Reader, + NonceSize: nonceSize, + ExtendDADTransmits: maxMulticastSolicit, + Protocol: &e.mu.ndp, + NICID: nic.ID(), + } + + e.mu.Lock() + e.mu.addressableEndpointState.Init(e, stack.AddressableEndpointStateOptions{HiddenWhileDisabled: true}) + e.mu.ndp.init(e, dadOptions) + e.mu.mld.init(e) + e.dad.mu.Lock() + e.dad.mu.dad.Init(&e.dad.mu, p.options.DADConfigs, dadOptions) + e.dad.mu.Unlock() + e.mu.Unlock() + + stackStats := p.stack.Stats() + tcpip.InitStatCounters(reflect.ValueOf(&e.stats.localStats).Elem()) + e.stats.ip.Init(&e.stats.localStats.IP, &stackStats.IP) + e.stats.icmp.init(&e.stats.localStats.ICMP, &stackStats.ICMP.V6) + + p.mu.Lock() + defer p.mu.Unlock() + p.mu.eps[nic.ID()] = e + return e +} + +func (p *protocol) findEndpointWithAddress(addr tcpip.Address) *endpoint { + p.mu.RLock() + defer p.mu.RUnlock() + + for _, e := range p.mu.eps { + if addressEndpoint := e.AcquireAssignedAddress(addr, false /* allowTemp */, stack.NeverPrimaryEndpoint, true /* readOnly */); addressEndpoint != nil { + return e + } + } + + return nil +} + +func (p *protocol) getEndpointForNIC(id tcpip.NICID) (*endpoint, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + ep, ok := p.mu.eps[id] + return ep, ok +} + +func (p *protocol) forgetEndpoint(nicID tcpip.NICID) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.mu.eps, nicID) +} + +// SetOption implements stack.NetworkProtocol. +func (p *protocol) SetOption(option tcpip.SettableNetworkProtocolOption) tcpip.Error { + switch v := option.(type) { + case *tcpip.DefaultTTLOption: + p.SetDefaultTTL(uint8(*v)) + return nil + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// Option implements stack.NetworkProtocol. +func (p *protocol) Option(option tcpip.GettableNetworkProtocolOption) tcpip.Error { + switch v := option.(type) { + case *tcpip.DefaultTTLOption: + *v = tcpip.DefaultTTLOption(p.DefaultTTL()) + return nil + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// SetDefaultTTL sets the default TTL for endpoints created with this protocol. +func (p *protocol) SetDefaultTTL(ttl uint8) { + p.defaultTTL.Store(uint32(ttl)) +} + +// DefaultTTL returns the default TTL for endpoints created with this protocol. +func (p *protocol) DefaultTTL() uint8 { + return uint8(p.defaultTTL.Load()) +} + +// emitMulticastEvent emits a multicast forwarding event using the provided +// generator if a valid event dispatcher exists. +func (e *endpoint) emitMulticastEvent(eventGenerator func(stack.MulticastForwardingEventDispatcher)) { + e.protocol.mu.RLock() + defer e.protocol.mu.RUnlock() + if mcastDisp := e.protocol.mu.multicastForwardingDisp; mcastDisp != nil { + eventGenerator(mcastDisp) + } +} + +// Close implements stack.TransportProtocol. +func (p *protocol) Close() { + p.fragmentation.Release() + p.multicastRouteTable.Close() +} + +func validateUnicastSourceAndMulticastDestination(addresses stack.UnicastSourceAndMulticastDestination) tcpip.Error { + if !header.IsV6UnicastAddress(addresses.Source) || header.IsV6LinkLocalUnicastAddress(addresses.Source) { + return &tcpip.ErrBadAddress{} + } + + if !header.IsV6MulticastAddress(addresses.Destination) || header.IsV6LinkLocalMulticastAddress(addresses.Destination) { + return &tcpip.ErrBadAddress{} + } + + return nil +} + +func (p *protocol) multicastForwarding() bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.mu.multicastForwardingDisp != nil +} + +func (p *protocol) newInstalledRoute(route stack.MulticastRoute) (*multicast.InstalledRoute, tcpip.Error) { + if len(route.OutgoingInterfaces) == 0 { + return nil, &tcpip.ErrMissingRequiredFields{} + } + + if !p.stack.HasNIC(route.ExpectedInputInterface) { + return nil, &tcpip.ErrUnknownNICID{} + } + + for _, outgoingInterface := range route.OutgoingInterfaces { + if route.ExpectedInputInterface == outgoingInterface.ID { + return nil, &tcpip.ErrMulticastInputCannotBeOutput{} + } + + if !p.stack.HasNIC(outgoingInterface.ID) { + return nil, &tcpip.ErrUnknownNICID{} + } + } + return p.multicastRouteTable.NewInstalledRoute(route), nil +} + +// AddMulticastRoute implements stack.MulticastForwardingNetworkProtocol. +func (p *protocol) AddMulticastRoute(addresses stack.UnicastSourceAndMulticastDestination, route stack.MulticastRoute) tcpip.Error { + if !p.multicastForwarding() { + return &tcpip.ErrNotPermitted{} + } + + if err := validateUnicastSourceAndMulticastDestination(addresses); err != nil { + return err + } + + installedRoute, err := p.newInstalledRoute(route) + if err != nil { + return err + } + + pendingPackets := p.multicastRouteTable.AddInstalledRoute(addresses, installedRoute) + + for _, pkt := range pendingPackets { + p.forwardPendingMulticastPacket(pkt, installedRoute) + } + return nil +} + +// RemoveMulticastRoute implements +// stack.MulticastForwardingNetworkProtocol.RemoveMulticastRoute. +func (p *protocol) RemoveMulticastRoute(addresses stack.UnicastSourceAndMulticastDestination) tcpip.Error { + if err := validateUnicastSourceAndMulticastDestination(addresses); err != nil { + return err + } + + if removed := p.multicastRouteTable.RemoveInstalledRoute(addresses); !removed { + return &tcpip.ErrHostUnreachable{} + } + + return nil +} + +// MulticastRouteLastUsedTime implements +// stack.MulticastForwardingNetworkProtocol. +func (p *protocol) MulticastRouteLastUsedTime(addresses stack.UnicastSourceAndMulticastDestination) (tcpip.MonotonicTime, tcpip.Error) { + if err := validateUnicastSourceAndMulticastDestination(addresses); err != nil { + return tcpip.MonotonicTime{}, err + } + + timestamp, found := p.multicastRouteTable.GetLastUsedTimestamp(addresses) + + if !found { + return tcpip.MonotonicTime{}, &tcpip.ErrHostUnreachable{} + } + + return timestamp, nil +} + +// EnableMulticastForwarding implements +// stack.MulticastForwardingNetworkProtocol.EnableMulticastForwarding. +func (p *protocol) EnableMulticastForwarding(disp stack.MulticastForwardingEventDispatcher) (bool, tcpip.Error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.mu.multicastForwardingDisp != nil { + return true, nil + } + + if disp == nil { + return false, &tcpip.ErrInvalidOptionValue{} + } + + p.mu.multicastForwardingDisp = disp + return false, nil +} + +// DisableMulticastForwarding implements +// stack.MulticastForwardingNetworkProtocol.DisableMulticastForwarding. +func (p *protocol) DisableMulticastForwarding() { + p.mu.Lock() + defer p.mu.Unlock() + p.mu.multicastForwardingDisp = nil + p.multicastRouteTable.RemoveAllInstalledRoutes() +} + +func (p *protocol) forwardPendingMulticastPacket(pkt *stack.PacketBuffer, installedRoute *multicast.InstalledRoute) { + defer pkt.DecRef() + + // Attempt to forward the packet using the endpoint that it originally + // arrived on. This ensures that the packet is only forwarded if it + // matches the route's expected input interface (see 5a of RFC 1812 section + // 5.2.1.3). + ep, ok := p.getEndpointForNIC(pkt.NICID) + + if !ok { + // The endpoint that the packet arrived on no longer exists. Silently + // drop the pkt. + return + } + + if !ep.MulticastForwarding() { + return + } + + ep.handleForwardingError(ep.forwardValidatedMulticastPacket(pkt, installedRoute)) +} + +// Wait implements stack.TransportProtocol. +func (*protocol) Wait() {} + +// parseAndValidate parses the packet (including its transport layer header) and +// returns a view containing the parsed IP header. The caller is responsible +// for releasing the returned View. +// +// Returns true if the IP header was successfully parsed. +func (p *protocol) parseAndValidate(pkt *stack.PacketBuffer) (*buffer.View, bool) { + transProtoNum, hasTransportHdr, ok := p.Parse(pkt) + if !ok { + return nil, false + } + + h := header.IPv6(pkt.NetworkHeader().Slice()) + // Do not include the link header's size when calculating the size of the IP + // packet. + if !h.IsValid(pkt.Size() - len(pkt.LinkHeader().Slice())) { + return nil, false + } + + if hasTransportHdr { + p.parseTransport(pkt, transProtoNum) + } + + return pkt.NetworkHeader().View(), true +} + +func (p *protocol) parseTransport(pkt *stack.PacketBuffer, transProtoNum tcpip.TransportProtocolNumber) { + if transProtoNum == header.ICMPv6ProtocolNumber { + // The transport layer will handle transport layer parsing errors. + _ = parse.ICMPv6(pkt) + return + } + + switch err := p.stack.ParsePacketBufferTransport(transProtoNum, pkt); err { + case stack.ParsedOK: + case stack.UnknownTransportProtocol, stack.TransportLayerParseError: + // The transport layer will handle unknown protocols and transport layer + // parsing errors. + default: + panic(fmt.Sprintf("unexpected error parsing transport header = %d", err)) + } +} + +// Parse implements stack.NetworkProtocol. +func (*protocol) Parse(pkt *stack.PacketBuffer) (proto tcpip.TransportProtocolNumber, hasTransportHdr bool, ok bool) { + proto, _, fragOffset, fragMore, ok := parse.IPv6(pkt) + if !ok { + return 0, false, false + } + + return proto, !fragMore && fragOffset == 0, true +} + +// allowICMPReply reports whether an ICMP reply with provided type may +// be sent following the rate mask options and global ICMP rate limiter. +func (p *protocol) allowICMPReply(icmpType header.ICMPv6Type) bool { + p.mu.RLock() + defer p.mu.RUnlock() + + if _, ok := p.mu.icmpRateLimitedTypes[icmpType]; ok { + return p.stack.AllowICMPMessage() + } + return true +} + +// SendRejectionError implements stack.RejectIPv6WithHandler. +func (p *protocol) SendRejectionError(pkt *stack.PacketBuffer, rejectWith stack.RejectIPv6WithICMPType, inputHook bool) tcpip.Error { + switch rejectWith { + case stack.RejectIPv6WithICMPNoRoute: + return p.returnError(&icmpReasonNetUnreachable{}, pkt, inputHook) + case stack.RejectIPv6WithICMPAddrUnreachable: + return p.returnError(&icmpReasonHostUnreachable{}, pkt, inputHook) + case stack.RejectIPv6WithICMPPortUnreachable: + return p.returnError(&icmpReasonPortUnreachable{}, pkt, inputHook) + case stack.RejectIPv6WithICMPAdminProhibited: + return p.returnError(&icmpReasonAdministrativelyProhibited{}, pkt, inputHook) + default: + panic(fmt.Sprintf("unhandled %[1]T = %[1]d", rejectWith)) + } +} + +// calculateNetworkMTU calculates the network-layer payload MTU based on the +// link-layer payload MTU and the length of every IPv6 header. +// Note that this is different than the Payload Length field of the IPv6 header, +// which includes the length of the extension headers. +func calculateNetworkMTU(linkMTU, networkHeadersLen uint32) (uint32, tcpip.Error) { + if linkMTU < header.IPv6MinimumMTU { + return 0, &tcpip.ErrInvalidEndpointState{} + } + + // As per RFC 7112 section 5, we should discard packets if their IPv6 header + // is bigger than 1280 bytes (ie, the minimum link MTU) since we do not + // support PMTU discovery: + // Hosts that do not discover the Path MTU MUST limit the IPv6 Header Chain + // length to 1280 bytes. Limiting the IPv6 Header Chain length to 1280 + // bytes ensures that the header chain length does not exceed the IPv6 + // minimum MTU. + if networkHeadersLen > header.IPv6MinimumMTU { + return 0, &tcpip.ErrMalformedHeader{} + } + + networkMTU := linkMTU - networkHeadersLen + if networkMTU > maxPayloadSize { + networkMTU = maxPayloadSize + } + return networkMTU, nil +} + +// Options holds options to configure a new protocol. +// +// +stateify savable +type Options struct { + // NDPConfigs is the default NDP configurations used by interfaces. + NDPConfigs NDPConfigurations + + // AutoGenLinkLocal determines whether or not the stack attempts to + // auto-generate a link-local address for newly enabled non-loopback + // NICs. + // + // Note, setting this to true does not mean that a link-local address is + // assigned right away, or at all. If Duplicate Address Detection is enabled, + // an address is only assigned if it successfully resolves. If it fails, no + // further attempts are made to auto-generate a link-local address. + // + // The generated link-local address follows RFC 4291 Appendix A guidelines. + AutoGenLinkLocal bool + + // NDPDisp is the NDP event dispatcher that an integrator can provide to + // receive NDP related events. + NDPDisp NDPDispatcher + + // OpaqueIIDOpts hold the options for generating opaque interface + // identifiers (IIDs) as outlined by RFC 7217. + OpaqueIIDOpts OpaqueInterfaceIdentifierOptions + + // TempIIDSeed is used to seed the initial temporary interface identifier + // history value used to generate IIDs for temporary SLAAC addresses. + // + // Temporary SLAAC addresses are short-lived addresses which are unpredictable + // and random from the perspective of other nodes on the network. It is + // recommended that the seed be a random byte buffer of at least + // header.IIDSize bytes to make sure that temporary SLAAC addresses are + // sufficiently random. It should follow minimum randomness requirements for + // security as outlined by RFC 4086. + // + // Note: using a nil value, the same seed across netstack program runs, or a + // seed that is too small would reduce randomness and increase predictability, + // defeating the purpose of temporary SLAAC addresses. + TempIIDSeed []byte + + // MLD holds options for MLD. + MLD MLDOptions + + // DADConfigs holds the default DAD configurations used by IPv6 endpoints. + DADConfigs stack.DADConfigurations + + // AllowExternalLoopbackTraffic indicates that inbound loopback packets (i.e. + // martian loopback packets) should be accepted. + AllowExternalLoopbackTraffic bool +} + +// NewProtocolWithOptions returns an IPv6 network protocol. +func NewProtocolWithOptions(opts Options) stack.NetworkProtocolFactory { + opts.NDPConfigs.validate() + + return func(s *stack.Stack) stack.NetworkProtocol { + p := &protocol{ + stack: s, + options: opts, + } + p.fragmentation = fragmentation.NewFragmentation(header.IPv6FragmentExtHdrFragmentOffsetBytesPerUnit, fragmentation.HighFragThreshold, fragmentation.LowFragThreshold, ReassembleTimeout, s.Clock(), p) + p.mu.eps = make(map[tcpip.NICID]*endpoint) + p.SetDefaultTTL(DefaultTTL) + // Set default ICMP rate limiting to Linux defaults. + // + // Default: 0-1,3-127 (rate limit ICMPv6 errors except Packet Too Big) + // See https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt. + defaultIcmpTypes := make(map[header.ICMPv6Type]struct{}) + for i := header.ICMPv6Type(0); i < header.ICMPv6EchoRequest; i++ { + switch i { + case header.ICMPv6PacketTooBig: + // Do not rate limit packet too big by default. + default: + defaultIcmpTypes[i] = struct{}{} + } + } + p.mu.icmpRateLimitedTypes = defaultIcmpTypes + + if err := p.multicastRouteTable.Init(multicast.DefaultConfig(s.Clock())); err != nil { + panic(fmt.Sprintf("p.multicastRouteTable.Init(_): %s", err)) + } + + return p + } +} + +// NewProtocol is equivalent to NewProtocolWithOptions with an empty Options. +func NewProtocol(s *stack.Stack) stack.NetworkProtocol { + return NewProtocolWithOptions(Options{})(s) +} + +func calculateFragmentReserve(pkt *stack.PacketBuffer) int { + return pkt.AvailableHeaderBytes() + len(pkt.NetworkHeader().Slice()) + header.IPv6FragmentHeaderSize +} + +// getFragmentID returns a random uint32 number (other than zero) to be used as +// fragment ID in the IPv6 header. +func (e *endpoint) getFragmentID() uint32 { + rng := e.protocol.stack.SecureRNG() + id := rng.Uint32() + for id == 0 { + id = rng.Uint32() + } + return id +} + +func buildNextFragment(pf *fragmentation.PacketFragmenter, originalIPHeaders header.IPv6, transportProto tcpip.TransportProtocolNumber, id uint32) (*stack.PacketBuffer, bool) { + fragPkt, offset, copied, more := pf.BuildNextFragment() + fragPkt.NetworkProtocolNumber = ProtocolNumber + + originalIPHeadersLength := len(originalIPHeaders) + + s := header.IPv6ExtHdrSerializer{&header.IPv6SerializableFragmentExtHdr{ + FragmentOffset: uint16(offset / header.IPv6FragmentExtHdrFragmentOffsetBytesPerUnit), + M: more, + Identification: id, + }} + + fragmentIPHeadersLength := originalIPHeadersLength + s.Length() + fragmentIPHeaders := header.IPv6(fragPkt.NetworkHeader().Push(fragmentIPHeadersLength)) + + // Copy the IPv6 header and any extension headers already populated. + if copied := copy(fragmentIPHeaders, originalIPHeaders); copied != originalIPHeadersLength { + panic(fmt.Sprintf("wrong number of bytes copied into fragmentIPHeaders: got %d, want %d", copied, originalIPHeadersLength)) + } + + nextHeader, _ := s.Serialize(transportProto, fragmentIPHeaders[originalIPHeadersLength:]) + + fragmentIPHeaders.SetNextHeader(nextHeader) + fragmentIPHeaders.SetPayloadLength(uint16(copied + fragmentIPHeadersLength - header.IPv6MinimumSize)) + + return fragPkt, more +} + +func checkV4Mapped(h header.IPv6, stats ip.MultiCounterIPStats) bool { + // Disallow IPv4-mapped addresses per RFC 6890 section 2.2.3. + ret := true + if header.IsV4MappedAddress(h.SourceAddress()) { + stats.InvalidSourceAddressesReceived.Increment() + ret = false + } + if header.IsV4MappedAddress(h.DestinationAddress()) { + stats.InvalidDestinationAddressesReceived.Increment() + ret = false + } + return ret +} diff --git a/pkg/tcpip/network/ipv6/ipv6_export.go b/pkg/tcpip/network/ipv6/ipv6_export.go new file mode 100644 index 0000000..7a26711 --- /dev/null +++ b/pkg/tcpip/network/ipv6/ipv6_export.go @@ -0,0 +1,14 @@ +package ipv6 + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +type ExportedEndpoint interface { + WritePacketDirect(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error +} + +func (e *endpoint) WritePacketDirect(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error { + return e.writePacket(r, pkt, pkt.TransportProtocolNumber, true) +} diff --git a/pkg/tcpip/network/ipv6/ipv6_state_autogen.go b/pkg/tcpip/network/ipv6/ipv6_state_autogen.go new file mode 100644 index 0000000..2e97ea4 --- /dev/null +++ b/pkg/tcpip/network/ipv6/ipv6_state_autogen.go @@ -0,0 +1,1049 @@ +// automatically generated by stateify. + +package ipv6 + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (i *icmpv6DestinationUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv6.icmpv6DestinationUnreachableSockError" +} + +func (i *icmpv6DestinationUnreachableSockError) StateFields() []string { + return []string{} +} + +func (i *icmpv6DestinationUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv6DestinationUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() +} + +func (i *icmpv6DestinationUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv6DestinationUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (i *icmpv6DestinationNetworkUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv6.icmpv6DestinationNetworkUnreachableSockError" +} + +func (i *icmpv6DestinationNetworkUnreachableSockError) StateFields() []string { + return []string{ + "icmpv6DestinationUnreachableSockError", + } +} + +func (i *icmpv6DestinationNetworkUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv6DestinationNetworkUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv6DestinationUnreachableSockError) +} + +func (i *icmpv6DestinationNetworkUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv6DestinationNetworkUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv6DestinationUnreachableSockError) +} + +func (i *icmpv6DestinationPortUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv6.icmpv6DestinationPortUnreachableSockError" +} + +func (i *icmpv6DestinationPortUnreachableSockError) StateFields() []string { + return []string{ + "icmpv6DestinationUnreachableSockError", + } +} + +func (i *icmpv6DestinationPortUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv6DestinationPortUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv6DestinationUnreachableSockError) +} + +func (i *icmpv6DestinationPortUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv6DestinationPortUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv6DestinationUnreachableSockError) +} + +func (i *icmpv6DestinationAddressUnreachableSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv6.icmpv6DestinationAddressUnreachableSockError" +} + +func (i *icmpv6DestinationAddressUnreachableSockError) StateFields() []string { + return []string{ + "icmpv6DestinationUnreachableSockError", + } +} + +func (i *icmpv6DestinationAddressUnreachableSockError) beforeSave() {} + +// +checklocksignore +func (i *icmpv6DestinationAddressUnreachableSockError) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.icmpv6DestinationUnreachableSockError) +} + +func (i *icmpv6DestinationAddressUnreachableSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (i *icmpv6DestinationAddressUnreachableSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.icmpv6DestinationUnreachableSockError) +} + +func (e *icmpv6PacketTooBigSockError) StateTypeName() string { + return "pkg/tcpip/network/ipv6.icmpv6PacketTooBigSockError" +} + +func (e *icmpv6PacketTooBigSockError) StateFields() []string { + return []string{ + "mtu", + } +} + +func (e *icmpv6PacketTooBigSockError) beforeSave() {} + +// +checklocksignore +func (e *icmpv6PacketTooBigSockError) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.mtu) +} + +func (e *icmpv6PacketTooBigSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (e *icmpv6PacketTooBigSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.mtu) +} + +func (e *endpointMu) StateTypeName() string { + return "pkg/tcpip/network/ipv6.endpointMu" +} + +func (e *endpointMu) StateFields() []string { + return []string{ + "addressableEndpointState", + "ndp", + "mld", + } +} + +func (e *endpointMu) beforeSave() {} + +// +checklocksignore +func (e *endpointMu) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.addressableEndpointState) + stateSinkObject.Save(1, &e.ndp) + stateSinkObject.Save(2, &e.mld) +} + +func (e *endpointMu) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpointMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.addressableEndpointState) + stateSourceObject.Load(1, &e.ndp) + stateSourceObject.Load(2, &e.mld) +} + +func (d *dadMu) StateTypeName() string { + return "pkg/tcpip/network/ipv6.dadMu" +} + +func (d *dadMu) StateFields() []string { + return []string{ + "dad", + } +} + +func (d *dadMu) beforeSave() {} + +// +checklocksignore +func (d *dadMu) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.dad) +} + +func (d *dadMu) afterLoad(context.Context) {} + +// +checklocksignore +func (d *dadMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.dad) +} + +func (e *endpointDAD) StateTypeName() string { + return "pkg/tcpip/network/ipv6.endpointDAD" +} + +func (e *endpointDAD) StateFields() []string { + return []string{ + "mu", + } +} + +func (e *endpointDAD) beforeSave() {} + +// +checklocksignore +func (e *endpointDAD) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.mu) +} + +func (e *endpointDAD) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpointDAD) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.mu) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/network/ipv6.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "nic", + "dispatcher", + "protocol", + "stats", + "enabled", + "forwarding", + "multicastForwarding", + "mu", + "dad", + } +} + +func (e *endpoint) beforeSave() {} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.nic) + stateSinkObject.Save(1, &e.dispatcher) + stateSinkObject.Save(2, &e.protocol) + stateSinkObject.Save(3, &e.stats) + stateSinkObject.Save(4, &e.enabled) + stateSinkObject.Save(5, &e.forwarding) + stateSinkObject.Save(6, &e.multicastForwarding) + stateSinkObject.Save(7, &e.mu) + stateSinkObject.Save(8, &e.dad) +} + +func (e *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.nic) + stateSourceObject.Load(1, &e.dispatcher) + stateSourceObject.Load(2, &e.protocol) + stateSourceObject.Load(3, &e.stats) + stateSourceObject.Load(4, &e.enabled) + stateSourceObject.Load(5, &e.forwarding) + stateSourceObject.Load(6, &e.multicastForwarding) + stateSourceObject.Load(7, &e.mu) + stateSourceObject.Load(8, &e.dad) +} + +func (o *OpaqueInterfaceIdentifierOptions) StateTypeName() string { + return "pkg/tcpip/network/ipv6.OpaqueInterfaceIdentifierOptions" +} + +func (o *OpaqueInterfaceIdentifierOptions) StateFields() []string { + return []string{ + "SecretKey", + } +} + +func (o *OpaqueInterfaceIdentifierOptions) beforeSave() {} + +// +checklocksignore +func (o *OpaqueInterfaceIdentifierOptions) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.SecretKey) +} + +func (o *OpaqueInterfaceIdentifierOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (o *OpaqueInterfaceIdentifierOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.SecretKey) +} + +func (p *protocolMu) StateTypeName() string { + return "pkg/tcpip/network/ipv6.protocolMu" +} + +func (p *protocolMu) StateFields() []string { + return []string{ + "eps", + "icmpRateLimitedTypes", + "multicastForwardingDisp", + } +} + +func (p *protocolMu) beforeSave() {} + +// +checklocksignore +func (p *protocolMu) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.eps) + stateSinkObject.Save(1, &p.icmpRateLimitedTypes) + stateSinkObject.Save(2, &p.multicastForwardingDisp) +} + +func (p *protocolMu) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocolMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.eps) + stateSourceObject.Load(1, &p.icmpRateLimitedTypes) + stateSourceObject.Load(2, &p.multicastForwardingDisp) +} + +func (p *protocol) StateTypeName() string { + return "pkg/tcpip/network/ipv6.protocol" +} + +func (p *protocol) StateFields() []string { + return []string{ + "stack", + "options", + "mu", + "defaultTTL", + "fragmentation", + "icmpRateLimiter", + "multicastRouteTable", + } +} + +func (p *protocol) beforeSave() {} + +// +checklocksignore +func (p *protocol) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.stack) + stateSinkObject.Save(1, &p.options) + stateSinkObject.Save(2, &p.mu) + stateSinkObject.Save(3, &p.defaultTTL) + stateSinkObject.Save(4, &p.fragmentation) + stateSinkObject.Save(5, &p.icmpRateLimiter) + stateSinkObject.Save(6, &p.multicastRouteTable) +} + +func (p *protocol) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.stack) + stateSourceObject.Load(1, &p.options) + stateSourceObject.Load(2, &p.mu) + stateSourceObject.Load(3, &p.defaultTTL) + stateSourceObject.Load(4, &p.fragmentation) + stateSourceObject.Load(5, &p.icmpRateLimiter) + stateSourceObject.Load(6, &p.multicastRouteTable) +} + +func (o *Options) StateTypeName() string { + return "pkg/tcpip/network/ipv6.Options" +} + +func (o *Options) StateFields() []string { + return []string{ + "NDPConfigs", + "AutoGenLinkLocal", + "NDPDisp", + "OpaqueIIDOpts", + "TempIIDSeed", + "MLD", + "DADConfigs", + "AllowExternalLoopbackTraffic", + } +} + +func (o *Options) beforeSave() {} + +// +checklocksignore +func (o *Options) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.NDPConfigs) + stateSinkObject.Save(1, &o.AutoGenLinkLocal) + stateSinkObject.Save(2, &o.NDPDisp) + stateSinkObject.Save(3, &o.OpaqueIIDOpts) + stateSinkObject.Save(4, &o.TempIIDSeed) + stateSinkObject.Save(5, &o.MLD) + stateSinkObject.Save(6, &o.DADConfigs) + stateSinkObject.Save(7, &o.AllowExternalLoopbackTraffic) +} + +func (o *Options) afterLoad(context.Context) {} + +// +checklocksignore +func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.NDPConfigs) + stateSourceObject.Load(1, &o.AutoGenLinkLocal) + stateSourceObject.Load(2, &o.NDPDisp) + stateSourceObject.Load(3, &o.OpaqueIIDOpts) + stateSourceObject.Load(4, &o.TempIIDSeed) + stateSourceObject.Load(5, &o.MLD) + stateSourceObject.Load(6, &o.DADConfigs) + stateSourceObject.Load(7, &o.AllowExternalLoopbackTraffic) +} + +func (m *MLDOptions) StateTypeName() string { + return "pkg/tcpip/network/ipv6.MLDOptions" +} + +func (m *MLDOptions) StateFields() []string { + return []string{ + "Enabled", + } +} + +func (m *MLDOptions) beforeSave() {} + +// +checklocksignore +func (m *MLDOptions) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.Enabled) +} + +func (m *MLDOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (m *MLDOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.Enabled) +} + +func (mld *mldState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.mldState" +} + +func (mld *mldState) StateFields() []string { + return []string{ + "ep", + "genericMulticastProtocol", + } +} + +func (mld *mldState) beforeSave() {} + +// +checklocksignore +func (mld *mldState) StateSave(stateSinkObject state.Sink) { + mld.beforeSave() + stateSinkObject.Save(0, &mld.ep) + stateSinkObject.Save(1, &mld.genericMulticastProtocol) +} + +func (mld *mldState) afterLoad(context.Context) {} + +// +checklocksignore +func (mld *mldState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &mld.ep) + stateSourceObject.Load(1, &mld.genericMulticastProtocol) +} + +func (c *NDPConfigurations) StateTypeName() string { + return "pkg/tcpip/network/ipv6.NDPConfigurations" +} + +func (c *NDPConfigurations) StateFields() []string { + return []string{ + "MaxRtrSolicitations", + "RtrSolicitationInterval", + "MaxRtrSolicitationDelay", + "HandleRAs", + "DiscoverDefaultRouters", + "DiscoverMoreSpecificRoutes", + "DiscoverOnLinkPrefixes", + "AutoGenGlobalAddresses", + "AutoGenAddressConflictRetries", + "AutoGenTempGlobalAddresses", + "MaxTempAddrValidLifetime", + "MaxTempAddrPreferredLifetime", + "RegenAdvanceDuration", + } +} + +func (c *NDPConfigurations) beforeSave() {} + +// +checklocksignore +func (c *NDPConfigurations) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.MaxRtrSolicitations) + stateSinkObject.Save(1, &c.RtrSolicitationInterval) + stateSinkObject.Save(2, &c.MaxRtrSolicitationDelay) + stateSinkObject.Save(3, &c.HandleRAs) + stateSinkObject.Save(4, &c.DiscoverDefaultRouters) + stateSinkObject.Save(5, &c.DiscoverMoreSpecificRoutes) + stateSinkObject.Save(6, &c.DiscoverOnLinkPrefixes) + stateSinkObject.Save(7, &c.AutoGenGlobalAddresses) + stateSinkObject.Save(8, &c.AutoGenAddressConflictRetries) + stateSinkObject.Save(9, &c.AutoGenTempGlobalAddresses) + stateSinkObject.Save(10, &c.MaxTempAddrValidLifetime) + stateSinkObject.Save(11, &c.MaxTempAddrPreferredLifetime) + stateSinkObject.Save(12, &c.RegenAdvanceDuration) +} + +func (c *NDPConfigurations) afterLoad(context.Context) {} + +// +checklocksignore +func (c *NDPConfigurations) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.MaxRtrSolicitations) + stateSourceObject.Load(1, &c.RtrSolicitationInterval) + stateSourceObject.Load(2, &c.MaxRtrSolicitationDelay) + stateSourceObject.Load(3, &c.HandleRAs) + stateSourceObject.Load(4, &c.DiscoverDefaultRouters) + stateSourceObject.Load(5, &c.DiscoverMoreSpecificRoutes) + stateSourceObject.Load(6, &c.DiscoverOnLinkPrefixes) + stateSourceObject.Load(7, &c.AutoGenGlobalAddresses) + stateSourceObject.Load(8, &c.AutoGenAddressConflictRetries) + stateSourceObject.Load(9, &c.AutoGenTempGlobalAddresses) + stateSourceObject.Load(10, &c.MaxTempAddrValidLifetime) + stateSourceObject.Load(11, &c.MaxTempAddrPreferredLifetime) + stateSourceObject.Load(12, &c.RegenAdvanceDuration) +} + +func (t *timer) StateTypeName() string { + return "pkg/tcpip/network/ipv6.timer" +} + +func (t *timer) StateFields() []string { + return []string{ + "done", + "timer", + } +} + +func (t *timer) beforeSave() {} + +// +checklocksignore +func (t *timer) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.done) + stateSinkObject.Save(1, &t.timer) +} + +func (t *timer) afterLoad(context.Context) {} + +// +checklocksignore +func (t *timer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.done) + stateSourceObject.Load(1, &t.timer) +} + +func (o *offLinkRoute) StateTypeName() string { + return "pkg/tcpip/network/ipv6.offLinkRoute" +} + +func (o *offLinkRoute) StateFields() []string { + return []string{ + "dest", + "router", + } +} + +func (o *offLinkRoute) beforeSave() {} + +// +checklocksignore +func (o *offLinkRoute) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.dest) + stateSinkObject.Save(1, &o.router) +} + +func (o *offLinkRoute) afterLoad(context.Context) {} + +// +checklocksignore +func (o *offLinkRoute) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.dest) + stateSourceObject.Load(1, &o.router) +} + +func (ndp *ndpState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.ndpState" +} + +func (ndp *ndpState) StateFields() []string { + return []string{ + "ep", + "configs", + "dad", + "offLinkRoutes", + "rtrSolicitTimer", + "onLinkPrefixes", + "slaacPrefixes", + "dhcpv6Configuration", + "temporaryIIDHistory", + "temporaryAddressDesyncFactor", + } +} + +func (ndp *ndpState) beforeSave() {} + +// +checklocksignore +func (ndp *ndpState) StateSave(stateSinkObject state.Sink) { + ndp.beforeSave() + stateSinkObject.Save(0, &ndp.ep) + stateSinkObject.Save(1, &ndp.configs) + stateSinkObject.Save(2, &ndp.dad) + stateSinkObject.Save(3, &ndp.offLinkRoutes) + stateSinkObject.Save(4, &ndp.rtrSolicitTimer) + stateSinkObject.Save(5, &ndp.onLinkPrefixes) + stateSinkObject.Save(6, &ndp.slaacPrefixes) + stateSinkObject.Save(7, &ndp.dhcpv6Configuration) + stateSinkObject.Save(8, &ndp.temporaryIIDHistory) + stateSinkObject.Save(9, &ndp.temporaryAddressDesyncFactor) +} + +func (ndp *ndpState) afterLoad(context.Context) {} + +// +checklocksignore +func (ndp *ndpState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ndp.ep) + stateSourceObject.Load(1, &ndp.configs) + stateSourceObject.Load(2, &ndp.dad) + stateSourceObject.Load(3, &ndp.offLinkRoutes) + stateSourceObject.Load(4, &ndp.rtrSolicitTimer) + stateSourceObject.Load(5, &ndp.onLinkPrefixes) + stateSourceObject.Load(6, &ndp.slaacPrefixes) + stateSourceObject.Load(7, &ndp.dhcpv6Configuration) + stateSourceObject.Load(8, &ndp.temporaryIIDHistory) + stateSourceObject.Load(9, &ndp.temporaryAddressDesyncFactor) +} + +func (o *offLinkRouteState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.offLinkRouteState" +} + +func (o *offLinkRouteState) StateFields() []string { + return []string{ + "prf", + "invalidationJob", + } +} + +func (o *offLinkRouteState) beforeSave() {} + +// +checklocksignore +func (o *offLinkRouteState) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.prf) + stateSinkObject.Save(1, &o.invalidationJob) +} + +func (o *offLinkRouteState) afterLoad(context.Context) {} + +// +checklocksignore +func (o *offLinkRouteState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.prf) + stateSourceObject.Load(1, &o.invalidationJob) +} + +func (o *onLinkPrefixState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.onLinkPrefixState" +} + +func (o *onLinkPrefixState) StateFields() []string { + return []string{ + "invalidationJob", + } +} + +func (o *onLinkPrefixState) beforeSave() {} + +// +checklocksignore +func (o *onLinkPrefixState) StateSave(stateSinkObject state.Sink) { + o.beforeSave() + stateSinkObject.Save(0, &o.invalidationJob) +} + +func (o *onLinkPrefixState) afterLoad(context.Context) {} + +// +checklocksignore +func (o *onLinkPrefixState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &o.invalidationJob) +} + +func (t *tempSLAACAddrState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.tempSLAACAddrState" +} + +func (t *tempSLAACAddrState) StateFields() []string { + return []string{ + "deprecationJob", + "invalidationJob", + "regenJob", + "createdAt", + "addressEndpoint", + "regenerated", + } +} + +func (t *tempSLAACAddrState) beforeSave() {} + +// +checklocksignore +func (t *tempSLAACAddrState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.deprecationJob) + stateSinkObject.Save(1, &t.invalidationJob) + stateSinkObject.Save(2, &t.regenJob) + stateSinkObject.Save(3, &t.createdAt) + stateSinkObject.Save(4, &t.addressEndpoint) + stateSinkObject.Save(5, &t.regenerated) +} + +func (t *tempSLAACAddrState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *tempSLAACAddrState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.deprecationJob) + stateSourceObject.Load(1, &t.invalidationJob) + stateSourceObject.Load(2, &t.regenJob) + stateSourceObject.Load(3, &t.createdAt) + stateSourceObject.Load(4, &t.addressEndpoint) + stateSourceObject.Load(5, &t.regenerated) +} + +func (s *stableAddrState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.stableAddrState" +} + +func (s *stableAddrState) StateFields() []string { + return []string{ + "addressEndpoint", + "localGenerationFailures", + } +} + +func (s *stableAddrState) beforeSave() {} + +// +checklocksignore +func (s *stableAddrState) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.addressEndpoint) + stateSinkObject.Save(1, &s.localGenerationFailures) +} + +func (s *stableAddrState) afterLoad(context.Context) {} + +// +checklocksignore +func (s *stableAddrState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.addressEndpoint) + stateSourceObject.Load(1, &s.localGenerationFailures) +} + +func (s *slaacPrefixState) StateTypeName() string { + return "pkg/tcpip/network/ipv6.slaacPrefixState" +} + +func (s *slaacPrefixState) StateFields() []string { + return []string{ + "deprecationJob", + "invalidationJob", + "validUntil", + "preferredUntil", + "stableAddr", + "tempAddrs", + "generationAttempts", + "maxGenerationAttempts", + } +} + +func (s *slaacPrefixState) beforeSave() {} + +// +checklocksignore +func (s *slaacPrefixState) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.deprecationJob) + stateSinkObject.Save(1, &s.invalidationJob) + stateSinkObject.Save(2, &s.validUntil) + stateSinkObject.Save(3, &s.preferredUntil) + stateSinkObject.Save(4, &s.stableAddr) + stateSinkObject.Save(5, &s.tempAddrs) + stateSinkObject.Save(6, &s.generationAttempts) + stateSinkObject.Save(7, &s.maxGenerationAttempts) +} + +func (s *slaacPrefixState) afterLoad(context.Context) {} + +// +checklocksignore +func (s *slaacPrefixState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.deprecationJob) + stateSourceObject.Load(1, &s.invalidationJob) + stateSourceObject.Load(2, &s.validUntil) + stateSourceObject.Load(3, &s.preferredUntil) + stateSourceObject.Load(4, &s.stableAddr) + stateSourceObject.Load(5, &s.tempAddrs) + stateSourceObject.Load(6, &s.generationAttempts) + stateSourceObject.Load(7, &s.maxGenerationAttempts) +} + +func (s *Stats) StateTypeName() string { + return "pkg/tcpip/network/ipv6.Stats" +} + +func (s *Stats) StateFields() []string { + return []string{ + "IP", + "ICMP", + "UnhandledRouterAdvertisements", + } +} + +func (s *Stats) beforeSave() {} + +// +checklocksignore +func (s *Stats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.IP) + stateSinkObject.Save(1, &s.ICMP) + stateSinkObject.Save(2, &s.UnhandledRouterAdvertisements) +} + +func (s *Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.IP) + stateSourceObject.Load(1, &s.ICMP) + stateSourceObject.Load(2, &s.UnhandledRouterAdvertisements) +} + +func (s *sharedStats) StateTypeName() string { + return "pkg/tcpip/network/ipv6.sharedStats" +} + +func (s *sharedStats) StateFields() []string { + return []string{ + "localStats", + "ip", + "icmp", + } +} + +func (s *sharedStats) beforeSave() {} + +// +checklocksignore +func (s *sharedStats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.localStats) + stateSinkObject.Save(1, &s.ip) + stateSinkObject.Save(2, &s.icmp) +} + +func (s *sharedStats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *sharedStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.localStats) + stateSourceObject.Load(1, &s.ip) + stateSourceObject.Load(2, &s.icmp) +} + +func (m *multiCounterICMPv6PacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv6.multiCounterICMPv6PacketStats" +} + +func (m *multiCounterICMPv6PacketStats) StateFields() []string { + return []string{ + "echoRequest", + "echoReply", + "dstUnreachable", + "packetTooBig", + "timeExceeded", + "paramProblem", + "routerSolicit", + "routerAdvert", + "neighborSolicit", + "neighborAdvert", + "redirectMsg", + "multicastListenerQuery", + "multicastListenerReport", + "multicastListenerReportV2", + "multicastListenerDone", + } +} + +func (m *multiCounterICMPv6PacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv6PacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.echoRequest) + stateSinkObject.Save(1, &m.echoReply) + stateSinkObject.Save(2, &m.dstUnreachable) + stateSinkObject.Save(3, &m.packetTooBig) + stateSinkObject.Save(4, &m.timeExceeded) + stateSinkObject.Save(5, &m.paramProblem) + stateSinkObject.Save(6, &m.routerSolicit) + stateSinkObject.Save(7, &m.routerAdvert) + stateSinkObject.Save(8, &m.neighborSolicit) + stateSinkObject.Save(9, &m.neighborAdvert) + stateSinkObject.Save(10, &m.redirectMsg) + stateSinkObject.Save(11, &m.multicastListenerQuery) + stateSinkObject.Save(12, &m.multicastListenerReport) + stateSinkObject.Save(13, &m.multicastListenerReportV2) + stateSinkObject.Save(14, &m.multicastListenerDone) +} + +func (m *multiCounterICMPv6PacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv6PacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.echoRequest) + stateSourceObject.Load(1, &m.echoReply) + stateSourceObject.Load(2, &m.dstUnreachable) + stateSourceObject.Load(3, &m.packetTooBig) + stateSourceObject.Load(4, &m.timeExceeded) + stateSourceObject.Load(5, &m.paramProblem) + stateSourceObject.Load(6, &m.routerSolicit) + stateSourceObject.Load(7, &m.routerAdvert) + stateSourceObject.Load(8, &m.neighborSolicit) + stateSourceObject.Load(9, &m.neighborAdvert) + stateSourceObject.Load(10, &m.redirectMsg) + stateSourceObject.Load(11, &m.multicastListenerQuery) + stateSourceObject.Load(12, &m.multicastListenerReport) + stateSourceObject.Load(13, &m.multicastListenerReportV2) + stateSourceObject.Load(14, &m.multicastListenerDone) +} + +func (m *multiCounterICMPv6SentPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv6.multiCounterICMPv6SentPacketStats" +} + +func (m *multiCounterICMPv6SentPacketStats) StateFields() []string { + return []string{ + "multiCounterICMPv6PacketStats", + "dropped", + "rateLimited", + } +} + +func (m *multiCounterICMPv6SentPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv6SentPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.multiCounterICMPv6PacketStats) + stateSinkObject.Save(1, &m.dropped) + stateSinkObject.Save(2, &m.rateLimited) +} + +func (m *multiCounterICMPv6SentPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv6SentPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.multiCounterICMPv6PacketStats) + stateSourceObject.Load(1, &m.dropped) + stateSourceObject.Load(2, &m.rateLimited) +} + +func (m *multiCounterICMPv6ReceivedPacketStats) StateTypeName() string { + return "pkg/tcpip/network/ipv6.multiCounterICMPv6ReceivedPacketStats" +} + +func (m *multiCounterICMPv6ReceivedPacketStats) StateFields() []string { + return []string{ + "multiCounterICMPv6PacketStats", + "unrecognized", + "invalid", + "routerOnlyPacketsDroppedByHost", + } +} + +func (m *multiCounterICMPv6ReceivedPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv6ReceivedPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.multiCounterICMPv6PacketStats) + stateSinkObject.Save(1, &m.unrecognized) + stateSinkObject.Save(2, &m.invalid) + stateSinkObject.Save(3, &m.routerOnlyPacketsDroppedByHost) +} + +func (m *multiCounterICMPv6ReceivedPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv6ReceivedPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.multiCounterICMPv6PacketStats) + stateSourceObject.Load(1, &m.unrecognized) + stateSourceObject.Load(2, &m.invalid) + stateSourceObject.Load(3, &m.routerOnlyPacketsDroppedByHost) +} + +func (m *multiCounterICMPv6Stats) StateTypeName() string { + return "pkg/tcpip/network/ipv6.multiCounterICMPv6Stats" +} + +func (m *multiCounterICMPv6Stats) StateFields() []string { + return []string{ + "packetsSent", + "packetsReceived", + } +} + +func (m *multiCounterICMPv6Stats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterICMPv6Stats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.packetsSent) + stateSinkObject.Save(1, &m.packetsReceived) +} + +func (m *multiCounterICMPv6Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterICMPv6Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.packetsSent) + stateSourceObject.Load(1, &m.packetsReceived) +} + +func init() { + state.Register((*icmpv6DestinationUnreachableSockError)(nil)) + state.Register((*icmpv6DestinationNetworkUnreachableSockError)(nil)) + state.Register((*icmpv6DestinationPortUnreachableSockError)(nil)) + state.Register((*icmpv6DestinationAddressUnreachableSockError)(nil)) + state.Register((*icmpv6PacketTooBigSockError)(nil)) + state.Register((*endpointMu)(nil)) + state.Register((*dadMu)(nil)) + state.Register((*endpointDAD)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*OpaqueInterfaceIdentifierOptions)(nil)) + state.Register((*protocolMu)(nil)) + state.Register((*protocol)(nil)) + state.Register((*Options)(nil)) + state.Register((*MLDOptions)(nil)) + state.Register((*mldState)(nil)) + state.Register((*NDPConfigurations)(nil)) + state.Register((*timer)(nil)) + state.Register((*offLinkRoute)(nil)) + state.Register((*ndpState)(nil)) + state.Register((*offLinkRouteState)(nil)) + state.Register((*onLinkPrefixState)(nil)) + state.Register((*tempSLAACAddrState)(nil)) + state.Register((*stableAddrState)(nil)) + state.Register((*slaacPrefixState)(nil)) + state.Register((*Stats)(nil)) + state.Register((*sharedStats)(nil)) + state.Register((*multiCounterICMPv6PacketStats)(nil)) + state.Register((*multiCounterICMPv6SentPacketStats)(nil)) + state.Register((*multiCounterICMPv6ReceivedPacketStats)(nil)) + state.Register((*multiCounterICMPv6Stats)(nil)) +} diff --git a/pkg/tcpip/network/ipv6/mld.go b/pkg/tcpip/network/ipv6/mld.go new file mode 100644 index 0000000..79e1f3c --- /dev/null +++ b/pkg/tcpip/network/ipv6/mld.go @@ -0,0 +1,478 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv6 + +import ( + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // UnsolicitedReportIntervalMax is the maximum delay between sending + // unsolicited MLD reports. + // + // Obtained from RFC 2710 Section 7.10. + UnsolicitedReportIntervalMax = 10 * time.Second +) + +// MLDVersion is the forced version of MLD. +type MLDVersion int + +const ( + _ MLDVersion = iota + // MLDVersion1 indicates MLDv1. + MLDVersion1 + // MLDVersion2 indicates MLDv2. Note that MLD may still fallback to V1 + // compatibility mode as required by MLDv2. + MLDVersion2 +) + +// MLDEndpoint is a network endpoint that supports MLD. +type MLDEndpoint interface { + // SetMLDVersions sets the MLD version. + // + // Returns the previous MLD version. + SetMLDVersion(MLDVersion) MLDVersion + + // GetMLDVersion returns the MLD version. + GetMLDVersion() MLDVersion +} + +// MLDOptions holds options for MLD. +// +// +stateify savable +type MLDOptions struct { + // Enabled indicates whether MLD will be performed. + // + // When enabled, MLD may transmit MLD report and done messages when + // joining and leaving multicast groups respectively, and handle incoming + // MLD packets. + // + // This field is ignored and is always assumed to be false for interfaces + // without neighbouring nodes (e.g. loopback). + Enabled bool +} + +var _ ip.MulticastGroupProtocol = (*mldState)(nil) + +// mldState is the per-interface MLD state. +// +// mldState.init MUST be called to initialize the MLD state. +// +// +stateify savable +type mldState struct { + // The IPv6 endpoint this mldState is for. + ep *endpoint + + genericMulticastProtocol ip.GenericMulticastProtocolState +} + +// Enabled implements ip.MulticastGroupProtocol. +func (mld *mldState) Enabled() bool { + // No need to perform MLD on loopback interfaces since they don't have + // neighbouring nodes. + return mld.ep.protocol.options.MLD.Enabled && !mld.ep.nic.IsLoopback() && mld.ep.Enabled() +} + +// SendReport implements ip.MulticastGroupProtocol. +// +// Precondition: mld.ep.mu must be read locked. +func (mld *mldState) SendReport(groupAddress tcpip.Address) (bool, tcpip.Error) { + return mld.writePacket(groupAddress, groupAddress, header.ICMPv6MulticastListenerReport) +} + +// SendLeave implements ip.MulticastGroupProtocol. +// +// Precondition: mld.ep.mu must be read locked. +func (mld *mldState) SendLeave(groupAddress tcpip.Address) tcpip.Error { + _, err := mld.writePacket(header.IPv6AllRoutersLinkLocalMulticastAddress, groupAddress, header.ICMPv6MulticastListenerDone) + return err +} + +// ShouldPerformProtocol implements ip.MulticastGroupProtocol. +func (mld *mldState) ShouldPerformProtocol(groupAddress tcpip.Address) bool { + // As per RFC 2710 section 5 page 10, + // + // The link-scope all-nodes address (FF02::1) is handled as a special + // case. The node starts in Idle Listener state for that address on + // every interface, never transitions to another state, and never sends + // a Report or Done for that address. + // + // MLD messages are never sent for multicast addresses whose scope is 0 + // (reserved) or 1 (node-local). + if groupAddress == header.IPv6AllNodesMulticastAddress { + return false + } + + scope := header.V6MulticastScope(groupAddress) + return scope != header.IPv6Reserved0MulticastScope && scope != header.IPv6InterfaceLocalMulticastScope +} + +type mldv2ReportBuilder struct { + mld *mldState + + records []header.MLDv2ReportMulticastAddressRecordSerializer +} + +// AddRecord implements ip.MulticastGroupProtocolV2ReportBuilder. +func (b *mldv2ReportBuilder) AddRecord(genericRecordType ip.MulticastGroupProtocolV2ReportRecordType, groupAddress tcpip.Address) { + var recordType header.MLDv2ReportRecordType + switch genericRecordType { + case ip.MulticastGroupProtocolV2ReportRecordModeIsInclude: + recordType = header.MLDv2ReportRecordModeIsInclude + case ip.MulticastGroupProtocolV2ReportRecordModeIsExclude: + recordType = header.MLDv2ReportRecordModeIsExclude + case ip.MulticastGroupProtocolV2ReportRecordChangeToIncludeMode: + recordType = header.MLDv2ReportRecordChangeToIncludeMode + case ip.MulticastGroupProtocolV2ReportRecordChangeToExcludeMode: + recordType = header.MLDv2ReportRecordChangeToExcludeMode + case ip.MulticastGroupProtocolV2ReportRecordAllowNewSources: + recordType = header.MLDv2ReportRecordAllowNewSources + case ip.MulticastGroupProtocolV2ReportRecordBlockOldSources: + recordType = header.MLDv2ReportRecordBlockOldSources + default: + panic(fmt.Sprintf("unrecognied genericRecordType = %d", genericRecordType)) + } + + b.records = append(b.records, header.MLDv2ReportMulticastAddressRecordSerializer{ + RecordType: recordType, + MulticastAddress: groupAddress, + Sources: nil, + }) +} + +// Send implements ip.MulticastGroupProtocolV2ReportBuilder. +func (b *mldv2ReportBuilder) Send() (sent bool, err tcpip.Error) { + if len(b.records) == 0 { + return false, err + } + + extensionHeaders := header.IPv6ExtHdrSerializer{ + header.IPv6SerializableHopByHopExtHdr{ + &header.IPv6RouterAlertOption{Value: header.IPv6RouterAlertMLD}, + }, + } + mtu := int(b.mld.ep.MTU()) - extensionHeaders.Length() + + allSentWithSpecifiedAddress := true + var firstErr tcpip.Error + for records := b.records; len(records) != 0; { + spaceLeft := mtu + maxRecords := 0 + + for ; maxRecords < len(records); maxRecords++ { + tmp := spaceLeft - records[maxRecords].Length() + if tmp > 0 { + spaceLeft = tmp + } else { + break + } + } + + serializer := header.MLDv2ReportSerializer{Records: records[:maxRecords]} + records = records[maxRecords:] + + icmpView := buffer.NewViewSize(header.ICMPv6HeaderSize + serializer.Length()) + icmp := header.ICMPv6(icmpView.AsSlice()) + serializer.SerializeInto(icmp.MessageBody()) + if sentWithSpecifiedAddress, err := b.mld.writePacketInner( + icmpView, + header.ICMPv6MulticastListenerV2Report, + b.mld.ep.stats.icmp.packetsSent.multicastListenerReportV2, + extensionHeaders, + header.MLDv2RoutersAddress, + ); err != nil { + if firstErr != nil { + firstErr = nil + } + allSentWithSpecifiedAddress = false + } else if !sentWithSpecifiedAddress { + allSentWithSpecifiedAddress = false + } + } + + return allSentWithSpecifiedAddress, firstErr +} + +// NewReportV2Builder implements ip.MulticastGroupProtocol. +func (mld *mldState) NewReportV2Builder() ip.MulticastGroupProtocolV2ReportBuilder { + return &mldv2ReportBuilder{mld: mld} +} + +// V2QueryMaxRespCodeToV2Delay implements ip.MulticastGroupProtocol. +func (*mldState) V2QueryMaxRespCodeToV2Delay(code uint16) time.Duration { + return header.MLDv2MaximumResponseDelay(code) +} + +// V2QueryMaxRespCodeToV1Delay implements ip.MulticastGroupProtocol. +func (*mldState) V2QueryMaxRespCodeToV1Delay(code uint16) time.Duration { + return time.Duration(code) * time.Millisecond +} + +// init sets up an mldState struct, and is required to be called before using +// a new mldState. +// +// Must only be called once for the lifetime of mld. +func (mld *mldState) init(ep *endpoint) { + mld.ep = ep + mld.genericMulticastProtocol.Init(&ep.mu.RWMutex, ip.GenericMulticastProtocolOptions{ + Rand: ep.protocol.stack.InsecureRNG(), + Clock: ep.protocol.stack.Clock(), + Protocol: mld, + MaxUnsolicitedReportDelay: UnsolicitedReportIntervalMax, + }) +} + +// handleMulticastListenerQuery handles a query message. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) handleMulticastListenerQuery(mldHdr header.MLD) { + mld.genericMulticastProtocol.HandleQueryLocked(mldHdr.MulticastAddress(), mldHdr.MaximumResponseDelay()) +} + +// handleMulticastListenerQueryV2 handles a V2 query message. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) handleMulticastListenerQueryV2(mldHdr header.MLDv2Query) { + sources, ok := mldHdr.Sources() + if !ok { + return + } + + mld.genericMulticastProtocol.HandleQueryV2Locked( + mldHdr.MulticastAddress(), + mldHdr.MaximumResponseCode(), + sources, + mldHdr.QuerierRobustnessVariable(), + mldHdr.QuerierQueryInterval(), + ) +} + +// handleMulticastListenerReport handles a report message. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) handleMulticastListenerReport(mldHdr header.MLD) { + mld.genericMulticastProtocol.HandleReportLocked(mldHdr.MulticastAddress()) +} + +// joinGroup handles joining a new group and sending and scheduling the required +// messages. +// +// If the group is already joined, returns *tcpip.ErrDuplicateAddress. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) joinGroup(groupAddress tcpip.Address) { + mld.genericMulticastProtocol.JoinGroupLocked(groupAddress) +} + +// isInGroup returns true if the specified group has been joined locally. +// +// Precondition: mld.ep.mu must be read locked. +func (mld *mldState) isInGroup(groupAddress tcpip.Address) bool { + return mld.genericMulticastProtocol.IsLocallyJoinedRLocked(groupAddress) +} + +// leaveGroup handles removing the group from the membership map, cancels any +// delay timers associated with that group, and sends the Done message, if +// required. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) leaveGroup(groupAddress tcpip.Address) tcpip.Error { + // LeaveGroup returns false only if the group was not joined. + if mld.genericMulticastProtocol.LeaveGroupLocked(groupAddress) { + return nil + } + + return &tcpip.ErrBadLocalAddress{} +} + +// softLeaveAll leaves all groups from the perspective of MLD, but remains +// joined locally. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) softLeaveAll() { + mld.genericMulticastProtocol.MakeAllNonMemberLocked() +} + +// initializeAll attempts to initialize the MLD state for each group that has +// been joined locally. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) initializeAll() { + mld.genericMulticastProtocol.InitializeGroupsLocked() +} + +// sendQueuedReports attempts to send any reports that are queued for sending. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) sendQueuedReports() { + mld.genericMulticastProtocol.SendQueuedReportsLocked() +} + +// setVersion sets the MLD version. +// +// Precondition: mld.ep.mu must be locked. +func (mld *mldState) setVersion(v MLDVersion) MLDVersion { + var prev bool + switch v { + case MLDVersion2: + prev = mld.genericMulticastProtocol.SetV1ModeLocked(false) + case MLDVersion1: + prev = mld.genericMulticastProtocol.SetV1ModeLocked(true) + default: + panic(fmt.Sprintf("unrecognized version = %d", v)) + } + + return toMLDVersion(prev) +} + +func toMLDVersion(v1Generic bool) MLDVersion { + if v1Generic { + return MLDVersion1 + } + return MLDVersion2 +} + +// getVersion returns the MLD version. +// +// Precondition: mld.ep.mu must be read locked. +func (mld *mldState) getVersion() MLDVersion { + return toMLDVersion(mld.genericMulticastProtocol.GetV1ModeLocked()) +} + +// writePacket assembles and sends an MLD packet. +// +// Precondition: mld.ep.mu must be read locked. +func (mld *mldState) writePacket(destAddress, groupAddress tcpip.Address, mldType header.ICMPv6Type) (bool, tcpip.Error) { + sentStats := mld.ep.stats.icmp.packetsSent + var mldStat tcpip.MultiCounterStat + switch mldType { + case header.ICMPv6MulticastListenerReport: + mldStat = sentStats.multicastListenerReport + case header.ICMPv6MulticastListenerDone: + mldStat = sentStats.multicastListenerDone + default: + panic(fmt.Sprintf("unrecognized mld type = %d", mldType)) + } + + icmpView := buffer.NewViewSize(header.ICMPv6HeaderSize + header.MLDMinimumSize) + + icmp := header.ICMPv6(icmpView.AsSlice()) + header.MLD(icmp.MessageBody()).SetMulticastAddress(groupAddress) + extensionHeaders := header.IPv6ExtHdrSerializer{ + header.IPv6SerializableHopByHopExtHdr{ + &header.IPv6RouterAlertOption{Value: header.IPv6RouterAlertMLD}, + }, + } + + return mld.writePacketInner( + icmpView, + mldType, + mldStat, + extensionHeaders, + destAddress, + ) +} + +func (mld *mldState) writePacketInner(buf *buffer.View, mldType header.ICMPv6Type, reportStat tcpip.MultiCounterStat, extensionHeaders header.IPv6ExtHdrSerializer, destAddress tcpip.Address) (bool, tcpip.Error) { + icmp := header.ICMPv6(buf.AsSlice()) + icmp.SetType(mldType) + + // As per RFC 2710 section 3, + // + // All MLD messages described in this document are sent with a link-local + // IPv6 Source Address, an IPv6 Hop Limit of 1, and an IPv6 Router Alert + // option in a Hop-by-Hop Options header. + // + // However, this would cause problems with Duplicate Address Detection with + // the first address as MLD snooping switches may not send multicast traffic + // that DAD depends on to the node performing DAD without the MLD report, as + // documented in RFC 4816: + // + // Note that when a node joins a multicast address, it typically sends a + // Multicast Listener Discovery (MLD) report message [RFC2710] [RFC3810] + // for the multicast address. In the case of Duplicate Address + // Detection, the MLD report message is required in order to inform MLD- + // snooping switches, rather than routers, to forward multicast packets. + // In the above description, the delay for joining the multicast address + // thus means delaying transmission of the corresponding MLD report + // message. Since the MLD specifications do not request a random delay + // to avoid race conditions, just delaying Neighbor Solicitation would + // cause congestion by the MLD report messages. The congestion would + // then prevent the MLD-snooping switches from working correctly and, as + // a result, prevent Duplicate Address Detection from working. The + // requirement to include the delay for the MLD report in this case + // avoids this scenario. [RFC3590] also talks about some interaction + // issues between Duplicate Address Detection and MLD, and specifies + // which source address should be used for the MLD report in this case. + // + // As per RFC 3590 section 4, we should still send out MLD reports with an + // unspecified source address if we do not have an assigned link-local + // address to use as the source address to ensure DAD works as expected on + // networks with MLD snooping switches: + // + // MLD Report and Done messages are sent with a link-local address as + // the IPv6 source address, if a valid address is available on the + // interface. If a valid link-local address is not available (e.g., one + // has not been configured), the message is sent with the unspecified + // address (::) as the IPv6 source address. + // + // Once a valid link-local address is available, a node SHOULD generate + // new MLD Report messages for all multicast addresses joined on the + // interface. + // + // Routers receiving an MLD Report or Done message with the unspecified + // address as the IPv6 source address MUST silently discard the packet + // without taking any action on the packets contents. + // + // Snooping switches MUST manage multicast forwarding state based on MLD + // Report and Done messages sent with the unspecified address as the + // IPv6 source address. + localAddress := mld.ep.getLinkLocalAddressRLocked() + if localAddress.BitLen() == 0 { + localAddress = header.IPv6Any + } + + icmp.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmp, + Src: localAddress, + Dst: destAddress, + })) + + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(mld.ep.MaxHeaderLength()) + extensionHeaders.Length(), + Payload: buffer.MakeWithView(buf), + }) + defer pkt.DecRef() + + if err := addIPHeader(localAddress, destAddress, pkt, stack.NetworkHeaderParams{ + Protocol: header.ICMPv6ProtocolNumber, + TTL: header.MLDHopLimit, + }, extensionHeaders); err != nil { + panic(fmt.Sprintf("failed to add IP header: %s", err)) + } + if err := mld.ep.nic.WritePacketToRemote(header.EthernetAddressFromMulticastIPv6Address(destAddress), pkt); err != nil { + mld.ep.stats.icmp.packetsSent.dropped.Increment() + return false, err + } + reportStat.Increment() + return localAddress != header.IPv6Any, nil +} diff --git a/pkg/tcpip/network/ipv6/ndp.go b/pkg/tcpip/network/ipv6/ndp.go new file mode 100644 index 0000000..a2e3d24 --- /dev/null +++ b/pkg/tcpip/network/ipv6/ndp.go @@ -0,0 +1,2033 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv6 + +import ( + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // defaultMaxRtrSolicitations is the default number of Router + // Solicitation messages to send when an IPv6 endpoint becomes enabled. + // + // Default = 3 (from RFC 4861 section 10). + defaultMaxRtrSolicitations = 3 + + // defaultRtrSolicitationInterval is the default amount of time between + // sending Router Solicitation messages. + // + // Default = 4s (from 4861 section 10). + defaultRtrSolicitationInterval = 4 * time.Second + + // defaultMaxRtrSolicitationDelay is the default maximum amount of time + // to wait before sending the first Router Solicitation message. + // + // Default = 1s (from 4861 section 10). + defaultMaxRtrSolicitationDelay = time.Second + + // defaultHandleRAs is the default configuration for whether or not to + // handle incoming Router Advertisements as a host. + defaultHandleRAs = HandlingRAsEnabledWhenForwardingDisabled + + // defaultDiscoverDefaultRouters is the default configuration for + // whether or not to discover default routers from incoming Router + // Advertisements, as a host. + defaultDiscoverDefaultRouters = true + + // defaultDiscoverMoreSpecificRoutes is the default configuration for + // whether or not to discover more-specific routes from incoming Router + // Advertisements, as a host. + defaultDiscoverMoreSpecificRoutes = true + + // defaultDiscoverOnLinkPrefixes is the default configuration for + // whether or not to discover on-link prefixes from incoming Router + // Advertisements' Prefix Information option, as a host. + defaultDiscoverOnLinkPrefixes = true + + // defaultAutoGenGlobalAddresses is the default configuration for + // whether or not to generate global IPv6 addresses in response to + // receiving a new Prefix Information option with its Autonomous + // Address AutoConfiguration flag set, as a host. + // + // Default = true. + defaultAutoGenGlobalAddresses = true + + // minimumRtrSolicitationInterval is the minimum amount of time to wait + // between sending Router Solicitation messages. This limit is imposed + // to make sure that Router Solicitation messages are not sent all at + // once, defeating the purpose of sending the initial few messages. + minimumRtrSolicitationInterval = 500 * time.Millisecond + + // minimumMaxRtrSolicitationDelay is the minimum amount of time to wait + // before sending the first Router Solicitation message. It is 0 because + // we cannot have a negative delay. + minimumMaxRtrSolicitationDelay = 0 + + // MaxDiscoveredOffLinkRoutes is the maximum number of discovered off-link + // routes. The stack should stop discovering new off-link routes after + // this limit is reached. + // + // This value MUST be at minimum 2 as per RFC 4861 section 6.3.4, and + // SHOULD be more. + MaxDiscoveredOffLinkRoutes = 10 + + // MaxDiscoveredOnLinkPrefixes is the maximum number of discovered + // on-link prefixes. The stack should stop discovering new on-link + // prefixes after discovering MaxDiscoveredOnLinkPrefixes on-link + // prefixes. + MaxDiscoveredOnLinkPrefixes = 10 + + // MaxDiscoveredSLAACPrefixes is the maximum number of discovered + // SLAAC prefixes. The stack will stop discovering new SLAAC + // prefixes after discovering MaxDiscoveredSLAACPrefixes SLAAC prefixes. + MaxDiscoveredSLAACPrefixes = 10 + + // validPrefixLenForAutoGen is the expected prefix length that an + // address can be generated for. Must be 64 bits as the interface + // identifier (IID) is 64 bits and an IPv6 address is 128 bits, so + // 128 - 64 = 64. + validPrefixLenForAutoGen = 64 + + // defaultAutoGenTempGlobalAddresses is the default configuration for whether + // or not to generate temporary SLAAC addresses. + defaultAutoGenTempGlobalAddresses = true + + // defaultMaxTempAddrValidLifetime is the default maximum valid lifetime + // for temporary SLAAC addresses generated as part of RFC 4941. + // + // Default = 7 days (from RFC 4941 section 5). + defaultMaxTempAddrValidLifetime = 7 * 24 * time.Hour + + // defaultMaxTempAddrPreferredLifetime is the default preferred lifetime + // for temporary SLAAC addresses generated as part of RFC 4941. + // + // Default = 1 day (from RFC 4941 section 5). + defaultMaxTempAddrPreferredLifetime = 24 * time.Hour + + // defaultRegenAdvanceDuration is the default duration before the deprecation + // of a temporary address when a new address will be generated. + // + // Default = 5s (from RFC 4941 section 5). + defaultRegenAdvanceDuration = 5 * time.Second + + // minRegenAdvanceDuration is the minimum duration before the deprecation + // of a temporary address when a new address will be generated. + minRegenAdvanceDuration = time.Duration(0) + + // maxSLAACAddrLocalRegenAttempts is the maximum number of times to attempt + // SLAAC address regenerations in response to an IPv6 endpoint-local conflict. + maxSLAACAddrLocalRegenAttempts = 10 + + // MinPrefixInformationValidLifetimeForUpdate is the minimum Valid + // Lifetime to update the valid lifetime of a generated address by + // SLAAC. + // + // Min = 2hrs. + MinPrefixInformationValidLifetimeForUpdate = 2 * time.Hour + + // MaxDesyncFactor is the upper bound for the preferred lifetime's desync + // factor for temporary SLAAC addresses. + // + // Must be greater than 0. + // + // Max = 10m (from RFC 4941 section 5). + MaxDesyncFactor = 10 * time.Minute + + // MinMaxTempAddrPreferredLifetime is the minimum value allowed for the + // maximum preferred lifetime for temporary SLAAC addresses. + // + // This value guarantees that a temporary address is preferred for at + // least 1hr if the SLAAC prefix is valid for at least that time. + MinMaxTempAddrPreferredLifetime = defaultRegenAdvanceDuration + MaxDesyncFactor + time.Hour + + // MinMaxTempAddrValidLifetime is the minimum value allowed for the + // maximum valid lifetime for temporary SLAAC addresses. + // + // This value guarantees that a temporary address is valid for at least + // 2hrs if the SLAAC prefix is valid for at least that time. + MinMaxTempAddrValidLifetime = 2 * time.Hour +) + +// NDPEndpoint is an endpoint that supports NDP. +type NDPEndpoint interface { + // SetNDPConfigurations sets the NDP configurations. + SetNDPConfigurations(NDPConfigurations) + + // NDPConfigurations returns the NDP configurations. + NDPConfigurations() NDPConfigurations +} + +// DHCPv6ConfigurationFromNDPRA is a configuration available via DHCPv6 that an +// NDP Router Advertisement informed the Stack about. +type DHCPv6ConfigurationFromNDPRA int + +const ( + _ DHCPv6ConfigurationFromNDPRA = iota + + // DHCPv6NoConfiguration indicates that no configurations are available via + // DHCPv6. + DHCPv6NoConfiguration + + // DHCPv6ManagedAddress indicates that addresses are available via DHCPv6. + // + // DHCPv6ManagedAddress also implies DHCPv6OtherConfigurations because DHCPv6 + // returns all available configuration information when serving addresses. + DHCPv6ManagedAddress + + // DHCPv6OtherConfigurations indicates that other configuration information is + // available via DHCPv6. + // + // Other configurations are configurations other than addresses. Examples of + // other configurations are recursive DNS server list, DNS search lists and + // default gateway. + DHCPv6OtherConfigurations +) + +// NDPDispatcher is the interface integrators of netstack must implement to +// receive and handle NDP related events. +type NDPDispatcher interface { + // OnDuplicateAddressDetectionResult is called when the DAD process for an + // address on a NIC completes. + // + // This function is not permitted to block indefinitely. This function + // is also not permitted to call into the stack. + OnDuplicateAddressDetectionResult(tcpip.NICID, tcpip.Address, stack.DADResult) + + // OnOffLinkRouteUpdated is called when an off-link route is updated. + // + // This function is not permitted to block indefinitely. This function + // is also not permitted to call into the stack. + OnOffLinkRouteUpdated(tcpip.NICID, tcpip.Subnet, tcpip.Address, header.NDPRoutePreference) + + // OnOffLinkRouteInvalidated is called when an off-link route is invalidated. + // + // This function is not permitted to block indefinitely. This function + // is also not permitted to call into the stack. + OnOffLinkRouteInvalidated(tcpip.NICID, tcpip.Subnet, tcpip.Address) + + // OnOnLinkPrefixDiscovered is called when a new on-link prefix is discovered. + // + // This function is not permitted to block indefinitely. This function + // is also not permitted to call into the stack. + OnOnLinkPrefixDiscovered(tcpip.NICID, tcpip.Subnet) + + // OnOnLinkPrefixInvalidated is called when a discovered on-link prefix that + // was remembered is invalidated. + // + // This function is not permitted to block indefinitely. This function + // is also not permitted to call into the stack. + OnOnLinkPrefixInvalidated(tcpip.NICID, tcpip.Subnet) + + // OnAutoGenAddress is called when a new prefix with its autonomous address- + // configuration flag set is received and SLAAC was performed. + // + // This function is not permitted to block indefinitely. It must not + // call functions on the stack itself. + // + // If a non-nil AddressDispatcher is returned, events related to the address + // will be sent to the dispatcher. + OnAutoGenAddress(tcpip.NICID, tcpip.AddressWithPrefix) stack.AddressDispatcher + + // OnAutoGenAddressDeprecated is called when an auto-generated address (SLAAC) + // is deprecated, but is still considered valid. Note, if an address is + // invalidated at the same time it is deprecated, the deprecation event may + // not be received. + // + // This function is not permitted to block indefinitely. It must not + // call functions on the stack itself. + OnAutoGenAddressDeprecated(tcpip.NICID, tcpip.AddressWithPrefix) + + // OnAutoGenAddressInvalidated is called when an auto-generated address + // (SLAAC) is invalidated. + // + // This function is not permitted to block indefinitely. It must not + // call functions on the stack itself. + OnAutoGenAddressInvalidated(tcpip.NICID, tcpip.AddressWithPrefix) + + // OnRecursiveDNSServerOption is called when the stack learns of DNS servers + // through NDP. Note, the addresses may contain link-local addresses. + // + // It is up to the caller to use the DNS Servers only for their valid + // lifetime. OnRecursiveDNSServerOption may be called for new or + // already known DNS servers. If called with known DNS servers, their + // valid lifetimes must be refreshed to the lifetime (it may be increased, + // decreased, or completely invalidated when the lifetime = 0). + // + // This function is not permitted to block indefinitely. It must not + // call functions on the stack itself. + OnRecursiveDNSServerOption(tcpip.NICID, []tcpip.Address, time.Duration) + + // OnDNSSearchListOption is called when the stack learns of DNS search lists + // through NDP. + // + // It is up to the caller to use the domain names in the search list + // for only their valid lifetime. OnDNSSearchListOption may be called + // with new or already known domain names. If called with known domain + // names, their valid lifetimes must be refreshed to the lifetime (it may + // be increased, decreased or completely invalidated when the lifetime = 0. + OnDNSSearchListOption(tcpip.NICID, []string, time.Duration) + + // OnDHCPv6Configuration is called with an updated configuration that is + // available via DHCPv6 for the passed NIC. + // + // This function is not permitted to block indefinitely. It must not + // call functions on the stack itself. + OnDHCPv6Configuration(tcpip.NICID, DHCPv6ConfigurationFromNDPRA) +} + +var _ fmt.Stringer = HandleRAsConfiguration(0) + +// HandleRAsConfiguration enumerates when RAs may be handled. +type HandleRAsConfiguration int + +const ( + // HandlingRAsDisabled indicates that Router Advertisements will not be + // handled. + HandlingRAsDisabled HandleRAsConfiguration = iota + + // HandlingRAsEnabledWhenForwardingDisabled indicates that router + // advertisements will only be handled when forwarding is disabled. + HandlingRAsEnabledWhenForwardingDisabled + + // HandlingRAsAlwaysEnabled indicates that Router Advertisements will always + // be handled, even when forwarding is enabled. + HandlingRAsAlwaysEnabled +) + +// String implements fmt.Stringer. +func (c HandleRAsConfiguration) String() string { + switch c { + case HandlingRAsDisabled: + return "HandlingRAsDisabled" + case HandlingRAsEnabledWhenForwardingDisabled: + return "HandlingRAsEnabledWhenForwardingDisabled" + case HandlingRAsAlwaysEnabled: + return "HandlingRAsAlwaysEnabled" + default: + return fmt.Sprintf("HandleRAsConfiguration(%d)", c) + } +} + +// enabled returns true iff Router Advertisements may be handled given the +// specified forwarding status. +func (c HandleRAsConfiguration) enabled(forwarding bool) bool { + switch c { + case HandlingRAsDisabled: + return false + case HandlingRAsEnabledWhenForwardingDisabled: + return !forwarding + case HandlingRAsAlwaysEnabled: + return true + default: + panic(fmt.Sprintf("unhandled HandleRAsConfiguration = %d", c)) + } +} + +// NDPConfigurations is the NDP configurations for the netstack. +// +// +stateify savable +type NDPConfigurations struct { + // The number of Router Solicitation messages to send when the IPv6 endpoint + // becomes enabled. + // + // Ignored unless configured to handle Router Advertisements. + MaxRtrSolicitations uint8 + + // The amount of time between transmitting Router Solicitation messages. + // + // Must be greater than or equal to 0.5s. + RtrSolicitationInterval time.Duration + + // The maximum amount of time before transmitting the first Router + // Solicitation message. + // + // Must be greater than or equal to 0s. + MaxRtrSolicitationDelay time.Duration + + // HandleRAs is the configuration for when Router Advertisements should be + // handled. + HandleRAs HandleRAsConfiguration + + // DiscoverDefaultRouters determines whether or not default routers are + // discovered from Router Advertisements, as per RFC 4861 section 6. This + // configuration is ignored if RAs will not be processed (see HandleRAs). + DiscoverDefaultRouters bool + + // DiscoverMoreSpecificRoutes determines whether or not more specific routes + // are discovered from Router Advertisements, as per RFC 4191. This + // configuration is ignored if RAs will not be processed (see HandleRAs). + DiscoverMoreSpecificRoutes bool + + // DiscoverOnLinkPrefixes determines whether or not on-link prefixes are + // discovered from Router Advertisements' Prefix Information option, as per + // RFC 4861 section 6. This configuration is ignored if RAs will not be + // processed (see HandleRAs). + DiscoverOnLinkPrefixes bool + + // AutoGenGlobalAddresses determines whether or not an IPv6 endpoint performs + // SLAAC to auto-generate global SLAAC addresses in response to Prefix + // Information options, as per RFC 4862. + // + // Note, if an address was already generated for some unique prefix, as + // part of SLAAC, this option does not affect whether or not the + // lifetime(s) of the generated address changes; this option only + // affects the generation of new addresses as part of SLAAC. + AutoGenGlobalAddresses bool + + // AutoGenAddressConflictRetries determines how many times to attempt to retry + // generation of a permanent auto-generated address in response to DAD + // conflicts. + // + // If the method used to generate the address does not support creating + // alternative addresses (e.g. IIDs based on the modified EUI64 of a NIC's + // MAC address), then no attempt is made to resolve the conflict. + AutoGenAddressConflictRetries uint8 + + // AutoGenTempGlobalAddresses determines whether or not temporary SLAAC + // addresses are generated for an IPv6 endpoint as part of SLAAC privacy + // extensions, as per RFC 4941. + // + // Ignored if AutoGenGlobalAddresses is false. + AutoGenTempGlobalAddresses bool + + // MaxTempAddrValidLifetime is the maximum valid lifetime for temporary + // SLAAC addresses. + MaxTempAddrValidLifetime time.Duration + + // MaxTempAddrPreferredLifetime is the maximum preferred lifetime for + // temporary SLAAC addresses. + MaxTempAddrPreferredLifetime time.Duration + + // RegenAdvanceDuration is the duration before the deprecation of a temporary + // address when a new address will be generated. + RegenAdvanceDuration time.Duration +} + +// DefaultNDPConfigurations returns an NDPConfigurations populated with +// default values. +func DefaultNDPConfigurations() NDPConfigurations { + return NDPConfigurations{ + MaxRtrSolicitations: defaultMaxRtrSolicitations, + RtrSolicitationInterval: defaultRtrSolicitationInterval, + MaxRtrSolicitationDelay: defaultMaxRtrSolicitationDelay, + HandleRAs: defaultHandleRAs, + DiscoverDefaultRouters: defaultDiscoverDefaultRouters, + DiscoverMoreSpecificRoutes: defaultDiscoverMoreSpecificRoutes, + DiscoverOnLinkPrefixes: defaultDiscoverOnLinkPrefixes, + AutoGenGlobalAddresses: defaultAutoGenGlobalAddresses, + AutoGenTempGlobalAddresses: defaultAutoGenTempGlobalAddresses, + MaxTempAddrValidLifetime: defaultMaxTempAddrValidLifetime, + MaxTempAddrPreferredLifetime: defaultMaxTempAddrPreferredLifetime, + RegenAdvanceDuration: defaultRegenAdvanceDuration, + } +} + +// validate modifies an NDPConfigurations with valid values. If invalid values +// are present in c, the corresponding default values are used instead. +func (c *NDPConfigurations) validate() { + if c.RtrSolicitationInterval < minimumRtrSolicitationInterval { + c.RtrSolicitationInterval = defaultRtrSolicitationInterval + } + + if c.MaxRtrSolicitationDelay < minimumMaxRtrSolicitationDelay { + c.MaxRtrSolicitationDelay = defaultMaxRtrSolicitationDelay + } + + if c.MaxTempAddrValidLifetime < MinMaxTempAddrValidLifetime { + c.MaxTempAddrValidLifetime = MinMaxTempAddrValidLifetime + } + + if c.MaxTempAddrPreferredLifetime < MinMaxTempAddrPreferredLifetime || c.MaxTempAddrPreferredLifetime > c.MaxTempAddrValidLifetime { + c.MaxTempAddrPreferredLifetime = MinMaxTempAddrPreferredLifetime + } + + if c.RegenAdvanceDuration < minRegenAdvanceDuration { + c.RegenAdvanceDuration = minRegenAdvanceDuration + } +} + +// +stateify savable +type timer struct { + // done indicates to the timer that the timer was stopped. + done *bool + + timer tcpip.Timer +} + +// +stateify savable +type offLinkRoute struct { + dest tcpip.Subnet + router tcpip.Address +} + +// ndpState is the per-Interface NDP state. +// +// +stateify savable +type ndpState struct { + // Do not allow overwriting this state. + _ sync.NoCopy `state:"nosave"` + + // The IPv6 endpoint this ndpState is for. + ep *endpoint + + // configs is the per-interface NDP configurations. + configs NDPConfigurations + + // The DAD timers to send the next NS message, or resolve the address. + dad ip.DAD + + // The off-link routes discovered through Router Advertisements. + offLinkRoutes map[offLinkRoute]offLinkRouteState + + // rtrSolicitTimer is the timer used to send the next router solicitation + // message. + // + // rtrSolicitTimer is the zero value when NDP is not soliciting routers. + rtrSolicitTimer timer + + // The on-link prefixes discovered through Router Advertisements' Prefix + // Information option. + onLinkPrefixes map[tcpip.Subnet]onLinkPrefixState + + // The SLAAC prefixes discovered through Router Advertisements' Prefix + // Information option. + slaacPrefixes map[tcpip.Subnet]slaacPrefixState + + // The last learned DHCPv6 configuration from an NDP RA. + dhcpv6Configuration DHCPv6ConfigurationFromNDPRA + + // temporaryIIDHistory is the history value used to generate a new temporary + // IID. + temporaryIIDHistory [header.IIDSize]byte + + // temporaryAddressDesyncFactor is the preferred lifetime's desync factor for + // temporary SLAAC addresses. + temporaryAddressDesyncFactor time.Duration +} + +// offLinkRouteState holds data associated with an off-link route discovered by +// a Router Advertisement (RA). +// +// +stateify savable +type offLinkRouteState struct { + prf header.NDPRoutePreference + + // Job to invalidate the route. + // + // Must not be nil. + invalidationJob *tcpip.Job +} + +// onLinkPrefixState holds data associated with an on-link prefix discovered by +// a Router Advertisement's Prefix Information option (PI) when the NDP +// configurations was configured to do so. +// +// +stateify savable +type onLinkPrefixState struct { + // Job to invalidate the on-link prefix. + // + // Must not be nil. + invalidationJob *tcpip.Job +} + +// tempSLAACAddrState holds state associated with a temporary SLAAC address. +// +// +stateify savable +type tempSLAACAddrState struct { + // Job to deprecate the temporary SLAAC address. + // + // Must not be nil. + deprecationJob *tcpip.Job + + // Job to invalidate the temporary SLAAC address. + // + // Must not be nil. + invalidationJob *tcpip.Job + + // Job to regenerate the temporary SLAAC address. + // + // Must not be nil. + regenJob *tcpip.Job + + createdAt tcpip.MonotonicTime + + // The address's endpoint. + // + // Must not be nil. + addressEndpoint stack.AddressEndpoint + + // Has a new temporary SLAAC address already been regenerated? + regenerated bool +} + +// +stateify savable +type stableAddrState struct { + // The address's endpoint. + // + // May only be nil when the address is being (re-)generated. Otherwise, + // must not be nil as all SLAAC prefixes must have a stable address. + addressEndpoint stack.AddressEndpoint + + // The number of times an address has been generated locally where the IPv6 + // endpoint already had the generated address. + localGenerationFailures uint8 +} + +// slaacPrefixState holds state associated with a SLAAC prefix. +// +// +stateify savable +type slaacPrefixState struct { + // Job to deprecate the prefix. + // + // Must not be nil. + deprecationJob *tcpip.Job + + // Job to invalidate the prefix. + // + // Must not be nil. + invalidationJob *tcpip.Job + + // nil iff the address is valid forever. + validUntil *tcpip.MonotonicTime + + // nil iff the address is preferred forever. + preferredUntil *tcpip.MonotonicTime + + // State associated with the stable address generated for the prefix. + stableAddr stableAddrState + + // The temporary (short-lived) addresses generated for the SLAAC prefix. + tempAddrs map[tcpip.Address]tempSLAACAddrState + + // The next two fields are used by both stable and temporary addresses + // generated for a SLAAC prefix. This is safe as only 1 address is in the + // generation and DAD process at any time. That is, no two addresses are + // generated at the same time for a given SLAAC prefix. + + // The number of times an address has been generated and added to the IPv6 + // endpoint. + // + // Addresses may be regenerated in reseponse to a DAD conflicts. + generationAttempts uint8 + + // The maximum number of times to attempt regeneration of a SLAAC address + // in response to DAD conflicts. + maxGenerationAttempts uint8 +} + +// startDuplicateAddressDetection performs Duplicate Address Detection. +// +// This function must only be called by IPv6 addresses that are currently +// tentative. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) startDuplicateAddressDetection(addr tcpip.Address, addressEndpoint stack.AddressEndpoint) tcpip.Error { + // addr must be a valid unicast IPv6 address. + if !header.IsV6UnicastAddress(addr) { + return &tcpip.ErrAddressFamilyNotSupported{} + } + + if addressEndpoint.GetKind() != stack.PermanentTentative { + // The endpoint should be marked as tentative since we are starting DAD. + panic(fmt.Sprintf("ndpdad: addr %s is not tentative on NIC(%d)", addr, ndp.ep.nic.ID())) + } + + ret := ndp.dad.CheckDuplicateAddressLocked(addr, func(r stack.DADResult) { + if addressEndpoint.GetKind() != stack.PermanentTentative { + // The endpoint should still be marked as tentative since we are still + // performing DAD on it. + panic(fmt.Sprintf("ndpdad: addr %s is no longer tentative on NIC(%d)", addr, ndp.ep.nic.ID())) + } + + var dadSucceeded bool + switch r.(type) { + case *stack.DADAborted, *stack.DADError, *stack.DADDupAddrDetected: + dadSucceeded = false + case *stack.DADSucceeded: + dadSucceeded = true + default: + panic(fmt.Sprintf("unrecognized DAD result = %T", r)) + } + + if dadSucceeded { + addressEndpoint.SetKind(stack.Permanent) + } + + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnDuplicateAddressDetectionResult(ndp.ep.nic.ID(), addr, r) + } + + if dadSucceeded { + if addressEndpoint.ConfigType() == stack.AddressConfigSlaac && !addressEndpoint.Temporary() { + // Reset the generation attempts counter as we are starting the + // generation of a new address for the SLAAC prefix. + ndp.regenerateTempSLAACAddr(addressEndpoint.AddressWithPrefix().Subnet(), true /* resetGenAttempts */) + } + ndp.ep.onAddressAssignedLocked(addr) + } + }) + + switch ret { + case stack.DADStarting: + case stack.DADAlreadyRunning: + panic(fmt.Sprintf("ndpdad: already performing DAD for addr %s on NIC(%d)", addr, ndp.ep.nic.ID())) + case stack.DADDisabled: + addressEndpoint.SetKind(stack.Permanent) + + // Consider DAD to have resolved even if no DAD messages were actually + // transmitted. + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnDuplicateAddressDetectionResult(ndp.ep.nic.ID(), addr, &stack.DADSucceeded{}) + } + + ndp.ep.onAddressAssignedLocked(addr) + } + + return nil +} + +// stopDuplicateAddressDetection ends a running Duplicate Address Detection +// process. Note, this may leave the DAD process for a tentative address in +// such a state forever, unless some other external event resolves the DAD +// process (receiving an NA from the true owner of addr, or an NS for addr +// (implying another node is attempting to use addr)). It is up to the caller +// of this function to handle such a scenario. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) stopDuplicateAddressDetection(addr tcpip.Address, reason stack.DADResult) { + ndp.dad.StopLocked(addr, reason) +} + +// handleRA handles a Router Advertisement message that arrived on the NIC +// this ndp is for. Does nothing if the NIC is configured to not handle RAs. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) handleRA(ip tcpip.Address, ra header.NDPRouterAdvert) { + // Is the IPv6 endpoint configured to handle RAs at all? + // + // Currently, the stack does not determine router interface status on a + // per-interface basis; it is a protocol-wide configuration, so we check the + // protocol's forwarding flag to determine if the IPv6 endpoint is forwarding + // packets. + if !ndp.configs.HandleRAs.enabled(ndp.ep.Forwarding()) { + ndp.ep.stats.localStats.UnhandledRouterAdvertisements.Increment() + return + } + + // Only worry about the DHCPv6 configuration if we have an NDPDispatcher as we + // only inform the dispatcher on configuration changes. We do nothing else + // with the information. + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + var configuration DHCPv6ConfigurationFromNDPRA + switch { + case ra.ManagedAddrConfFlag(): + configuration = DHCPv6ManagedAddress + + case ra.OtherConfFlag(): + configuration = DHCPv6OtherConfigurations + + default: + configuration = DHCPv6NoConfiguration + } + + if ndp.dhcpv6Configuration != configuration { + ndp.dhcpv6Configuration = configuration + ndpDisp.OnDHCPv6Configuration(ndp.ep.nic.ID(), configuration) + } + } + + // Is the IPv6 endpoint configured to discover default routers? + if ndp.configs.DiscoverDefaultRouters { + prf := ra.DefaultRouterPreference() + if prf == header.ReservedRoutePreference { + // As per RFC 4191 section 2.2, + // + // Prf (Default Router Preference) + // + // If the Reserved (10) value is received, the receiver MUST treat the + // value as if it were (00). + // + // Note that the value 00 is the medium (default) router preference value. + prf = header.MediumRoutePreference + } + + // We represent default routers with a default (off-link) route through the + // router. + ndp.handleOffLinkRouteDiscovery(offLinkRoute{dest: header.IPv6EmptySubnet, router: ip}, ra.RouterLifetime(), prf) + } + + // TODO(b/141556115): Do (RetransTimer, ReachableTime)) Parameter + // Discovery. + + // We know the options is valid as far as wire format is concerned since + // we got the Router Advertisement, as documented by this fn. Given this + // we do not check the iterator for errors on calls to Next. + it, _ := ra.Options().Iter(false) + for opt, done, _ := it.Next(); !done; opt, done, _ = it.Next() { + switch opt := opt.(type) { + case header.NDPRecursiveDNSServer: + if ndp.ep.protocol.options.NDPDisp == nil { + continue + } + + addrs, _ := opt.Addresses() + ndp.ep.protocol.options.NDPDisp.OnRecursiveDNSServerOption(ndp.ep.nic.ID(), addrs, opt.Lifetime()) + + case header.NDPDNSSearchList: + if ndp.ep.protocol.options.NDPDisp == nil { + continue + } + + domainNames, _ := opt.DomainNames() + ndp.ep.protocol.options.NDPDisp.OnDNSSearchListOption(ndp.ep.nic.ID(), domainNames, opt.Lifetime()) + + case header.NDPPrefixInformation: + prefix := opt.Subnet() + + // Is the prefix a link-local? + if header.IsV6LinkLocalUnicastAddress(prefix.ID()) { + // ...Yes, skip as per RFC 4861 section 6.3.4, + // and RFC 4862 section 5.5.3.b (for SLAAC). + continue + } + + // Is the Prefix Length 0? + if prefix.Prefix() == 0 { + // ...Yes, skip as this is an invalid prefix + // as all IPv6 addresses cannot be on-link. + continue + } + + if opt.OnLinkFlag() { + ndp.handleOnLinkPrefixInformation(opt) + } + + if opt.AutonomousAddressConfigurationFlag() { + ndp.handleAutonomousPrefixInformation(opt) + } + + case header.NDPRouteInformation: + if !ndp.configs.DiscoverMoreSpecificRoutes { + continue + } + + dest, err := opt.Prefix() + if err != nil { + panic(fmt.Sprintf("%T.Prefix(): %s", opt, err)) + } + + prf := opt.RoutePreference() + if prf == header.ReservedRoutePreference { + // As per RFC 4191 section 2.3, + // + // Prf (Route Preference) + // 2-bit signed integer. The Route Preference indicates + // whether to prefer the router associated with this prefix + // over others, when multiple identical prefixes (for + // different routers) have been received. If the Reserved + // (10) value is received, the Route Information Option MUST + // be ignored. + continue + } + + ndp.handleOffLinkRouteDiscovery(offLinkRoute{dest: dest, router: ip}, opt.RouteLifetime(), prf) + } + + // TODO(b/141556115): Do (MTU) Parameter Discovery. + } +} + +// invalidateOffLinkRoute invalidates a discovered off-link route. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) invalidateOffLinkRoute(route offLinkRoute) { + state, ok := ndp.offLinkRoutes[route] + if !ok { + return + } + + state.invalidationJob.Cancel() + delete(ndp.offLinkRoutes, route) + + // Let the integrator know a discovered off-link route is invalidated. + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnOffLinkRouteInvalidated(ndp.ep.nic.ID(), route.dest, route.router) + } +} + +// handleOffLinkRouteDiscovery handles the discovery of an off-link route. +// +// Precondition: ndp.ep.mu must be locked. +func (ndp *ndpState) handleOffLinkRouteDiscovery(route offLinkRoute, lifetime time.Duration, prf header.NDPRoutePreference) { + ndpDisp := ndp.ep.protocol.options.NDPDisp + if ndpDisp == nil { + return + } + + state, ok := ndp.offLinkRoutes[route] + switch { + case !ok && lifetime != 0: + // This is a new route we are discovering. + // + // Only remember it if we currently know about less than + // MaxDiscoveredOffLinkRoutes routers. + if len(ndp.offLinkRoutes) < MaxDiscoveredOffLinkRoutes { + // Inform the integrator when we discovered an off-link route. + ndpDisp.OnOffLinkRouteUpdated(ndp.ep.nic.ID(), route.dest, route.router, prf) + + state := offLinkRouteState{ + prf: prf, + invalidationJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + ndp.invalidateOffLinkRoute(route) + }), + } + + state.invalidationJob.Schedule(lifetime) + + ndp.offLinkRoutes[route] = state + } + + case ok && lifetime != 0: + // This is an already discovered off-link route. Update the lifetime. + state.invalidationJob.Cancel() + state.invalidationJob.Schedule(lifetime) + + if prf != state.prf { + state.prf = prf + + // Inform the integrator about route preference updates. + ndpDisp.OnOffLinkRouteUpdated(ndp.ep.nic.ID(), route.dest, route.router, prf) + } + + ndp.offLinkRoutes[route] = state + + case ok && lifetime == 0: + // The already discovered off-link route is no longer considered valid so we + // invalidate it immediately. + ndp.invalidateOffLinkRoute(route) + } +} + +// rememberOnLinkPrefix remembers a newly discovered on-link prefix with IPv6 +// address with prefix prefix with lifetime l. +// +// The prefix identified by prefix MUST NOT already be known. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) rememberOnLinkPrefix(prefix tcpip.Subnet, l time.Duration) { + ndpDisp := ndp.ep.protocol.options.NDPDisp + if ndpDisp == nil { + return + } + + // Inform the integrator when we discovered an on-link prefix. + ndpDisp.OnOnLinkPrefixDiscovered(ndp.ep.nic.ID(), prefix) + + state := onLinkPrefixState{ + invalidationJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + ndp.invalidateOnLinkPrefix(prefix) + }), + } + + if l < header.NDPInfiniteLifetime { + state.invalidationJob.Schedule(l) + } + + ndp.onLinkPrefixes[prefix] = state +} + +// invalidateOnLinkPrefix invalidates a discovered on-link prefix. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) invalidateOnLinkPrefix(prefix tcpip.Subnet) { + s, ok := ndp.onLinkPrefixes[prefix] + + // Is the on-link prefix still discovered? + if !ok { + // ...Nope, do nothing further. + return + } + + s.invalidationJob.Cancel() + delete(ndp.onLinkPrefixes, prefix) + + // Let the integrator know a discovered on-link prefix is invalidated. + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnOnLinkPrefixInvalidated(ndp.ep.nic.ID(), prefix) + } +} + +// handleOnLinkPrefixInformation handles a Prefix Information option with +// its on-link flag set, as per RFC 4861 section 6.3.4. +// +// handleOnLinkPrefixInformation assumes that the prefix this pi is for is +// not the link-local prefix and the on-link flag is set. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) handleOnLinkPrefixInformation(pi header.NDPPrefixInformation) { + prefix := pi.Subnet() + prefixState, ok := ndp.onLinkPrefixes[prefix] + vl := pi.ValidLifetime() + + if !ok && vl == 0 { + // Don't know about this prefix but it has a zero valid + // lifetime, so just ignore. + return + } + + if !ok && vl != 0 { + // This is a new on-link prefix we are discovering + // + // Only remember it if we currently know about less than + // MaxDiscoveredOnLinkPrefixes on-link prefixes. + if ndp.configs.DiscoverOnLinkPrefixes && len(ndp.onLinkPrefixes) < MaxDiscoveredOnLinkPrefixes { + ndp.rememberOnLinkPrefix(prefix, vl) + } + return + } + + if ok && vl == 0 { + // We know about the on-link prefix, but it is + // no longer to be considered on-link, so + // invalidate it. + ndp.invalidateOnLinkPrefix(prefix) + return + } + + // This is an already discovered on-link prefix with a + // new non-zero valid lifetime. + // + // Update the invalidation job. + + prefixState.invalidationJob.Cancel() + + if vl < header.NDPInfiniteLifetime { + // Prefix is valid for a finite lifetime, schedule the job to execute after + // the new valid lifetime. + prefixState.invalidationJob.Schedule(vl) + } + + ndp.onLinkPrefixes[prefix] = prefixState +} + +// handleAutonomousPrefixInformation handles a Prefix Information option with +// its autonomous flag set, as per RFC 4862 section 5.5.3. +// +// handleAutonomousPrefixInformation assumes that the prefix this pi is for is +// not the link-local prefix and the autonomous flag is set. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) handleAutonomousPrefixInformation(pi header.NDPPrefixInformation) { + vl := pi.ValidLifetime() + pl := pi.PreferredLifetime() + + // If the preferred lifetime is greater than the valid lifetime, + // silently ignore the Prefix Information option, as per RFC 4862 + // section 5.5.3.c. + if pl > vl { + return + } + + prefix := pi.Subnet() + + // Check if we already maintain SLAAC state for prefix. + if state, ok := ndp.slaacPrefixes[prefix]; ok { + // As per RFC 4862 section 5.5.3.e, refresh prefix's SLAAC lifetimes. + ndp.refreshSLAACPrefixLifetimes(prefix, &state, pl, vl) + ndp.slaacPrefixes[prefix] = state + return + } + + // prefix is a new SLAAC prefix. Do the work as outlined by RFC 4862 section + // 5.5.3.d if ndp is configured to auto-generate new addresses via SLAAC. + if !ndp.configs.AutoGenGlobalAddresses { + return + } + + // Limit the number of discovered SLAAC prefixes. + if len(ndp.slaacPrefixes) == MaxDiscoveredSLAACPrefixes { + return + } + + ndp.doSLAAC(prefix, pl, vl) +} + +// doSLAAC generates a new SLAAC address with the provided lifetimes +// for prefix. +// +// pl is the new preferred lifetime. vl is the new valid lifetime. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) doSLAAC(prefix tcpip.Subnet, pl, vl time.Duration) { + // If we do not already have an address for this prefix and the valid + // lifetime is 0, no need to do anything further, as per RFC 4862 + // section 5.5.3.d. + if vl == 0 { + return + } + + // Make sure the prefix is valid (as far as its length is concerned) to + // generate a valid IPv6 address from an interface identifier (IID), as + // per RFC 4862 sectiion 5.5.3.d. + if prefix.Prefix() != validPrefixLenForAutoGen { + return + } + + state := slaacPrefixState{ + deprecationJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + state, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: must have a slaacPrefixes entry for the deprecated SLAAC prefix %s", prefix)) + } + + ndp.deprecateSLAACAddress(state.stableAddr.addressEndpoint) + }), + invalidationJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + state, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: must have a slaacPrefixes entry for the invalidated SLAAC prefix %s", prefix)) + } + + ndp.invalidateSLAACPrefix(prefix, state) + }), + tempAddrs: make(map[tcpip.Address]tempSLAACAddrState), + maxGenerationAttempts: ndp.configs.AutoGenAddressConflictRetries + 1, + } + + now := ndp.ep.protocol.stack.Clock().NowMonotonic() + + // The time an address is preferred until is needed to properly generate the + // address. + if pl < header.NDPInfiniteLifetime { + t := now.Add(pl) + state.preferredUntil = &t + } + // The time at which an address is invalidated is exposed as a property of the + // address. + if vl < header.NDPInfiniteLifetime { + t := now.Add(vl) + state.validUntil = &t + } + + if !ndp.generateSLAACAddr(prefix, &state) { + // We were unable to generate an address for the prefix, we do not nothing + // further as there is no reason to maintain state or jobs for a prefix we + // do not have an address for. + return + } + + // Setup the initial jobs to deprecate and invalidate prefix. + + if pl < header.NDPInfiniteLifetime && pl != 0 { + state.deprecationJob.Schedule(pl) + } + + if vl < header.NDPInfiniteLifetime { + state.invalidationJob.Schedule(vl) + } + + // If the address is assigned (DAD resolved), generate a temporary address. + if state.stableAddr.addressEndpoint.GetKind() == stack.Permanent { + // Reset the generation attempts counter as we are starting the generation + // of a new address for the SLAAC prefix. + ndp.generateTempSLAACAddr(prefix, &state, true /* resetGenAttempts */) + } + + ndp.slaacPrefixes[prefix] = state +} + +// addAndAcquireSLAACAddr adds a SLAAC address to the IPv6 endpoint. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) addAndAcquireSLAACAddr(addr tcpip.AddressWithPrefix, temporary bool, lifetimes stack.AddressLifetimes) stack.AddressEndpoint { + addressEndpoint, err := ndp.ep.addAndAcquirePermanentAddressLocked(addr, stack.AddressProperties{ + PEB: stack.FirstPrimaryEndpoint, + ConfigType: stack.AddressConfigSlaac, + Lifetimes: lifetimes, + Temporary: temporary, + }) + if err != nil { + panic(fmt.Sprintf("ndp: error when adding SLAAC address %+v: %s", addr, err)) + } + + // Inform the integrator that we have a new SLAAC address. + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + if disp := ndpDisp.OnAutoGenAddress(ndp.ep.nic.ID(), addr); disp != nil { + addressEndpoint.RegisterDispatcher(disp) + } + } + + return addressEndpoint +} + +// generateSLAACAddr generates a SLAAC address for prefix. +// +// Returns true if an address was successfully generated. +// +// Panics if the prefix is not a SLAAC prefix or it already has an address. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) generateSLAACAddr(prefix tcpip.Subnet, state *slaacPrefixState) bool { + if addressEndpoint := state.stableAddr.addressEndpoint; addressEndpoint != nil { + panic(fmt.Sprintf("ndp: SLAAC prefix %s already has a permanent address %s", prefix, addressEndpoint.AddressWithPrefix())) + } + + // If we have already reached the maximum address generation attempts for the + // prefix, do not generate another address. + if state.generationAttempts == state.maxGenerationAttempts { + return false + } + + var generatedAddr tcpip.AddressWithPrefix + prefixID := prefix.ID() + addrBytes := prefixID.AsSlice() + + for i := 0; ; i++ { + // If we were unable to generate an address after the maximum SLAAC address + // local regeneration attempts, do nothing further. + if i == maxSLAACAddrLocalRegenAttempts { + return false + } + + dadCounter := state.generationAttempts + state.stableAddr.localGenerationFailures + if oIID := ndp.ep.protocol.options.OpaqueIIDOpts; oIID.NICNameFromID != nil { + addrBytes = header.AppendOpaqueInterfaceIdentifier( + addrBytes[:header.IIDOffsetInIPv6Address], + prefix, + oIID.NICNameFromID(ndp.ep.nic.ID(), ndp.ep.nic.Name()), + dadCounter, + oIID.SecretKey, + ) + } else if dadCounter == 0 { + // Modified-EUI64 based IIDs have no way to resolve DAD conflicts, so if + // the DAD counter is non-zero, we cannot use this method. + // + // Only attempt to generate an interface-specific IID if we have a valid + // link address. + // + // TODO(b/141011931): Validate a LinkEndpoint's link address (provided by + // LinkEndpoint.LinkAddress) before reaching this point. + linkAddr := ndp.ep.nic.LinkAddress() + if !header.IsValidUnicastEthernetAddress(linkAddr) { + return false + } + + // Generate an address within prefix from the modified EUI-64 of ndp's + // NIC's Ethernet MAC address. + header.EthernetAdddressToModifiedEUI64IntoBuf(linkAddr, addrBytes[header.IIDOffsetInIPv6Address:]) + } else { + // We have no way to regenerate an address in response to an address + // conflict when addresses are not generated with opaque IIDs. + return false + } + + generatedAddr = tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom16Slice(addrBytes), + PrefixLen: validPrefixLenForAutoGen, + } + + if !ndp.ep.hasPermanentAddressRLocked(generatedAddr.Address) { + break + } + + state.stableAddr.localGenerationFailures++ + } + + deprecated := state.preferredUntil != nil && !state.preferredUntil.After(ndp.ep.protocol.stack.Clock().NowMonotonic()) + var preferredUntil tcpip.MonotonicTime + if !deprecated { + if state.preferredUntil != nil { + preferredUntil = *state.preferredUntil + } else { + preferredUntil = tcpip.MonotonicTimeInfinite() + } + } + validUntil := tcpip.MonotonicTimeInfinite() + if state.validUntil != nil { + validUntil = *state.validUntil + } + if addressEndpoint := ndp.addAndAcquireSLAACAddr(generatedAddr, false /* temporary */, stack.AddressLifetimes{ + Deprecated: deprecated, + PreferredUntil: preferredUntil, + ValidUntil: validUntil, + }); addressEndpoint != nil { + state.stableAddr.addressEndpoint = addressEndpoint + state.generationAttempts++ + return true + } + + return false +} + +// regenerateSLAACAddr regenerates an address for a SLAAC prefix. +// +// If generating a new address for the prefix fails, the prefix is invalidated. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) regenerateSLAACAddr(prefix tcpip.Subnet) { + state, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: SLAAC prefix state not found to regenerate address for %s", prefix)) + } + + if ndp.generateSLAACAddr(prefix, &state) { + ndp.slaacPrefixes[prefix] = state + return + } + + // We were unable to generate a permanent address for the SLAAC prefix so + // invalidate the prefix as there is no reason to maintain state for a + // SLAAC prefix we do not have an address for. + ndp.invalidateSLAACPrefix(prefix, state) +} + +// generateTempSLAACAddr generates a new temporary SLAAC address. +// +// If resetGenAttempts is true, the prefix's generation counter is reset. +// +// Returns true if a new address was generated. +func (ndp *ndpState) generateTempSLAACAddr(prefix tcpip.Subnet, prefixState *slaacPrefixState, resetGenAttempts bool) bool { + // Are we configured to auto-generate new temporary global addresses for the + // prefix? + if !ndp.configs.AutoGenTempGlobalAddresses || prefix == header.IPv6LinkLocalPrefix.Subnet() { + return false + } + + if resetGenAttempts { + prefixState.generationAttempts = 0 + prefixState.maxGenerationAttempts = ndp.configs.AutoGenAddressConflictRetries + 1 + } + + // If we have already reached the maximum address generation attempts for the + // prefix, do not generate another address. + if prefixState.generationAttempts == prefixState.maxGenerationAttempts { + return false + } + + stableAddr := prefixState.stableAddr.addressEndpoint.AddressWithPrefix().Address + now := ndp.ep.protocol.stack.Clock().NowMonotonic() + + // As per RFC 4941 section 3.3 step 4, the valid lifetime of a temporary + // address is the lower of the valid lifetime of the stable address or the + // maximum temporary address valid lifetime. + vl := ndp.configs.MaxTempAddrValidLifetime + if prefixState.validUntil != nil { + if prefixVL := prefixState.validUntil.Sub(now); vl > prefixVL { + vl = prefixVL + } + } + + if vl <= 0 { + // Cannot create an address without a valid lifetime. + return false + } + + // As per RFC 4941 section 3.3 step 4, the preferred lifetime of a temporary + // address is the lower of the preferred lifetime of the stable address or the + // maximum temporary address preferred lifetime - the temporary address desync + // factor. + pl := ndp.configs.MaxTempAddrPreferredLifetime - ndp.temporaryAddressDesyncFactor + if prefixState.preferredUntil != nil { + if prefixPL := prefixState.preferredUntil.Sub(now); pl > prefixPL { + // Respect the preferred lifetime of the prefix, as per RFC 4941 section + // 3.3 step 4. + pl = prefixPL + } + } + + // As per RFC 4941 section 3.3 step 5, a temporary address is created only if + // the calculated preferred lifetime is greater than the advance regeneration + // duration. In particular, we MUST NOT create a temporary address with a zero + // Preferred Lifetime. + if pl <= ndp.configs.RegenAdvanceDuration { + return false + } + + // Attempt to generate a new address that is not already assigned to the IPv6 + // endpoint. + var generatedAddr tcpip.AddressWithPrefix + for i := 0; ; i++ { + // If we were unable to generate an address after the maximum SLAAC address + // local regeneration attempts, do nothing further. + if i == maxSLAACAddrLocalRegenAttempts { + return false + } + + generatedAddr = header.GenerateTempIPv6SLAACAddr(ndp.temporaryIIDHistory[:], stableAddr) + if !ndp.ep.hasPermanentAddressRLocked(generatedAddr.Address) { + break + } + } + + // As per RFC RFC 4941 section 3.3 step 5, we MUST NOT create a temporary + // address with a zero preferred lifetime. The checks above ensure this + // so we know the address is not deprecated. + addressEndpoint := ndp.addAndAcquireSLAACAddr(generatedAddr, true /* temporary */, stack.AddressLifetimes{ + Deprecated: false, + PreferredUntil: now.Add(pl), + ValidUntil: now.Add(vl), + }) + if addressEndpoint == nil { + return false + } + + state := tempSLAACAddrState{ + deprecationJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + prefixState, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: must have a slaacPrefixes entry for %s to deprecate temporary address %s", prefix, generatedAddr)) + } + + tempAddrState, ok := prefixState.tempAddrs[generatedAddr.Address] + if !ok { + panic(fmt.Sprintf("ndp: must have a tempAddr entry to deprecate temporary address %s", generatedAddr)) + } + + ndp.deprecateSLAACAddress(tempAddrState.addressEndpoint) + }), + invalidationJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + prefixState, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: must have a slaacPrefixes entry for %s to invalidate temporary address %s", prefix, generatedAddr)) + } + + tempAddrState, ok := prefixState.tempAddrs[generatedAddr.Address] + if !ok { + panic(fmt.Sprintf("ndp: must have a tempAddr entry to invalidate temporary address %s", generatedAddr)) + } + + ndp.invalidateTempSLAACAddr(prefixState.tempAddrs, generatedAddr.Address, tempAddrState) + }), + regenJob: tcpip.NewJob(ndp.ep.protocol.stack.Clock(), &ndp.ep.mu, func() { + prefixState, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: must have a slaacPrefixes entry for %s to regenerate temporary address after %s", prefix, generatedAddr)) + } + + tempAddrState, ok := prefixState.tempAddrs[generatedAddr.Address] + if !ok { + panic(fmt.Sprintf("ndp: must have a tempAddr entry to regenerate temporary address after %s", generatedAddr)) + } + + // If an address has already been regenerated for this address, don't + // regenerate another address. + if tempAddrState.regenerated { + return + } + + // Reset the generation attempts counter as we are starting the generation + // of a new address for the SLAAC prefix. + tempAddrState.regenerated = ndp.generateTempSLAACAddr(prefix, &prefixState, true /* resetGenAttempts */) + prefixState.tempAddrs[generatedAddr.Address] = tempAddrState + ndp.slaacPrefixes[prefix] = prefixState + }), + createdAt: now, + addressEndpoint: addressEndpoint, + } + + state.deprecationJob.Schedule(pl) + state.invalidationJob.Schedule(vl) + state.regenJob.Schedule(pl - ndp.configs.RegenAdvanceDuration) + + prefixState.generationAttempts++ + prefixState.tempAddrs[generatedAddr.Address] = state + + return true +} + +// regenerateTempSLAACAddr regenerates a temporary address for a SLAAC prefix. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) regenerateTempSLAACAddr(prefix tcpip.Subnet, resetGenAttempts bool) { + state, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: SLAAC prefix state not found to regenerate temporary address for %s", prefix)) + } + + ndp.generateTempSLAACAddr(prefix, &state, resetGenAttempts) + ndp.slaacPrefixes[prefix] = state +} + +// refreshSLAACPrefixLifetimes refreshes the lifetimes of a SLAAC prefix. +// +// pl is the new preferred lifetime. vl is the new valid lifetime. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) refreshSLAACPrefixLifetimes(prefix tcpip.Subnet, prefixState *slaacPrefixState, pl, vl time.Duration) { + // If prefix was preferred for some finite lifetime before, cancel the + // deprecation job so it can be reset. + prefixState.deprecationJob.Cancel() + + now := ndp.ep.protocol.stack.Clock().NowMonotonic() + + // Schedule the deprecation job if prefix has a finite preferred lifetime. + deprecated := pl == 0 + if pl < header.NDPInfiniteLifetime { + if !deprecated { + prefixState.deprecationJob.Schedule(pl) + } + t := now.Add(pl) + prefixState.preferredUntil = &t + } else { + prefixState.preferredUntil = nil + } + + // As per RFC 4862 section 5.5.3.e, update the valid lifetime for prefix: + // + // 1) If the received Valid Lifetime is greater than 2 hours or greater than + // RemainingLifetime, set the valid lifetime of the prefix to the + // advertised Valid Lifetime. + // + // 2) If RemainingLifetime is less than or equal to 2 hours, ignore the + // advertised Valid Lifetime. + // + // 3) Otherwise, reset the valid lifetime of the prefix to 2 hours. + + if vl >= header.NDPInfiniteLifetime { + // Handle the infinite valid lifetime separately as we do not schedule a + // job in this case. + prefixState.invalidationJob.Cancel() + prefixState.validUntil = nil + } else { + var effectiveVl time.Duration + var rl time.Duration + + // If the prefix was originally set to be valid forever, assume the + // remaining time to be the maximum possible value. + if prefixState.validUntil == nil { + rl = header.NDPInfiniteLifetime + } else { + rl = prefixState.validUntil.Sub(now) + } + + if vl > MinPrefixInformationValidLifetimeForUpdate || vl > rl { + effectiveVl = vl + } else if rl > MinPrefixInformationValidLifetimeForUpdate { + effectiveVl = MinPrefixInformationValidLifetimeForUpdate + } + + if effectiveVl != 0 { + prefixState.invalidationJob.Cancel() + prefixState.invalidationJob.Schedule(effectiveVl) + t := now.Add(effectiveVl) + prefixState.validUntil = &t + } + } + + // If the preferred lifetime is zero, then the prefix should be deprecated. + { + var preferredUntil tcpip.MonotonicTime + if !deprecated { + if prefixState.preferredUntil == nil { + preferredUntil = tcpip.MonotonicTimeInfinite() + } else { + preferredUntil = *prefixState.preferredUntil + } + } + validUntil := tcpip.MonotonicTimeInfinite() + if prefixState.validUntil != nil { + validUntil = *prefixState.validUntil + } + if addressEndpoint := prefixState.stableAddr.addressEndpoint; !addressEndpoint.Deprecated() && deprecated { + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnAutoGenAddressDeprecated(ndp.ep.nic.ID(), addressEndpoint.AddressWithPrefix()) + } + } + prefixState.stableAddr.addressEndpoint.SetLifetimes(stack.AddressLifetimes{ + Deprecated: deprecated, + PreferredUntil: preferredUntil, + ValidUntil: validUntil, + }) + } + + // If DAD is not yet complete on the stable address, there is no need to do + // work with temporary addresses. + if prefixState.stableAddr.addressEndpoint.GetKind() != stack.Permanent { + return + } + + // Note, we do not need to update the entries in the temporary address map + // after updating the jobs because the jobs are held as pointers. + var regenForAddr tcpip.Address + allAddressesRegenerated := true + for tempAddr, tempAddrState := range prefixState.tempAddrs { + // As per RFC 4941 section 3.3 step 4, the valid lifetime of a temporary + // address is the lower of the valid lifetime of the stable address or the + // maximum temporary address valid lifetime. Note, the valid lifetime of a + // temporary address is relative to the address's creation time. + validUntil := tempAddrState.createdAt.Add(ndp.configs.MaxTempAddrValidLifetime) + if prefixState.validUntil != nil && prefixState.validUntil.Before(validUntil) { + validUntil = *prefixState.validUntil + } + + // If the address is no longer valid, invalidate it immediately. Otherwise, + // reset the invalidation job. + newValidLifetime := validUntil.Sub(now) + if newValidLifetime <= 0 { + ndp.invalidateTempSLAACAddr(prefixState.tempAddrs, tempAddr, tempAddrState) + continue + } + tempAddrState.invalidationJob.Cancel() + tempAddrState.invalidationJob.Schedule(newValidLifetime) + + // As per RFC 4941 section 3.3 step 4, the preferred lifetime of a temporary + // address is the lower of the preferred lifetime of the stable address or + // the maximum temporary address preferred lifetime - the temporary address + // desync factor. Note, the preferred lifetime of a temporary address is + // relative to the address's creation time. + preferredUntil := tempAddrState.createdAt.Add(ndp.configs.MaxTempAddrPreferredLifetime - ndp.temporaryAddressDesyncFactor) + if prefixState.preferredUntil != nil && prefixState.preferredUntil.Before(preferredUntil) { + preferredUntil = *prefixState.preferredUntil + } + + // If the address is no longer preferred, deprecate it immediately. + // Otherwise, schedule the deprecation job again. + newPreferredLifetime := preferredUntil.Sub(now) + tempAddrState.deprecationJob.Cancel() + deprecated := newPreferredLifetime <= 0 + if !deprecated { + tempAddrState.deprecationJob.Schedule(newPreferredLifetime) + } + + if addressEndpoint := tempAddrState.addressEndpoint; !addressEndpoint.Deprecated() && deprecated { + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnAutoGenAddressDeprecated(ndp.ep.nic.ID(), addressEndpoint.AddressWithPrefix()) + } + } + tempAddrState.addressEndpoint.SetLifetimes(stack.AddressLifetimes{ + Deprecated: deprecated, + ValidUntil: validUntil, + PreferredUntil: preferredUntil, + }) + + tempAddrState.regenJob.Cancel() + if tempAddrState.regenerated { + } else { + allAddressesRegenerated = false + + if newPreferredLifetime <= ndp.configs.RegenAdvanceDuration { + // The new preferred lifetime is less than the advance regeneration + // duration so regenerate an address for this temporary address + // immediately after we finish iterating over the temporary addresses. + regenForAddr = tempAddr + } else { + tempAddrState.regenJob.Schedule(newPreferredLifetime - ndp.configs.RegenAdvanceDuration) + } + } + } + + // Generate a new temporary address if all of the existing temporary addresses + // have been regenerated, or we need to immediately regenerate an address + // due to an update in preferred lifetime. + // + // If each temporary address has already been regenerated, no new temporary + // address is generated. To ensure continuation of temporary SLAAC addresses, + // we manually try to regenerate an address here. + if regenForAddr.BitLen() != 0 || allAddressesRegenerated { + // Reset the generation attempts counter as we are starting the generation + // of a new address for the SLAAC prefix. + if state, ok := prefixState.tempAddrs[regenForAddr]; ndp.generateTempSLAACAddr(prefix, prefixState, true /* resetGenAttempts */) && ok { + state.regenerated = true + prefixState.tempAddrs[regenForAddr] = state + } + } +} + +// deprecateSLAACAddress marks the address as deprecated and notifies the NDP +// dispatcher that address has been deprecated. +// +// deprecateSLAACAddress does nothing if the address is already deprecated. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) deprecateSLAACAddress(addressEndpoint stack.AddressEndpoint) { + if addressEndpoint.Deprecated() { + return + } + + addressEndpoint.SetDeprecated(true) + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnAutoGenAddressDeprecated(ndp.ep.nic.ID(), addressEndpoint.AddressWithPrefix()) + } +} + +// invalidateSLAACPrefix invalidates a SLAAC prefix. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) invalidateSLAACPrefix(prefix tcpip.Subnet, state slaacPrefixState) { + ndp.cleanupSLAACPrefixResources(prefix, state) + + if addressEndpoint := state.stableAddr.addressEndpoint; addressEndpoint != nil { + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnAutoGenAddressInvalidated(ndp.ep.nic.ID(), addressEndpoint.AddressWithPrefix()) + } + + if err := ndp.ep.removePermanentEndpointInnerLocked(addressEndpoint, stack.AddressRemovalInvalidated, &stack.DADAborted{}); err != nil { + panic(fmt.Sprintf("ndp: error removing stable SLAAC address %s: %s", addressEndpoint.AddressWithPrefix(), err)) + } + } +} + +// cleanupSLAACAddrResourcesAndNotify cleans up an invalidated SLAAC address's +// resources. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) cleanupSLAACAddrResourcesAndNotify(addr tcpip.AddressWithPrefix, invalidatePrefix bool) { + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnAutoGenAddressInvalidated(ndp.ep.nic.ID(), addr) + } + + prefix := addr.Subnet() + state, ok := ndp.slaacPrefixes[prefix] + if !ok || state.stableAddr.addressEndpoint == nil || addr.Address != state.stableAddr.addressEndpoint.AddressWithPrefix().Address { + return + } + + if !invalidatePrefix { + // If the prefix is not being invalidated, disassociate the address from the + // prefix and do nothing further. + state.stableAddr.addressEndpoint.DecRef() + state.stableAddr.addressEndpoint = nil + ndp.slaacPrefixes[prefix] = state + return + } + + ndp.cleanupSLAACPrefixResources(prefix, state) +} + +// cleanupSLAACPrefixResources cleans up a SLAAC prefix's jobs and entry. +// +// Panics if the SLAAC prefix is not known. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) cleanupSLAACPrefixResources(prefix tcpip.Subnet, state slaacPrefixState) { + // Invalidate all temporary addresses. + for tempAddr, tempAddrState := range state.tempAddrs { + ndp.invalidateTempSLAACAddr(state.tempAddrs, tempAddr, tempAddrState) + } + + if state.stableAddr.addressEndpoint != nil { + state.stableAddr.addressEndpoint.DecRef() + state.stableAddr.addressEndpoint = nil + } + state.deprecationJob.Cancel() + state.invalidationJob.Cancel() + delete(ndp.slaacPrefixes, prefix) +} + +// invalidateTempSLAACAddr invalidates a temporary SLAAC address. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) invalidateTempSLAACAddr(tempAddrs map[tcpip.Address]tempSLAACAddrState, tempAddr tcpip.Address, tempAddrState tempSLAACAddrState) { + ndp.cleanupTempSLAACAddrResourcesAndNotifyInner(tempAddrs, tempAddr, tempAddrState) + + if err := ndp.ep.removePermanentEndpointInnerLocked(tempAddrState.addressEndpoint, stack.AddressRemovalInvalidated, &stack.DADAborted{}); err != nil { + panic(fmt.Sprintf("error removing temporary SLAAC address %s: %s", tempAddrState.addressEndpoint.AddressWithPrefix(), err)) + } +} + +// cleanupTempSLAACAddrResourcesAndNotify cleans up an invalidated temporary +// SLAAC address's resources from ndp and notifies the NDP dispatcher that the +// address was invalidated. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) cleanupTempSLAACAddrResourcesAndNotify(addr tcpip.AddressWithPrefix) { + prefix := addr.Subnet() + state, ok := ndp.slaacPrefixes[prefix] + if !ok { + panic(fmt.Sprintf("ndp: must have a slaacPrefixes entry to clean up temp addr %s resources", addr)) + } + + tempAddrState, ok := state.tempAddrs[addr.Address] + if !ok { + panic(fmt.Sprintf("ndp: must have a tempAddr entry to clean up temp addr %s resources", addr)) + } + + ndp.cleanupTempSLAACAddrResourcesAndNotifyInner(state.tempAddrs, addr.Address, tempAddrState) +} + +// cleanupTempSLAACAddrResourcesAndNotifyInner is like +// cleanupTempSLAACAddrResourcesAndNotify except it does not lookup the +// temporary address's state in ndp - it assumes the passed state is valid. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) cleanupTempSLAACAddrResourcesAndNotifyInner(tempAddrs map[tcpip.Address]tempSLAACAddrState, tempAddr tcpip.Address, tempAddrState tempSLAACAddrState) { + if ndpDisp := ndp.ep.protocol.options.NDPDisp; ndpDisp != nil { + ndpDisp.OnAutoGenAddressInvalidated(ndp.ep.nic.ID(), tempAddrState.addressEndpoint.AddressWithPrefix()) + } + + tempAddrState.addressEndpoint.DecRef() + tempAddrState.addressEndpoint = nil + tempAddrState.deprecationJob.Cancel() + tempAddrState.invalidationJob.Cancel() + tempAddrState.regenJob.Cancel() + delete(tempAddrs, tempAddr) +} + +// cleanupState cleans up ndp's state. +// +// This function invalidates all discovered on-link prefixes, discovered +// routers, and auto-generated addresses. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) cleanupState() { + for prefix, state := range ndp.slaacPrefixes { + ndp.invalidateSLAACPrefix(prefix, state) + } + + for prefix := range ndp.onLinkPrefixes { + ndp.invalidateOnLinkPrefix(prefix) + } + + if got := len(ndp.onLinkPrefixes); got != 0 { + panic(fmt.Sprintf("ndp: still have discovered on-link prefixes after cleaning up; found = %d", got)) + } + + for route := range ndp.offLinkRoutes { + ndp.invalidateOffLinkRoute(route) + } + + if got := len(ndp.offLinkRoutes); got != 0 { + panic(fmt.Sprintf("ndp: still have discovered off-link routes after cleaning up; found = %d", got)) + } + + ndp.dhcpv6Configuration = 0 +} + +// startSolicitingRouters starts soliciting routers, as per RFC 4861 section +// 6.3.7. If routers are already being solicited, this function does nothing. +// +// If ndp is not configured to handle Router Advertisements, routers will not +// be solicited as there is no point soliciting routers if we don't handle their +// advertisements. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) startSolicitingRouters() { + if ndp.rtrSolicitTimer.timer != nil { + // We are already soliciting routers. + return + } + + remaining := ndp.configs.MaxRtrSolicitations + if remaining == 0 { + return + } + + if !ndp.configs.HandleRAs.enabled(ndp.ep.Forwarding()) { + return + } + + // Calculate the random delay before sending our first RS, as per RFC + // 4861 section 6.3.7. + var delay time.Duration + if ndp.configs.MaxRtrSolicitationDelay > 0 { + delay = time.Duration(ndp.ep.protocol.stack.InsecureRNG().Int63n(int64(ndp.configs.MaxRtrSolicitationDelay))) + } + + // Protected by ndp.ep.mu. + done := false + + ndp.rtrSolicitTimer = timer{ + done: &done, + timer: ndp.ep.protocol.stack.Clock().AfterFunc(delay, func() { + // As per RFC 4861 section 4.1: + // + // IP Fields: + // Source Address + // An IP address assigned to the sending interface, or + // the unspecified address if no address is assigned + // to the sending interface. + localAddr := header.IPv6Any + if addressEndpoint := ndp.ep.AcquireOutgoingPrimaryAddress(header.IPv6AllRoutersLinkLocalMulticastAddress, tcpip.Address{} /* srcHint */, false); addressEndpoint != nil { + localAddr = addressEndpoint.AddressWithPrefix().Address + addressEndpoint.DecRef() + } + + // As per RFC 4861 section 4.1, an NDP RS SHOULD include the source + // link-layer address option if the source address of the NDP RS is + // specified. This option MUST NOT be included if the source address is + // unspecified. + // + // TODO(b/141011931): Validate a LinkEndpoint's link address (provided by + // LinkEndpoint.LinkAddress) before reaching this point. + var optsSerializer header.NDPOptionsSerializer + linkAddress := ndp.ep.nic.LinkAddress() + if localAddr != header.IPv6Any && header.IsValidUnicastEthernetAddress(linkAddress) { + optsSerializer = header.NDPOptionsSerializer{ + header.NDPSourceLinkLayerAddressOption(linkAddress), + } + } + payloadSize := header.ICMPv6HeaderSize + header.NDPRSMinimumSize + optsSerializer.Length() + icmpView := buffer.NewView(payloadSize) + icmpView.Grow(payloadSize) + icmpData := header.ICMPv6(icmpView.AsSlice()) + icmpData.SetType(header.ICMPv6RouterSolicit) + rs := header.NDPRouterSolicit(icmpData.MessageBody()) + rs.Options().Serialize(optsSerializer) + icmpData.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmpData, + Src: localAddr, + Dst: header.IPv6AllRoutersLinkLocalMulticastAddress, + })) + + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(ndp.ep.MaxHeaderLength()), + Payload: buffer.MakeWithView(icmpView), + }) + defer pkt.DecRef() + + sent := ndp.ep.stats.icmp.packetsSent + if err := addIPHeader(localAddr, header.IPv6AllRoutersLinkLocalMulticastAddress, pkt, stack.NetworkHeaderParams{ + Protocol: header.ICMPv6ProtocolNumber, + TTL: header.NDPHopLimit, + }, nil /* extensionHeaders */); err != nil { + panic(fmt.Sprintf("failed to add IP header: %s", err)) + } + + if err := ndp.ep.nic.WritePacketToRemote(header.EthernetAddressFromMulticastIPv6Address(header.IPv6AllRoutersLinkLocalMulticastAddress), pkt); err != nil { + sent.dropped.Increment() + // Don't send any more messages if we had an error. + remaining = 0 + } else { + sent.routerSolicit.Increment() + remaining-- + } + + ndp.ep.mu.Lock() + defer ndp.ep.mu.Unlock() + + if done { + // Router solicitation was stopped. + return + } + + if remaining == 0 { + // We are done soliciting routers. + ndp.stopSolicitingRouters() + return + } + + ndp.rtrSolicitTimer.timer.Reset(ndp.configs.RtrSolicitationInterval) + }), + } +} + +// forwardingChanged handles a change in forwarding configuration. +// +// If transitioning to a host, router solicitation will be started. Otherwise, +// router solicitation will be stopped if NDP is not configured to handle RAs +// as a router. +// +// Precondition: ndp.ep.mu must be locked. +func (ndp *ndpState) forwardingChanged(forwarding bool) { + if forwarding { + if ndp.configs.HandleRAs.enabled(forwarding) { + return + } + + ndp.stopSolicitingRouters() + return + } + + // Solicit routers when transitioning to a host. + // + // If the endpoint is not currently enabled, routers will be solicited when + // the endpoint becomes enabled (if it is still a host). + if ndp.ep.Enabled() { + ndp.startSolicitingRouters() + } +} + +// stopSolicitingRouters stops soliciting routers. If routers are not currently +// being solicited, this function does nothing. +// +// The IPv6 endpoint that ndp belongs to MUST be locked. +func (ndp *ndpState) stopSolicitingRouters() { + if ndp.rtrSolicitTimer.timer == nil { + // Nothing to do. + return + } + + ndp.rtrSolicitTimer.timer.Stop() + *ndp.rtrSolicitTimer.done = true + ndp.rtrSolicitTimer = timer{} +} + +func (ndp *ndpState) init(ep *endpoint, dadOptions ip.DADOptions) { + if ndp.offLinkRoutes != nil { + panic("attempted to initialize NDP state twice") + } + + ndp.ep = ep + ndp.configs = ep.protocol.options.NDPConfigs + ndp.dad.Init(&ndp.ep.mu, ep.protocol.options.DADConfigs, dadOptions) + ndp.offLinkRoutes = make(map[offLinkRoute]offLinkRouteState) + ndp.onLinkPrefixes = make(map[tcpip.Subnet]onLinkPrefixState) + ndp.slaacPrefixes = make(map[tcpip.Subnet]slaacPrefixState) + + header.InitialTempIID(ndp.temporaryIIDHistory[:], ndp.ep.protocol.options.TempIIDSeed, ndp.ep.nic.ID()) + ndp.temporaryAddressDesyncFactor = time.Duration(ep.protocol.stack.InsecureRNG().Int63n(int64(MaxDesyncFactor))) +} + +func (ndp *ndpState) SendDADMessage(addr tcpip.Address, nonce []byte) tcpip.Error { + snmc := header.SolicitedNodeAddr(addr) + return ndp.ep.sendNDPNS(header.IPv6Any, snmc, addr, header.EthernetAddressFromMulticastIPv6Address(snmc), header.NDPOptionsSerializer{ + header.NDPNonceOption(nonce), + }) +} + +func (e *endpoint) sendNDPNS(srcAddr, dstAddr, targetAddr tcpip.Address, remoteLinkAddr tcpip.LinkAddress, opts header.NDPOptionsSerializer) tcpip.Error { + icmpView := buffer.NewView(header.ICMPv6NeighborSolicitMinimumSize + opts.Length()) + icmpView.Grow(header.ICMPv6NeighborSolicitMinimumSize + opts.Length()) + icmp := header.ICMPv6(icmpView.AsSlice()) + icmp.SetType(header.ICMPv6NeighborSolicit) + ns := header.NDPNeighborSolicit(icmp.MessageBody()) + ns.SetTargetAddress(targetAddr) + ns.Options().Serialize(opts) + icmp.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmp, + Src: srcAddr, + Dst: dstAddr, + })) + + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(e.MaxHeaderLength()), + Payload: buffer.MakeWithView(icmpView), + }) + defer pkt.DecRef() + + if err := addIPHeader(srcAddr, dstAddr, pkt, stack.NetworkHeaderParams{ + Protocol: header.ICMPv6ProtocolNumber, + TTL: header.NDPHopLimit, + }, nil /* extensionHeaders */); err != nil { + panic(fmt.Sprintf("failed to add IP header: %s", err)) + } + + sent := e.stats.icmp.packetsSent + err := e.nic.WritePacketToRemote(remoteLinkAddr, pkt) + if err != nil { + sent.dropped.Increment() + } else { + sent.neighborSolicit.Increment() + } + return err +} diff --git a/pkg/tcpip/network/ipv6/stats.go b/pkg/tcpip/network/ipv6/stats.go new file mode 100644 index 0000000..a93b41e --- /dev/null +++ b/pkg/tcpip/network/ipv6/stats.go @@ -0,0 +1,145 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipv6 + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/network/internal/ip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +var _ stack.IPNetworkEndpointStats = (*Stats)(nil) + +// Stats holds statistics related to the IPv6 protocol family. +// +// +stateify savable +type Stats struct { + // IP holds IPv6 statistics. + IP tcpip.IPStats + + // ICMP holds ICMPv6 statistics. + ICMP tcpip.ICMPv6Stats + + // UnhandledRouterAdvertisements is the number of Router Advertisements that + // were observed but not handled. + UnhandledRouterAdvertisements *tcpip.StatCounter +} + +// IsNetworkEndpointStats implements stack.NetworkEndpointStats. +func (*Stats) IsNetworkEndpointStats() {} + +// IPStats implements stack.IPNetworkEndointStats +func (s *Stats) IPStats() *tcpip.IPStats { + return &s.IP +} + +// +stateify savable +type sharedStats struct { + localStats Stats + ip ip.MultiCounterIPStats + icmp multiCounterICMPv6Stats +} + +// LINT.IfChange(multiCounterICMPv6PacketStats) + +// +stateify savable +type multiCounterICMPv6PacketStats struct { + echoRequest tcpip.MultiCounterStat + echoReply tcpip.MultiCounterStat + dstUnreachable tcpip.MultiCounterStat + packetTooBig tcpip.MultiCounterStat + timeExceeded tcpip.MultiCounterStat + paramProblem tcpip.MultiCounterStat + routerSolicit tcpip.MultiCounterStat + routerAdvert tcpip.MultiCounterStat + neighborSolicit tcpip.MultiCounterStat + neighborAdvert tcpip.MultiCounterStat + redirectMsg tcpip.MultiCounterStat + multicastListenerQuery tcpip.MultiCounterStat + multicastListenerReport tcpip.MultiCounterStat + multicastListenerReportV2 tcpip.MultiCounterStat + multicastListenerDone tcpip.MultiCounterStat +} + +func (m *multiCounterICMPv6PacketStats) init(a, b *tcpip.ICMPv6PacketStats) { + m.echoRequest.Init(a.EchoRequest, b.EchoRequest) + m.echoReply.Init(a.EchoReply, b.EchoReply) + m.dstUnreachable.Init(a.DstUnreachable, b.DstUnreachable) + m.packetTooBig.Init(a.PacketTooBig, b.PacketTooBig) + m.timeExceeded.Init(a.TimeExceeded, b.TimeExceeded) + m.paramProblem.Init(a.ParamProblem, b.ParamProblem) + m.routerSolicit.Init(a.RouterSolicit, b.RouterSolicit) + m.routerAdvert.Init(a.RouterAdvert, b.RouterAdvert) + m.neighborSolicit.Init(a.NeighborSolicit, b.NeighborSolicit) + m.neighborAdvert.Init(a.NeighborAdvert, b.NeighborAdvert) + m.redirectMsg.Init(a.RedirectMsg, b.RedirectMsg) + m.multicastListenerQuery.Init(a.MulticastListenerQuery, b.MulticastListenerQuery) + m.multicastListenerReport.Init(a.MulticastListenerReport, b.MulticastListenerReport) + m.multicastListenerReportV2.Init(a.MulticastListenerReportV2, b.MulticastListenerReportV2) + m.multicastListenerDone.Init(a.MulticastListenerDone, b.MulticastListenerDone) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv6PacketStats) + +// LINT.IfChange(multiCounterICMPv6SentPacketStats) + +// +stateify savable +type multiCounterICMPv6SentPacketStats struct { + multiCounterICMPv6PacketStats + dropped tcpip.MultiCounterStat + rateLimited tcpip.MultiCounterStat +} + +func (m *multiCounterICMPv6SentPacketStats) init(a, b *tcpip.ICMPv6SentPacketStats) { + m.multiCounterICMPv6PacketStats.init(&a.ICMPv6PacketStats, &b.ICMPv6PacketStats) + m.dropped.Init(a.Dropped, b.Dropped) + m.rateLimited.Init(a.RateLimited, b.RateLimited) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv6SentPacketStats) + +// LINT.IfChange(multiCounterICMPv6ReceivedPacketStats) + +// +stateify savable +type multiCounterICMPv6ReceivedPacketStats struct { + multiCounterICMPv6PacketStats + unrecognized tcpip.MultiCounterStat + invalid tcpip.MultiCounterStat + routerOnlyPacketsDroppedByHost tcpip.MultiCounterStat +} + +func (m *multiCounterICMPv6ReceivedPacketStats) init(a, b *tcpip.ICMPv6ReceivedPacketStats) { + m.multiCounterICMPv6PacketStats.init(&a.ICMPv6PacketStats, &b.ICMPv6PacketStats) + m.unrecognized.Init(a.Unrecognized, b.Unrecognized) + m.invalid.Init(a.Invalid, b.Invalid) + m.routerOnlyPacketsDroppedByHost.Init(a.RouterOnlyPacketsDroppedByHost, b.RouterOnlyPacketsDroppedByHost) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv6ReceivedPacketStats) + +// LINT.IfChange(multiCounterICMPv6Stats) + +// +stateify savable +type multiCounterICMPv6Stats struct { + packetsSent multiCounterICMPv6SentPacketStats + packetsReceived multiCounterICMPv6ReceivedPacketStats +} + +func (m *multiCounterICMPv6Stats) init(a, b *tcpip.ICMPv6Stats) { + m.packetsSent.init(&a.PacketsSent, &b.PacketsSent) + m.packetsReceived.init(&a.PacketsReceived, &b.PacketsReceived) +} + +// LINT.ThenChange(../../tcpip.go:ICMPv6Stats) diff --git a/pkg/tcpip/ports/flags.go b/pkg/tcpip/ports/flags.go new file mode 100644 index 0000000..251b82e --- /dev/null +++ b/pkg/tcpip/ports/flags.go @@ -0,0 +1,152 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ports + +// Flags represents the type of port reservation. +// +// +stateify savable +type Flags struct { + // MostRecent represents UDP SO_REUSEADDR. + MostRecent bool + + // LoadBalanced indicates SO_REUSEPORT. + // + // LoadBalanced takes precedence over MostRecent. + LoadBalanced bool + + // TupleOnly represents TCP SO_REUSEADDR. + TupleOnly bool +} + +// Bits converts the Flags to their bitset form. +func (f Flags) Bits() BitFlags { + var rf BitFlags + if f.MostRecent { + rf |= MostRecentFlag + } + if f.LoadBalanced { + rf |= LoadBalancedFlag + } + if f.TupleOnly { + rf |= TupleOnlyFlag + } + return rf +} + +// Effective returns the effective behavior of a flag config. +func (f Flags) Effective() Flags { + e := f + if e.LoadBalanced && e.MostRecent { + e.MostRecent = false + } + return e +} + +// BitFlags is a bitset representation of Flags. +type BitFlags uint32 + +const ( + // MostRecentFlag represents Flags.MostRecent. + MostRecentFlag BitFlags = 1 << iota + + // LoadBalancedFlag represents Flags.LoadBalanced. + LoadBalancedFlag + + // TupleOnlyFlag represents Flags.TupleOnly. + TupleOnlyFlag + + // nextFlag is the value that the next added flag will have. + // + // It is used to calculate FlagMask below. It is also the number of + // valid flag states. + nextFlag + + // FlagMask is a bit mask for BitFlags. + FlagMask = nextFlag - 1 + + // MultiBindFlagMask contains the flags that allow binding the same + // tuple multiple times. + MultiBindFlagMask = MostRecentFlag | LoadBalancedFlag +) + +// ToFlags converts the bitset into a Flags struct. +func (f BitFlags) ToFlags() Flags { + return Flags{ + MostRecent: f&MostRecentFlag != 0, + LoadBalanced: f&LoadBalancedFlag != 0, + TupleOnly: f&TupleOnlyFlag != 0, + } +} + +// FlagCounter counts how many references each flag combination has. +// +// +stateify savable +type FlagCounter struct { + // refs stores the count for each possible flag combination, (0 though + // FlagMask). + refs [nextFlag]int +} + +// AddRef increases the reference count for a specific flag combination. +func (c *FlagCounter) AddRef(flags BitFlags) { + c.refs[flags]++ +} + +// DropRef decreases the reference count for a specific flag combination. +func (c *FlagCounter) DropRef(flags BitFlags) { + c.refs[flags]-- +} + +// TotalRefs calculates the total number of references for all flag +// combinations. +func (c FlagCounter) TotalRefs() int { + var total int + for _, r := range c.refs { + total += r + } + return total +} + +// FlagRefs returns the number of references with all specified flags. +func (c FlagCounter) FlagRefs(flags BitFlags) int { + var total int + for i, r := range c.refs { + if BitFlags(i)&flags == flags { + total += r + } + } + return total +} + +// AllRefsHave returns if all references have all specified flags. +func (c FlagCounter) AllRefsHave(flags BitFlags) bool { + for i, r := range c.refs { + if BitFlags(i)&flags != flags && r > 0 { + return false + } + } + return true +} + +// SharedFlags returns the set of flags shared by all references. +func (c FlagCounter) SharedFlags() BitFlags { + intersection := FlagMask + for i, r := range c.refs { + if r > 0 { + intersection &= BitFlags(i) + } + } + return intersection +} diff --git a/pkg/tcpip/ports/ports.go b/pkg/tcpip/ports/ports.go new file mode 100644 index 0000000..7064a0e --- /dev/null +++ b/pkg/tcpip/ports/ports.go @@ -0,0 +1,496 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ports provides PortManager that manages allocating, reserving and +// releasing ports. +package ports + +import ( + "math" + + "github.com/sagernet/gvisor/pkg/rand" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +const ( + firstEphemeral = 16000 +) + +var anyIPAddress = tcpip.Address{} + +// Reservation describes a port reservation. +type Reservation struct { + // Networks is a list of network protocols to which the reservation + // applies. Can be IPv4, IPv6, or both. + Networks []tcpip.NetworkProtocolNumber + + // Transport is the transport protocol to which the reservation applies. + Transport tcpip.TransportProtocolNumber + + // Addr is the address of the local endpoint. + Addr tcpip.Address + + // Port is the local port number. + Port uint16 + + // Flags describe features of the reservation. + Flags Flags + + // BindToDevice is the NIC to which the reservation applies. + BindToDevice tcpip.NICID + + // Dest is the destination address. + Dest tcpip.FullAddress +} + +func (rs Reservation) dst() destination { + return destination{ + rs.Dest.Addr, + rs.Dest.Port, + } +} + +// +stateify savable +type portDescriptor struct { + network tcpip.NetworkProtocolNumber + transport tcpip.TransportProtocolNumber + port uint16 +} + +// +stateify savable +type destination struct { + addr tcpip.Address + port uint16 +} + +// destToCounter maps each destination to the FlagCounter that represents +// endpoints to that destination. +// +// destToCounter is never empty. When it has no elements, it is removed from +// the map that references it. +type destToCounter map[destination]FlagCounter + +// intersectionFlags calculates the intersection of flag bit values which affect +// the specified destination. +// +// If no destinations are present, all flag values are returned as there are no +// entries to limit possible flag values of a new entry. +// +// In addition to the intersection, the number of intersecting refs is +// returned. +func (dc destToCounter) intersectionFlags(res Reservation) (BitFlags, int) { + intersection := FlagMask + var count int + + for dest, counter := range dc { + if dest == res.dst() { + intersection &= counter.SharedFlags() + count++ + continue + } + // Wildcard destinations affect all destinations for TupleOnly. + if dest.addr == anyIPAddress || res.Dest.Addr == anyIPAddress { + // Only bitwise and the TupleOnlyFlag. + intersection &= (^TupleOnlyFlag) | counter.SharedFlags() + count++ + } + } + + return intersection, count +} + +// deviceToDest maps NICs to destinations for which there are port reservations. +// +// deviceToDest is never empty. When it has no elements, it is removed from the +// map that references it. +type deviceToDest map[tcpip.NICID]destToCounter + +// isAvailable checks whether binding is possible by device. If not binding to +// a device, check against all FlagCounters. If binding to a specific device, +// check against the unspecified device and the provided device. +// +// If either of the port reuse flags is enabled on any of the nodes, all nodes +// sharing a port must share at least one reuse flag. This matches Linux's +// behavior. +func (dd deviceToDest) isAvailable(res Reservation, portSpecified bool) bool { + flagBits := res.Flags.Bits() + if res.BindToDevice == 0 { + intersection := FlagMask + for _, dest := range dd { + flags, count := dest.intersectionFlags(res) + if count == 0 { + continue + } + intersection &= flags + if intersection&flagBits == 0 { + // Can't bind because the (addr,port) was + // previously bound without reuse. + return false + } + } + if !portSpecified && res.Transport == header.TCPProtocolNumber { + return false + } + return true + } + + intersection := FlagMask + + if dests, ok := dd[0]; ok { + var count int + intersection, count = dests.intersectionFlags(res) + if count > 0 { + if intersection&flagBits == 0 { + return false + } + if !portSpecified && res.Transport == header.TCPProtocolNumber { + return false + } + } + } + + if dests, ok := dd[res.BindToDevice]; ok { + flags, count := dests.intersectionFlags(res) + intersection &= flags + if count > 0 { + if intersection&flagBits == 0 { + return false + } + if !portSpecified && res.Transport == header.TCPProtocolNumber { + return false + } + } + } + + return true +} + +// addrToDevice maps IP addresses to NICs that have port reservations. +type addrToDevice map[tcpip.Address]deviceToDest + +// isAvailable checks whether an IP address is available to bind to. If the +// address is the "any" address, check all other addresses. Otherwise, just +// check against the "any" address and the provided address. +func (ad addrToDevice) isAvailable(res Reservation, portSpecified bool) bool { + if res.Addr == anyIPAddress { + // If binding to the "any" address then check that there are no + // conflicts with all addresses. + for _, devices := range ad { + if !devices.isAvailable(res, portSpecified) { + return false + } + } + return true + } + + // Check that there is no conflict with the "any" address. + if devices, ok := ad[anyIPAddress]; ok { + if !devices.isAvailable(res, portSpecified) { + return false + } + } + + // Check that this is no conflict with the provided address. + if devices, ok := ad[res.Addr]; ok { + if !devices.isAvailable(res, portSpecified) { + return false + } + } + + return true +} + +// PortManager manages allocating, reserving and releasing ports. +// +// +stateify savable +type PortManager struct { + // mu protects allocatedPorts. + // LOCK ORDERING: mu > ephemeralMu. + mu sync.RWMutex `state:"nosave"` + // allocatedPorts is a nesting of maps that ultimately map Reservations + // to FlagCounters describing whether the Reservation is valid and can + // be reused. + allocatedPorts map[portDescriptor]addrToDevice + + // ephemeralMu protects firstEphemeral and numEphemeral. + ephemeralMu sync.RWMutex `state:"nosave"` + firstEphemeral uint16 + numEphemeral uint16 +} + +// NewPortManager creates new PortManager. +func NewPortManager() *PortManager { + return &PortManager{ + allocatedPorts: make(map[portDescriptor]addrToDevice), + firstEphemeral: firstEphemeral, + numEphemeral: math.MaxUint16 - firstEphemeral + 1, + } +} + +// PortTester indicates whether the passed in port is suitable. Returning an +// error causes the function to which the PortTester is passed to return that +// error. +type PortTester func(port uint16) (good bool, err tcpip.Error) + +// PickEphemeralPort randomly chooses a starting point and iterates over all +// possible ephemeral ports, allowing the caller to decide whether a given port +// is suitable for its needs, and stopping when a port is found or an error +// occurs. +func (pm *PortManager) PickEphemeralPort(rng rand.RNG, testPort PortTester) (port uint16, err tcpip.Error) { + pm.ephemeralMu.RLock() + firstEphemeral := pm.firstEphemeral + numEphemeral := pm.numEphemeral + pm.ephemeralMu.RUnlock() + + return pickEphemeralPort(rng.Uint32(), firstEphemeral, numEphemeral, testPort) +} + +// pickEphemeralPort starts at the offset specified from the FirstEphemeral port +// and iterates over the number of ports specified by count and allows the +// caller to decide whether a given port is suitable for its needs, and stopping +// when a port is found or an error occurs. +func pickEphemeralPort(offset uint32, first, count uint16, testPort PortTester) (port uint16, err tcpip.Error) { + // This implements Algorithm 1 as per RFC 6056 Section 3.3.1. + for i := uint32(0); i < uint32(count); i++ { + port := uint16(uint32(first) + (offset+i)%uint32(count)) + ok, err := testPort(port) + if err != nil { + return 0, err + } + + if ok { + return port, nil + } + } + + return 0, &tcpip.ErrNoPortAvailable{} +} + +// ReservePort marks a port/IP combination as reserved so that it cannot be +// reserved by another endpoint. If port is zero, ReservePort will search for +// an unreserved ephemeral port and reserve it, returning its value in the +// "port" return value. +// +// An optional PortTester can be passed in which if provided will be used to +// test if the picked port can be used. The function should return true if the +// port is safe to use, false otherwise. +func (pm *PortManager) ReservePort(rng rand.RNG, res Reservation, testPort PortTester) (reservedPort uint16, err tcpip.Error) { + pm.mu.Lock() + defer pm.mu.Unlock() + + // If a port is specified, just try to reserve it for all network + // protocols. + if res.Port != 0 { + if !pm.reserveSpecificPortLocked(res, true /* portSpecified */) { + return 0, &tcpip.ErrPortInUse{} + } + if testPort != nil { + ok, err := testPort(res.Port) + if err != nil { + pm.releasePortLocked(res) + return 0, err + } + if !ok { + pm.releasePortLocked(res) + return 0, &tcpip.ErrPortInUse{} + } + } + return res.Port, nil + } + + // A port wasn't specified, so try to find one. + return pm.PickEphemeralPort(rng, func(p uint16) (bool, tcpip.Error) { + res.Port = p + if !pm.reserveSpecificPortLocked(res, false /* portSpecified */) { + return false, nil + } + if testPort != nil { + ok, err := testPort(p) + if err != nil { + pm.releasePortLocked(res) + return false, err + } + if !ok { + pm.releasePortLocked(res) + return false, nil + } + } + return true, nil + }) +} + +// reserveSpecificPortLocked tries to reserve the given port on all given +// protocols. +func (pm *PortManager) reserveSpecificPortLocked(res Reservation, portSpecified bool) bool { + // Make sure the port is available. + for _, network := range res.Networks { + desc := portDescriptor{network, res.Transport, res.Port} + if addrs, ok := pm.allocatedPorts[desc]; ok { + if !addrs.isAvailable(res, portSpecified) { + return false + } + } + } + + // Reserve port on all network protocols. + flagBits := res.Flags.Bits() + dst := res.dst() + for _, network := range res.Networks { + desc := portDescriptor{network, res.Transport, res.Port} + addrToDev, ok := pm.allocatedPorts[desc] + if !ok { + addrToDev = make(addrToDevice) + pm.allocatedPorts[desc] = addrToDev + } + devToDest, ok := addrToDev[res.Addr] + if !ok { + devToDest = make(deviceToDest) + addrToDev[res.Addr] = devToDest + } + destToCntr := devToDest[res.BindToDevice] + if destToCntr == nil { + destToCntr = make(destToCounter) + } + counter := destToCntr[dst] + counter.AddRef(flagBits) + destToCntr[dst] = counter + devToDest[res.BindToDevice] = destToCntr + } + + return true +} + +// ReserveTuple adds a port reservation for the tuple on all given protocol. +func (pm *PortManager) ReserveTuple(res Reservation) bool { + flagBits := res.Flags.Bits() + dst := res.dst() + + pm.mu.Lock() + defer pm.mu.Unlock() + + // It is easier to undo the entire reservation, so if we find that the + // tuple can't be fully added, finish and undo the whole thing. + undo := false + + // Reserve port on all network protocols. + for _, network := range res.Networks { + desc := portDescriptor{network, res.Transport, res.Port} + addrToDev, ok := pm.allocatedPorts[desc] + if !ok { + addrToDev = make(addrToDevice) + pm.allocatedPorts[desc] = addrToDev + } + devToDest, ok := addrToDev[res.Addr] + if !ok { + devToDest = make(deviceToDest) + addrToDev[res.Addr] = devToDest + } + destToCntr := devToDest[res.BindToDevice] + if destToCntr == nil { + destToCntr = make(destToCounter) + } + + counter := destToCntr[dst] + if counter.TotalRefs() != 0 && counter.SharedFlags()&flagBits == 0 { + // Tuple already exists. + undo = true + } + counter.AddRef(flagBits) + destToCntr[dst] = counter + devToDest[res.BindToDevice] = destToCntr + } + + if undo { + // releasePortLocked decrements the counts (rather than setting + // them to zero), so it will undo the incorrect incrementing + // above. + pm.releasePortLocked(res) + return false + } + + return true +} + +// ReleasePort releases the reservation on a port/IP combination so that it can +// be reserved by other endpoints. +func (pm *PortManager) ReleasePort(res Reservation) { + pm.mu.Lock() + defer pm.mu.Unlock() + + pm.releasePortLocked(res) +} + +func (pm *PortManager) releasePortLocked(res Reservation) { + dst := res.dst() + for _, network := range res.Networks { + desc := portDescriptor{network, res.Transport, res.Port} + addrToDev, ok := pm.allocatedPorts[desc] + if !ok { + continue + } + devToDest, ok := addrToDev[res.Addr] + if !ok { + continue + } + destToCounter, ok := devToDest[res.BindToDevice] + if !ok { + continue + } + counter, ok := destToCounter[dst] + if !ok { + continue + } + counter.DropRef(res.Flags.Bits()) + if counter.TotalRefs() > 0 { + destToCounter[dst] = counter + continue + } + delete(destToCounter, dst) + if len(destToCounter) > 0 { + continue + } + delete(devToDest, res.BindToDevice) + if len(devToDest) > 0 { + continue + } + delete(addrToDev, res.Addr) + if len(addrToDev) > 0 { + continue + } + delete(pm.allocatedPorts, desc) + } +} + +// PortRange returns the UDP and TCP inclusive range of ephemeral ports used in +// both IPv4 and IPv6. +func (pm *PortManager) PortRange() (uint16, uint16) { + pm.ephemeralMu.RLock() + defer pm.ephemeralMu.RUnlock() + return pm.firstEphemeral, pm.firstEphemeral + pm.numEphemeral - 1 +} + +// SetPortRange sets the UDP and TCP IPv4 and IPv6 ephemeral port range +// (inclusive). +func (pm *PortManager) SetPortRange(start uint16, end uint16) tcpip.Error { + if start > end { + return &tcpip.ErrInvalidPortRange{} + } + pm.ephemeralMu.Lock() + defer pm.ephemeralMu.Unlock() + pm.firstEphemeral = start + pm.numEphemeral = end - start + 1 + return nil +} diff --git a/pkg/tcpip/ports/ports_state_autogen.go b/pkg/tcpip/ports/ports_state_autogen.go new file mode 100644 index 0000000..a901078 --- /dev/null +++ b/pkg/tcpip/ports/ports_state_autogen.go @@ -0,0 +1,163 @@ +// automatically generated by stateify. + +package ports + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (f *Flags) StateTypeName() string { + return "pkg/tcpip/ports.Flags" +} + +func (f *Flags) StateFields() []string { + return []string{ + "MostRecent", + "LoadBalanced", + "TupleOnly", + } +} + +func (f *Flags) beforeSave() {} + +// +checklocksignore +func (f *Flags) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.MostRecent) + stateSinkObject.Save(1, &f.LoadBalanced) + stateSinkObject.Save(2, &f.TupleOnly) +} + +func (f *Flags) afterLoad(context.Context) {} + +// +checklocksignore +func (f *Flags) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.MostRecent) + stateSourceObject.Load(1, &f.LoadBalanced) + stateSourceObject.Load(2, &f.TupleOnly) +} + +func (c *FlagCounter) StateTypeName() string { + return "pkg/tcpip/ports.FlagCounter" +} + +func (c *FlagCounter) StateFields() []string { + return []string{ + "refs", + } +} + +func (c *FlagCounter) beforeSave() {} + +// +checklocksignore +func (c *FlagCounter) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.refs) +} + +func (c *FlagCounter) afterLoad(context.Context) {} + +// +checklocksignore +func (c *FlagCounter) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.refs) +} + +func (p *portDescriptor) StateTypeName() string { + return "pkg/tcpip/ports.portDescriptor" +} + +func (p *portDescriptor) StateFields() []string { + return []string{ + "network", + "transport", + "port", + } +} + +func (p *portDescriptor) beforeSave() {} + +// +checklocksignore +func (p *portDescriptor) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.network) + stateSinkObject.Save(1, &p.transport) + stateSinkObject.Save(2, &p.port) +} + +func (p *portDescriptor) afterLoad(context.Context) {} + +// +checklocksignore +func (p *portDescriptor) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.network) + stateSourceObject.Load(1, &p.transport) + stateSourceObject.Load(2, &p.port) +} + +func (d *destination) StateTypeName() string { + return "pkg/tcpip/ports.destination" +} + +func (d *destination) StateFields() []string { + return []string{ + "addr", + "port", + } +} + +func (d *destination) beforeSave() {} + +// +checklocksignore +func (d *destination) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.addr) + stateSinkObject.Save(1, &d.port) +} + +func (d *destination) afterLoad(context.Context) {} + +// +checklocksignore +func (d *destination) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.addr) + stateSourceObject.Load(1, &d.port) +} + +func (pm *PortManager) StateTypeName() string { + return "pkg/tcpip/ports.PortManager" +} + +func (pm *PortManager) StateFields() []string { + return []string{ + "allocatedPorts", + "firstEphemeral", + "numEphemeral", + } +} + +func (pm *PortManager) beforeSave() {} + +// +checklocksignore +func (pm *PortManager) StateSave(stateSinkObject state.Sink) { + pm.beforeSave() + stateSinkObject.Save(0, &pm.allocatedPorts) + stateSinkObject.Save(1, &pm.firstEphemeral) + stateSinkObject.Save(2, &pm.numEphemeral) +} + +func (pm *PortManager) afterLoad(context.Context) {} + +// +checklocksignore +func (pm *PortManager) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &pm.allocatedPorts) + stateSourceObject.Load(1, &pm.firstEphemeral) + stateSourceObject.Load(2, &pm.numEphemeral) +} + +func init() { + state.Register((*Flags)(nil)) + state.Register((*FlagCounter)(nil)) + state.Register((*portDescriptor)(nil)) + state.Register((*destination)(nil)) + state.Register((*PortManager)(nil)) +} diff --git a/pkg/tcpip/route_list.go b/pkg/tcpip/route_list.go new file mode 100644 index 0000000..ddc7c23 --- /dev/null +++ b/pkg/tcpip/route_list.go @@ -0,0 +1,239 @@ +package tcpip + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type RouteElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (RouteElementMapper) linkerFor(elem *Route) *Route { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type RouteList struct { + head *Route + tail *Route +} + +// Reset resets list l to the empty state. +func (l *RouteList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *RouteList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *RouteList) Front() *Route { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *RouteList) Back() *Route { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *RouteList) Len() (count int) { + for e := l.Front(); e != nil; e = (RouteElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *RouteList) PushFront(e *Route) { + linker := RouteElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + RouteElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *RouteList) PushFrontList(m *RouteList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + RouteElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + RouteElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *RouteList) PushBack(e *Route) { + linker := RouteElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + RouteElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *RouteList) PushBackList(m *RouteList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + RouteElementMapper{}.linkerFor(l.tail).SetNext(m.head) + RouteElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *RouteList) InsertAfter(b, e *Route) { + bLinker := RouteElementMapper{}.linkerFor(b) + eLinker := RouteElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + RouteElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *RouteList) InsertBefore(a, e *Route) { + aLinker := RouteElementMapper{}.linkerFor(a) + eLinker := RouteElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + RouteElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *RouteList) Remove(e *Route) { + linker := RouteElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + RouteElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + RouteElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type RouteEntry struct { + next *Route + prev *Route +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *RouteEntry) Next() *Route { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *RouteEntry) Prev() *Route { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *RouteEntry) SetNext(elem *Route) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *RouteEntry) SetPrev(elem *Route) { + e.prev = elem +} diff --git a/pkg/tcpip/seqnum/seqnum.go b/pkg/tcpip/seqnum/seqnum.go new file mode 100644 index 0000000..d3bea7d --- /dev/null +++ b/pkg/tcpip/seqnum/seqnum.go @@ -0,0 +1,62 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package seqnum defines the types and methods for TCP sequence numbers such +// that they fit in 32-bit words and work properly when overflows occur. +package seqnum + +// Value represents the value of a sequence number. +type Value uint32 + +// Size represents the size (length) of a sequence number window. +type Size uint32 + +// LessThan checks if v is before w, i.e., v < w. +func (v Value) LessThan(w Value) bool { + return int32(v-w) < 0 +} + +// LessThanEq returns true if v==w or v is before i.e., v < w. +func (v Value) LessThanEq(w Value) bool { + if v == w { + return true + } + return v.LessThan(w) +} + +// InRange checks if v is in the range [a,b), i.e., a <= v < b. +func (v Value) InRange(a, b Value) bool { + return v-a < b-a +} + +// InWindow checks if v is in the window that starts at 'first' and spans 'size' +// sequence numbers. +func (v Value) InWindow(first Value, size Size) bool { + return v.InRange(first, first.Add(size)) +} + +// Add calculates the sequence number following the [v, v+s) window. +func (v Value) Add(s Size) Value { + return v + Value(s) +} + +// Size calculates the size of the window defined by [v, w). +func (v Value) Size(w Value) Size { + return Size(w - v) +} + +// UpdateForward updates v such that it becomes v + s. +func (v *Value) UpdateForward(s Size) { + *v += Value(s) +} diff --git a/pkg/tcpip/seqnum/seqnum_state_autogen.go b/pkg/tcpip/seqnum/seqnum_state_autogen.go new file mode 100644 index 0000000..23e7981 --- /dev/null +++ b/pkg/tcpip/seqnum/seqnum_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package seqnum diff --git a/pkg/tcpip/sock_err_list.go b/pkg/tcpip/sock_err_list.go new file mode 100644 index 0000000..47d8716 --- /dev/null +++ b/pkg/tcpip/sock_err_list.go @@ -0,0 +1,239 @@ +package tcpip + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type sockErrorElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (sockErrorElementMapper) linkerFor(elem *SockError) *SockError { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type sockErrorList struct { + head *SockError + tail *SockError +} + +// Reset resets list l to the empty state. +func (l *sockErrorList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *sockErrorList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *sockErrorList) Front() *SockError { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *sockErrorList) Back() *SockError { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *sockErrorList) Len() (count int) { + for e := l.Front(); e != nil; e = (sockErrorElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *sockErrorList) PushFront(e *SockError) { + linker := sockErrorElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + sockErrorElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *sockErrorList) PushFrontList(m *sockErrorList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + sockErrorElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + sockErrorElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *sockErrorList) PushBack(e *SockError) { + linker := sockErrorElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + sockErrorElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *sockErrorList) PushBackList(m *sockErrorList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + sockErrorElementMapper{}.linkerFor(l.tail).SetNext(m.head) + sockErrorElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *sockErrorList) InsertAfter(b, e *SockError) { + bLinker := sockErrorElementMapper{}.linkerFor(b) + eLinker := sockErrorElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + sockErrorElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *sockErrorList) InsertBefore(a, e *SockError) { + aLinker := sockErrorElementMapper{}.linkerFor(a) + eLinker := sockErrorElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + sockErrorElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *sockErrorList) Remove(e *SockError) { + linker := sockErrorElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + sockErrorElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + sockErrorElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type sockErrorEntry struct { + next *SockError + prev *SockError +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *sockErrorEntry) Next() *SockError { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *sockErrorEntry) Prev() *SockError { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *sockErrorEntry) SetNext(elem *SockError) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *sockErrorEntry) SetPrev(elem *SockError) { + e.prev = elem +} diff --git a/pkg/tcpip/socketops.go b/pkg/tcpip/socketops.go new file mode 100644 index 0000000..a83d5ad --- /dev/null +++ b/pkg/tcpip/socketops.go @@ -0,0 +1,773 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpip + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" +) + +// SocketOptionsHandler holds methods that help define endpoint specific +// behavior for socket level socket options. These must be implemented by +// endpoints to get notified when socket level options are set. +type SocketOptionsHandler interface { + // OnReuseAddressSet is invoked when SO_REUSEADDR is set for an endpoint. + OnReuseAddressSet(v bool) + + // OnReusePortSet is invoked when SO_REUSEPORT is set for an endpoint. + OnReusePortSet(v bool) + + // OnKeepAliveSet is invoked when SO_KEEPALIVE is set for an endpoint. + OnKeepAliveSet(v bool) + + // OnDelayOptionSet is invoked when TCP_NODELAY is set for an endpoint. + // Note that v will be the inverse of TCP_NODELAY option. + OnDelayOptionSet(v bool) + + // OnCorkOptionSet is invoked when TCP_CORK is set for an endpoint. + OnCorkOptionSet(v bool) + + // LastError is invoked when SO_ERROR is read for an endpoint. + LastError() Error + + // UpdateLastError updates the endpoint specific last error field. + UpdateLastError(err Error) + + // HasNIC is invoked to check if the NIC is valid for SO_BINDTODEVICE. + HasNIC(v int32) bool + + // OnSetSendBufferSize is invoked when the send buffer size for an endpoint is + // changed. The handler is invoked with the new value for the socket send + // buffer size. It also returns the newly set value. + OnSetSendBufferSize(v int64) (newSz int64) + + // OnSetReceiveBufferSize is invoked by SO_RCVBUF and SO_RCVBUFFORCE. The + // handler can optionally return a callback which will be called after + // the buffer size is updated to newSz. + OnSetReceiveBufferSize(v, oldSz int64) (newSz int64, postSet func()) + + // WakeupWriters is invoked when the send buffer size for an endpoint is + // changed. The handler notifies the writers if the send buffer size is + // increased with setsockopt(2) for TCP endpoints. + WakeupWriters() + + // GetAcceptConn returns true if the socket is a TCP socket and is in + // listening state. + GetAcceptConn() bool +} + +// DefaultSocketOptionsHandler is an embeddable type that implements no-op +// implementations for SocketOptionsHandler methods. +type DefaultSocketOptionsHandler struct{} + +var _ SocketOptionsHandler = (*DefaultSocketOptionsHandler)(nil) + +// OnReuseAddressSet implements SocketOptionsHandler.OnReuseAddressSet. +func (*DefaultSocketOptionsHandler) OnReuseAddressSet(bool) {} + +// OnReusePortSet implements SocketOptionsHandler.OnReusePortSet. +func (*DefaultSocketOptionsHandler) OnReusePortSet(bool) {} + +// OnKeepAliveSet implements SocketOptionsHandler.OnKeepAliveSet. +func (*DefaultSocketOptionsHandler) OnKeepAliveSet(bool) {} + +// OnDelayOptionSet implements SocketOptionsHandler.OnDelayOptionSet. +func (*DefaultSocketOptionsHandler) OnDelayOptionSet(bool) {} + +// OnCorkOptionSet implements SocketOptionsHandler.OnCorkOptionSet. +func (*DefaultSocketOptionsHandler) OnCorkOptionSet(bool) {} + +// LastError implements SocketOptionsHandler.LastError. +func (*DefaultSocketOptionsHandler) LastError() Error { + return nil +} + +// UpdateLastError implements SocketOptionsHandler.UpdateLastError. +func (*DefaultSocketOptionsHandler) UpdateLastError(Error) {} + +// HasNIC implements SocketOptionsHandler.HasNIC. +func (*DefaultSocketOptionsHandler) HasNIC(int32) bool { + return false +} + +// OnSetSendBufferSize implements SocketOptionsHandler.OnSetSendBufferSize. +func (*DefaultSocketOptionsHandler) OnSetSendBufferSize(v int64) (newSz int64) { + return v +} + +// WakeupWriters implements SocketOptionsHandler.WakeupWriters. +func (*DefaultSocketOptionsHandler) WakeupWriters() {} + +// OnSetReceiveBufferSize implements SocketOptionsHandler.OnSetReceiveBufferSize. +func (*DefaultSocketOptionsHandler) OnSetReceiveBufferSize(v, oldSz int64) (newSz int64, postSet func()) { + return v, nil +} + +// GetAcceptConn implements SocketOptionsHandler.GetAcceptConn. +func (*DefaultSocketOptionsHandler) GetAcceptConn() bool { + return false +} + +// StackHandler holds methods to access the stack options. These must be +// implemented by the stack. +type StackHandler interface { + // Option allows retrieving stack wide options. + Option(option any) Error + + // TransportProtocolOption allows retrieving individual protocol level + // option values. + TransportProtocolOption(proto TransportProtocolNumber, option GettableTransportProtocolOption) Error +} + +// SocketOptions contains all the variables which store values for SOL_SOCKET, +// SOL_IP, SOL_IPV6 and SOL_TCP level options. +// +// +stateify savable +type SocketOptions struct { + handler SocketOptionsHandler + + // StackHandler is initialized at the creation time and will not change. + stackHandler StackHandler `state:"manual"` + + // These fields are accessed and modified using atomic operations. + + // broadcastEnabled determines whether datagram sockets are allowed to + // send packets to a broadcast address. + broadcastEnabled atomicbitops.Uint32 + + // passCredEnabled determines whether SCM_CREDENTIALS socket control + // messages are enabled. + passCredEnabled atomicbitops.Uint32 + + // noChecksumEnabled determines whether UDP checksum is disabled while + // transmitting for this socket. + noChecksumEnabled atomicbitops.Uint32 + + // reuseAddressEnabled determines whether Bind() should allow reuse of + // local address. + reuseAddressEnabled atomicbitops.Uint32 + + // reusePortEnabled determines whether to permit multiple sockets to be + // bound to an identical socket address. + reusePortEnabled atomicbitops.Uint32 + + // keepAliveEnabled determines whether TCP keepalive is enabled for this + // socket. + keepAliveEnabled atomicbitops.Uint32 + + // multicastLoopEnabled determines whether multicast packets sent over a + // non-loopback interface will be looped back. + multicastLoopEnabled atomicbitops.Uint32 + + // receiveTOSEnabled is used to specify if the TOS ancillary message is + // passed with incoming packets. + receiveTOSEnabled atomicbitops.Uint32 + + // receiveTTLEnabled is used to specify if the TTL ancillary message is passed + // with incoming packets. + receiveTTLEnabled atomicbitops.Uint32 + + // receiveHopLimitEnabled is used to specify if the HopLimit ancillary message + // is passed with incoming packets. + receiveHopLimitEnabled atomicbitops.Uint32 + + // receiveTClassEnabled is used to specify if the IPV6_TCLASS ancillary + // message is passed with incoming packets. + receiveTClassEnabled atomicbitops.Uint32 + + // receivePacketInfoEnabled is used to specify if more information is + // provided with incoming IPv4 packets. + receivePacketInfoEnabled atomicbitops.Uint32 + + // receivePacketInfoEnabled is used to specify if more information is + // provided with incoming IPv6 packets. + receiveIPv6PacketInfoEnabled atomicbitops.Uint32 + + // hdrIncludeEnabled is used to indicate for a raw endpoint that all packets + // being written have an IP header and the endpoint should not attach an IP + // header. + hdrIncludedEnabled atomicbitops.Uint32 + + // v6OnlyEnabled is used to determine whether an IPv6 socket is to be + // restricted to sending and receiving IPv6 packets only. + v6OnlyEnabled atomicbitops.Uint32 + + // quickAckEnabled is used to represent the value of TCP_QUICKACK option. + // It currently does not have any effect on the TCP endpoint. + quickAckEnabled atomicbitops.Uint32 + + // delayOptionEnabled is used to specify if data should be sent out immediately + // by the transport protocol. For TCP, it determines if the Nagle algorithm + // is on or off. + delayOptionEnabled atomicbitops.Uint32 + + // corkOptionEnabled is used to specify if data should be held until segments + // are full by the TCP transport protocol. + corkOptionEnabled atomicbitops.Uint32 + + // receiveOriginalDstAddress is used to specify if the original destination of + // the incoming packet should be returned as an ancillary message. + receiveOriginalDstAddress atomicbitops.Uint32 + + // ipv4RecvErrEnabled determines whether extended reliable error message + // passing is enabled for IPv4. + ipv4RecvErrEnabled atomicbitops.Uint32 + + // ipv6RecvErrEnabled determines whether extended reliable error message + // passing is enabled for IPv6. + ipv6RecvErrEnabled atomicbitops.Uint32 + + // errQueue is the per-socket error queue. It is protected by errQueueMu. + errQueueMu sync.Mutex `state:"nosave"` + errQueue sockErrorList + + // bindToDevice determines the device to which the socket is bound. + bindToDevice atomicbitops.Int32 + + // getSendBufferLimits provides the handler to get the min, default and max + // size for send buffer. It is initialized at the creation time and will not + // change. + getSendBufferLimits GetSendBufferLimits `state:"manual"` + + // sendBufferSize determines the send buffer size for this socket. + sendBufferSize atomicbitops.Int64 + + // getReceiveBufferLimits provides the handler to get the min, default and + // max size for receive buffer. It is initialized at the creation time and + // will not change. + getReceiveBufferLimits GetReceiveBufferLimits `state:"manual"` + + // receiveBufferSize determines the receive buffer size for this socket. + receiveBufferSize atomicbitops.Int64 + + // mu protects the access to the below fields. + mu sync.Mutex `state:"nosave"` + + // linger determines the amount of time the socket should linger before + // close. We currently implement this option for TCP socket only. + linger LingerOption + + // rcvlowat specifies the minimum number of bytes which should be + // received to indicate the socket as readable. + rcvlowat atomicbitops.Int32 + + // experimentOptionValue is the value set for the IP option experiment header + // if it is not zero. + experimentOptionValue atomicbitops.Uint32 +} + +// InitHandler initializes the handler. This must be called before using the +// socket options utility. +func (so *SocketOptions) InitHandler(handler SocketOptionsHandler, stack StackHandler, getSendBufferLimits GetSendBufferLimits, getReceiveBufferLimits GetReceiveBufferLimits) { + so.handler = handler + so.stackHandler = stack + so.getSendBufferLimits = getSendBufferLimits + so.getReceiveBufferLimits = getReceiveBufferLimits +} + +func storeAtomicBool(addr *atomicbitops.Uint32, v bool) { + var val uint32 + if v { + val = 1 + } + addr.Store(val) +} + +// SetLastError sets the last error for a socket. +func (so *SocketOptions) SetLastError(err Error) { + so.handler.UpdateLastError(err) +} + +// GetBroadcast gets value for SO_BROADCAST option. +func (so *SocketOptions) GetBroadcast() bool { + return so.broadcastEnabled.Load() != 0 +} + +// SetBroadcast sets value for SO_BROADCAST option. +func (so *SocketOptions) SetBroadcast(v bool) { + storeAtomicBool(&so.broadcastEnabled, v) +} + +// GetPassCred gets value for SO_PASSCRED option. +func (so *SocketOptions) GetPassCred() bool { + return so.passCredEnabled.Load() != 0 +} + +// SetPassCred sets value for SO_PASSCRED option. +func (so *SocketOptions) SetPassCred(v bool) { + storeAtomicBool(&so.passCredEnabled, v) +} + +// GetNoChecksum gets value for SO_NO_CHECK option. +func (so *SocketOptions) GetNoChecksum() bool { + return so.noChecksumEnabled.Load() != 0 +} + +// SetNoChecksum sets value for SO_NO_CHECK option. +func (so *SocketOptions) SetNoChecksum(v bool) { + storeAtomicBool(&so.noChecksumEnabled, v) +} + +// GetReuseAddress gets value for SO_REUSEADDR option. +func (so *SocketOptions) GetReuseAddress() bool { + return so.reuseAddressEnabled.Load() != 0 +} + +// SetReuseAddress sets value for SO_REUSEADDR option. +func (so *SocketOptions) SetReuseAddress(v bool) { + storeAtomicBool(&so.reuseAddressEnabled, v) + so.handler.OnReuseAddressSet(v) +} + +// GetReusePort gets value for SO_REUSEPORT option. +func (so *SocketOptions) GetReusePort() bool { + return so.reusePortEnabled.Load() != 0 +} + +// SetReusePort sets value for SO_REUSEPORT option. +func (so *SocketOptions) SetReusePort(v bool) { + storeAtomicBool(&so.reusePortEnabled, v) + so.handler.OnReusePortSet(v) +} + +// GetKeepAlive gets value for SO_KEEPALIVE option. +func (so *SocketOptions) GetKeepAlive() bool { + return so.keepAliveEnabled.Load() != 0 +} + +// SetKeepAlive sets value for SO_KEEPALIVE option. +func (so *SocketOptions) SetKeepAlive(v bool) { + storeAtomicBool(&so.keepAliveEnabled, v) + so.handler.OnKeepAliveSet(v) +} + +// GetMulticastLoop gets value for IP_MULTICAST_LOOP option. +func (so *SocketOptions) GetMulticastLoop() bool { + return so.multicastLoopEnabled.Load() != 0 +} + +// SetMulticastLoop sets value for IP_MULTICAST_LOOP option. +func (so *SocketOptions) SetMulticastLoop(v bool) { + storeAtomicBool(&so.multicastLoopEnabled, v) +} + +// GetReceiveTOS gets value for IP_RECVTOS option. +func (so *SocketOptions) GetReceiveTOS() bool { + return so.receiveTOSEnabled.Load() != 0 +} + +// SetReceiveTOS sets value for IP_RECVTOS option. +func (so *SocketOptions) SetReceiveTOS(v bool) { + storeAtomicBool(&so.receiveTOSEnabled, v) +} + +// GetReceiveTTL gets value for IP_RECVTTL option. +func (so *SocketOptions) GetReceiveTTL() bool { + return so.receiveTTLEnabled.Load() != 0 +} + +// SetReceiveTTL sets value for IP_RECVTTL option. +func (so *SocketOptions) SetReceiveTTL(v bool) { + storeAtomicBool(&so.receiveTTLEnabled, v) +} + +// GetReceiveHopLimit gets value for IP_RECVHOPLIMIT option. +func (so *SocketOptions) GetReceiveHopLimit() bool { + return so.receiveHopLimitEnabled.Load() != 0 +} + +// SetReceiveHopLimit sets value for IP_RECVHOPLIMIT option. +func (so *SocketOptions) SetReceiveHopLimit(v bool) { + storeAtomicBool(&so.receiveHopLimitEnabled, v) +} + +// GetReceiveTClass gets value for IPV6_RECVTCLASS option. +func (so *SocketOptions) GetReceiveTClass() bool { + return so.receiveTClassEnabled.Load() != 0 +} + +// SetReceiveTClass sets value for IPV6_RECVTCLASS option. +func (so *SocketOptions) SetReceiveTClass(v bool) { + storeAtomicBool(&so.receiveTClassEnabled, v) +} + +// GetReceivePacketInfo gets value for IP_PKTINFO option. +func (so *SocketOptions) GetReceivePacketInfo() bool { + return so.receivePacketInfoEnabled.Load() != 0 +} + +// SetReceivePacketInfo sets value for IP_PKTINFO option. +func (so *SocketOptions) SetReceivePacketInfo(v bool) { + storeAtomicBool(&so.receivePacketInfoEnabled, v) +} + +// GetIPv6ReceivePacketInfo gets value for IPV6_RECVPKTINFO option. +func (so *SocketOptions) GetIPv6ReceivePacketInfo() bool { + return so.receiveIPv6PacketInfoEnabled.Load() != 0 +} + +// SetIPv6ReceivePacketInfo sets value for IPV6_RECVPKTINFO option. +func (so *SocketOptions) SetIPv6ReceivePacketInfo(v bool) { + storeAtomicBool(&so.receiveIPv6PacketInfoEnabled, v) +} + +// GetHeaderIncluded gets value for IP_HDRINCL option. +func (so *SocketOptions) GetHeaderIncluded() bool { + return so.hdrIncludedEnabled.Load() != 0 +} + +// SetHeaderIncluded sets value for IP_HDRINCL option. +func (so *SocketOptions) SetHeaderIncluded(v bool) { + storeAtomicBool(&so.hdrIncludedEnabled, v) +} + +// GetV6Only gets value for IPV6_V6ONLY option. +func (so *SocketOptions) GetV6Only() bool { + return so.v6OnlyEnabled.Load() != 0 +} + +// SetV6Only sets value for IPV6_V6ONLY option. +// +// Preconditions: the backing TCP or UDP endpoint must be in initial state. +func (so *SocketOptions) SetV6Only(v bool) { + storeAtomicBool(&so.v6OnlyEnabled, v) +} + +// GetQuickAck gets value for TCP_QUICKACK option. +func (so *SocketOptions) GetQuickAck() bool { + return so.quickAckEnabled.Load() != 0 +} + +// SetQuickAck sets value for TCP_QUICKACK option. +func (so *SocketOptions) SetQuickAck(v bool) { + storeAtomicBool(&so.quickAckEnabled, v) +} + +// GetDelayOption gets inverted value for TCP_NODELAY option. +func (so *SocketOptions) GetDelayOption() bool { + return so.delayOptionEnabled.Load() != 0 +} + +// SetDelayOption sets inverted value for TCP_NODELAY option. +func (so *SocketOptions) SetDelayOption(v bool) { + storeAtomicBool(&so.delayOptionEnabled, v) + so.handler.OnDelayOptionSet(v) +} + +// GetCorkOption gets value for TCP_CORK option. +func (so *SocketOptions) GetCorkOption() bool { + return so.corkOptionEnabled.Load() != 0 +} + +// SetCorkOption sets value for TCP_CORK option. +func (so *SocketOptions) SetCorkOption(v bool) { + storeAtomicBool(&so.corkOptionEnabled, v) + so.handler.OnCorkOptionSet(v) +} + +// GetReceiveOriginalDstAddress gets value for IP(V6)_RECVORIGDSTADDR option. +func (so *SocketOptions) GetReceiveOriginalDstAddress() bool { + return so.receiveOriginalDstAddress.Load() != 0 +} + +// SetReceiveOriginalDstAddress sets value for IP(V6)_RECVORIGDSTADDR option. +func (so *SocketOptions) SetReceiveOriginalDstAddress(v bool) { + storeAtomicBool(&so.receiveOriginalDstAddress, v) +} + +// GetIPv4RecvError gets value for IP_RECVERR option. +func (so *SocketOptions) GetIPv4RecvError() bool { + return so.ipv4RecvErrEnabled.Load() != 0 +} + +// SetIPv4RecvError sets value for IP_RECVERR option. +func (so *SocketOptions) SetIPv4RecvError(v bool) { + storeAtomicBool(&so.ipv4RecvErrEnabled, v) + if !v { + so.pruneErrQueue() + } +} + +// GetIPv6RecvError gets value for IPV6_RECVERR option. +func (so *SocketOptions) GetIPv6RecvError() bool { + return so.ipv6RecvErrEnabled.Load() != 0 +} + +// SetIPv6RecvError sets value for IPV6_RECVERR option. +func (so *SocketOptions) SetIPv6RecvError(v bool) { + storeAtomicBool(&so.ipv6RecvErrEnabled, v) + if !v { + so.pruneErrQueue() + } +} + +// GetLastError gets value for SO_ERROR option. +func (so *SocketOptions) GetLastError() Error { + return so.handler.LastError() +} + +// GetOutOfBandInline gets value for SO_OOBINLINE option. +func (*SocketOptions) GetOutOfBandInline() bool { + return true +} + +// SetOutOfBandInline sets value for SO_OOBINLINE option. We currently do not +// support disabling this option. +func (*SocketOptions) SetOutOfBandInline(bool) {} + +// GetLinger gets value for SO_LINGER option. +func (so *SocketOptions) GetLinger() LingerOption { + so.mu.Lock() + linger := so.linger + so.mu.Unlock() + return linger +} + +// SetLinger sets value for SO_LINGER option. +func (so *SocketOptions) SetLinger(linger LingerOption) { + so.mu.Lock() + so.linger = linger + so.mu.Unlock() +} + +// GetExperimentOptionValue gets value for the experiment IP option header. +func (so *SocketOptions) GetExperimentOptionValue() uint16 { + v := so.experimentOptionValue.Load() + return uint16(v) +} + +// SetExperimentOptionValue sets the value for the experiment IP option header. +func (so *SocketOptions) SetExperimentOptionValue(v uint16) { + so.experimentOptionValue.Store(uint32(v)) +} + +// SockErrOrigin represents the constants for error origin. +type SockErrOrigin uint8 + +const ( + // SockExtErrorOriginNone represents an unknown error origin. + SockExtErrorOriginNone SockErrOrigin = iota + + // SockExtErrorOriginLocal indicates a local error. + SockExtErrorOriginLocal + + // SockExtErrorOriginICMP indicates an IPv4 ICMP error. + SockExtErrorOriginICMP + + // SockExtErrorOriginICMP6 indicates an IPv6 ICMP error. + SockExtErrorOriginICMP6 +) + +// IsICMPErr indicates if the error originated from an ICMP error. +func (origin SockErrOrigin) IsICMPErr() bool { + return origin == SockExtErrorOriginICMP || origin == SockExtErrorOriginICMP6 +} + +// SockErrorCause is the cause of a socket error. +type SockErrorCause interface { + // Origin is the source of the error. + Origin() SockErrOrigin + + // Type is the origin specific type of error. + Type() uint8 + + // Code is the origin and type specific error code. + Code() uint8 + + // Info is any extra information about the error. + Info() uint32 +} + +// LocalSockError is a socket error that originated from the local host. +// +// +stateify savable +type LocalSockError struct { + info uint32 +} + +// Origin implements SockErrorCause. +func (*LocalSockError) Origin() SockErrOrigin { + return SockExtErrorOriginLocal +} + +// Type implements SockErrorCause. +func (*LocalSockError) Type() uint8 { + return 0 +} + +// Code implements SockErrorCause. +func (*LocalSockError) Code() uint8 { + return 0 +} + +// Info implements SockErrorCause. +func (l *LocalSockError) Info() uint32 { + return l.info +} + +// SockError represents a queue entry in the per-socket error queue. +// +// +stateify savable +type SockError struct { + sockErrorEntry + + // Err is the error caused by the errant packet. + Err Error + // Cause is the detailed cause of the error. + Cause SockErrorCause + + // Payload is the errant packet's payload. + Payload *buffer.View + // Dst is the original destination address of the errant packet. + Dst FullAddress + // Offender is the original sender address of the errant packet. + Offender FullAddress + // NetProto is the network protocol being used to transmit the packet. + NetProto NetworkProtocolNumber +} + +// pruneErrQueue resets the queue. +func (so *SocketOptions) pruneErrQueue() { + so.errQueueMu.Lock() + so.errQueue.Reset() + so.errQueueMu.Unlock() +} + +// DequeueErr dequeues a socket extended error from the error queue and returns +// it. Returns nil if queue is empty. +func (so *SocketOptions) DequeueErr() *SockError { + so.errQueueMu.Lock() + defer so.errQueueMu.Unlock() + + err := so.errQueue.Front() + if err != nil { + so.errQueue.Remove(err) + } + return err +} + +// PeekErr returns the error in the front of the error queue. Returns nil if +// the error queue is empty. +func (so *SocketOptions) PeekErr() *SockError { + so.errQueueMu.Lock() + defer so.errQueueMu.Unlock() + return so.errQueue.Front() +} + +// QueueErr inserts the error at the back of the error queue. +// +// Preconditions: so.GetIPv4RecvError() or so.GetIPv6RecvError() is true. +func (so *SocketOptions) QueueErr(err *SockError) { + so.errQueueMu.Lock() + defer so.errQueueMu.Unlock() + so.errQueue.PushBack(err) +} + +// QueueLocalErr queues a local error onto the local queue. +func (so *SocketOptions) QueueLocalErr(err Error, net NetworkProtocolNumber, info uint32, dst FullAddress, payload *buffer.View) { + so.QueueErr(&SockError{ + Err: err, + Cause: &LocalSockError{info: info}, + Payload: payload, + Dst: dst, + NetProto: net, + }) +} + +// GetBindToDevice gets value for SO_BINDTODEVICE option. +func (so *SocketOptions) GetBindToDevice() int32 { + return so.bindToDevice.Load() +} + +// SetBindToDevice sets value for SO_BINDTODEVICE option. If bindToDevice is +// zero, the socket device binding is removed. +func (so *SocketOptions) SetBindToDevice(bindToDevice int32) Error { + if bindToDevice != 0 && !so.handler.HasNIC(bindToDevice) { + return &ErrUnknownDevice{} + } + + so.bindToDevice.Store(bindToDevice) + return nil +} + +// GetSendBufferSize gets value for SO_SNDBUF option. +func (so *SocketOptions) GetSendBufferSize() int64 { + return so.sendBufferSize.Load() +} + +// SendBufferLimits returns the [min, max) range of allowable send buffer +// sizes. +func (so *SocketOptions) SendBufferLimits() (min, max int64) { + limits := so.getSendBufferLimits(so.stackHandler) + return int64(limits.Min), int64(limits.Max) +} + +// SetSendBufferSize sets value for SO_SNDBUF option. notify indicates if the +// stack handler should be invoked to set the send buffer size. +func (so *SocketOptions) SetSendBufferSize(sendBufferSize int64, notify bool) { + if notify { + sendBufferSize = so.handler.OnSetSendBufferSize(sendBufferSize) + } + so.sendBufferSize.Store(sendBufferSize) + if notify { + so.handler.WakeupWriters() + } +} + +// GetReceiveBufferSize gets value for SO_RCVBUF option. +func (so *SocketOptions) GetReceiveBufferSize() int64 { + return so.receiveBufferSize.Load() +} + +// ReceiveBufferLimits returns the [min, max) range of allowable receive buffer +// sizes. +func (so *SocketOptions) ReceiveBufferLimits() (min, max int64) { + limits := so.getReceiveBufferLimits(so.stackHandler) + return int64(limits.Min), int64(limits.Max) +} + +// SetReceiveBufferSize sets the value of the SO_RCVBUF option, optionally +// notifying the owning endpoint. +func (so *SocketOptions) SetReceiveBufferSize(receiveBufferSize int64, notify bool) { + var postSet func() + if notify { + oldSz := so.receiveBufferSize.Load() + receiveBufferSize, postSet = so.handler.OnSetReceiveBufferSize(receiveBufferSize, oldSz) + } + so.receiveBufferSize.Store(receiveBufferSize) + if postSet != nil { + postSet() + } +} + +// GetRcvlowat gets value for SO_RCVLOWAT option. +func (so *SocketOptions) GetRcvlowat() int32 { + // TODO(b/226603727): Return so.rcvlowat after adding complete support + // for SO_RCVLOWAT option. For now, return the default value of 1. + defaultRcvlowat := int32(1) + return defaultRcvlowat +} + +// SetRcvlowat sets value for SO_RCVLOWAT option. +func (so *SocketOptions) SetRcvlowat(rcvlowat int32) Error { + so.rcvlowat.Store(rcvlowat) + return nil +} + +// GetAcceptConn gets value for SO_ACCEPTCONN option. +func (so *SocketOptions) GetAcceptConn() bool { + return so.handler.GetAcceptConn() +} diff --git a/pkg/tcpip/stack/address_state_mutex.go b/pkg/tcpip/stack/address_state_mutex.go new file mode 100644 index 0000000..eb8c8e0 --- /dev/null +++ b/pkg/tcpip/stack/address_state_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type addressStateRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var addressStatelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type addressStatelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *addressStateRWMutex) Lock() { + locking.AddGLock(addressStateprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *addressStateRWMutex) NestedLock(i addressStatelockNameIndex) { + locking.AddGLock(addressStateprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *addressStateRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(addressStateprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *addressStateRWMutex) NestedUnlock(i addressStatelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(addressStateprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *addressStateRWMutex) RLock() { + locking.AddGLock(addressStateprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *addressStateRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(addressStateprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *addressStateRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *addressStateRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *addressStateRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var addressStateprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func addressStateinitLockNames() {} + +func init() { + addressStateinitLockNames() + addressStateprefixIndex = locking.NewMutexClass(reflect.TypeOf(addressStateRWMutex{}), addressStatelockNames) +} diff --git a/pkg/tcpip/stack/address_state_refs.go b/pkg/tcpip/stack/address_state_refs.go new file mode 100644 index 0000000..96105d7 --- /dev/null +++ b/pkg/tcpip/stack/address_state_refs.go @@ -0,0 +1,141 @@ +package stack + +import ( + "context" + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/refs" +) + +// enableLogging indicates whether reference-related events should be logged (with +// stack traces). This is false by default and should only be set to true for +// debugging purposes, as it can generate an extremely large amount of output +// and drastically degrade performance. +const addressStateenableLogging = false + +// obj is used to customize logging. Note that we use a pointer to T so that +// we do not copy the entire object when passed as a format parameter. +var addressStateobj *addressState + +// Refs implements refs.RefCounter. It keeps a reference count using atomic +// operations and calls the destructor when the count reaches zero. +// +// NOTE: Do not introduce additional fields to the Refs struct. It is used by +// many filesystem objects, and we want to keep it as small as possible (i.e., +// the same size as using an int64 directly) to avoid taking up extra cache +// space. In general, this template should not be extended at the cost of +// performance. If it does not offer enough flexibility for a particular object +// (example: b/187877947), we should implement the RefCounter/CheckedObject +// interfaces manually. +// +// +stateify savable +type addressStateRefs struct { + // refCount is composed of two fields: + // + // [32-bit speculative references]:[32-bit real references] + // + // Speculative references are used for TryIncRef, to avoid a CompareAndSwap + // loop. See IncRef, DecRef and TryIncRef for details of how these fields are + // used. + refCount atomicbitops.Int64 +} + +// InitRefs initializes r with one reference and, if enabled, activates leak +// checking. +func (r *addressStateRefs) InitRefs() { + r.refCount.RacyStore(1) + refs.Register(r) +} + +// RefType implements refs.CheckedObject.RefType. +func (r *addressStateRefs) RefType() string { + return fmt.Sprintf("%T", addressStateobj)[1:] +} + +// LeakMessage implements refs.CheckedObject.LeakMessage. +func (r *addressStateRefs) LeakMessage() string { + return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs()) +} + +// LogRefs implements refs.CheckedObject.LogRefs. +func (r *addressStateRefs) LogRefs() bool { + return addressStateenableLogging +} + +// ReadRefs returns the current number of references. The returned count is +// inherently racy and is unsafe to use without external synchronization. +func (r *addressStateRefs) ReadRefs() int64 { + return r.refCount.Load() +} + +// IncRef implements refs.RefCounter.IncRef. +// +//go:nosplit +func (r *addressStateRefs) IncRef() { + v := r.refCount.Add(1) + if addressStateenableLogging { + refs.LogIncRef(r, v) + } + if v <= 1 { + panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType())) + } +} + +// TryIncRef implements refs.TryRefCounter.TryIncRef. +// +// To do this safely without a loop, a speculative reference is first acquired +// on the object. This allows multiple concurrent TryIncRef calls to distinguish +// other TryIncRef calls from genuine references held. +// +//go:nosplit +func (r *addressStateRefs) TryIncRef() bool { + const speculativeRef = 1 << 32 + if v := r.refCount.Add(speculativeRef); int32(v) == 0 { + + r.refCount.Add(-speculativeRef) + return false + } + + v := r.refCount.Add(-speculativeRef + 1) + if addressStateenableLogging { + refs.LogTryIncRef(r, v) + } + return true +} + +// DecRef implements refs.RefCounter.DecRef. +// +// Note that speculative references are counted here. Since they were added +// prior to real references reaching zero, they will successfully convert to +// real references. In other words, we see speculative references only in the +// following case: +// +// A: TryIncRef [speculative increase => sees non-negative references] +// B: DecRef [real decrease] +// A: TryIncRef [transform speculative to real] +// +//go:nosplit +func (r *addressStateRefs) DecRef(destroy func()) { + v := r.refCount.Add(-1) + if addressStateenableLogging { + refs.LogDecRef(r, v) + } + switch { + case v < 0: + panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType())) + + case v == 0: + refs.Unregister(r) + + if destroy != nil { + destroy() + } + } +} + +func (r *addressStateRefs) afterLoad(context.Context) { + if r.ReadRefs() > 0 { + refs.Register(r) + } +} diff --git a/pkg/tcpip/stack/addressable_endpoint_state.go b/pkg/tcpip/stack/addressable_endpoint_state.go new file mode 100644 index 0000000..8a00ef9 --- /dev/null +++ b/pkg/tcpip/stack/addressable_endpoint_state.go @@ -0,0 +1,950 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +func (lifetimes *AddressLifetimes) sanitize() { + if lifetimes.Deprecated { + lifetimes.PreferredUntil = tcpip.MonotonicTime{} + } +} + +var _ AddressableEndpoint = (*AddressableEndpointState)(nil) + +// AddressableEndpointState is an implementation of an AddressableEndpoint. +// +// +stateify savable +type AddressableEndpointState struct { + networkEndpoint NetworkEndpoint + options AddressableEndpointStateOptions + + // Lock ordering (from outer to inner lock ordering): + // + // AddressableEndpointState.mu + // addressState.mu + mu addressableEndpointStateRWMutex `state:"nosave"` + // TODO(b/361075310): Enable s/r for the below fields. + // + // +checklocks:mu + endpoints map[tcpip.Address]*addressState `state:"nosave"` + // +checklocks:mu + primary []*addressState `state:"nosave"` +} + +// AddressableEndpointStateOptions contains options used to configure an +// AddressableEndpointState. +// +// +stateify savable +type AddressableEndpointStateOptions struct { + // HiddenWhileDisabled determines whether addresses should be returned to + // callers while the NetworkEndpoint this AddressableEndpointState belongs + // to is disabled. + HiddenWhileDisabled bool +} + +// Init initializes the AddressableEndpointState with networkEndpoint. +// +// Must be called before calling any other function on m. +func (a *AddressableEndpointState) Init(networkEndpoint NetworkEndpoint, options AddressableEndpointStateOptions) { + a.networkEndpoint = networkEndpoint + a.options = options + + a.mu.Lock() + defer a.mu.Unlock() + a.endpoints = make(map[tcpip.Address]*addressState) +} + +// OnNetworkEndpointEnabledChanged must be called every time the +// NetworkEndpoint this AddressableEndpointState belongs to is enabled or +// disabled so that any AddressDispatchers can be notified of the NIC enabled +// change. +func (a *AddressableEndpointState) OnNetworkEndpointEnabledChanged() { + a.mu.RLock() + defer a.mu.RUnlock() + + for _, ep := range a.endpoints { + ep.mu.Lock() + ep.notifyChangedLocked() + ep.mu.Unlock() + } +} + +// GetAddress returns the AddressEndpoint for the passed address. +// +// GetAddress does not increment the address's reference count or check if the +// address is considered bound to the endpoint. +// +// Returns nil if the passed address is not associated with the endpoint. +func (a *AddressableEndpointState) GetAddress(addr tcpip.Address) AddressEndpoint { + a.mu.RLock() + defer a.mu.RUnlock() + + ep, ok := a.endpoints[addr] + if !ok { + return nil + } + return ep +} + +// ForEachEndpoint calls f for each address. +// +// Once f returns false, f will no longer be called. +func (a *AddressableEndpointState) ForEachEndpoint(f func(AddressEndpoint) bool) { + a.mu.RLock() + defer a.mu.RUnlock() + + for _, ep := range a.endpoints { + if !f(ep) { + return + } + } +} + +// ForEachPrimaryEndpoint calls f for each primary address. +// +// Once f returns false, f will no longer be called. +func (a *AddressableEndpointState) ForEachPrimaryEndpoint(f func(AddressEndpoint) bool) { + a.mu.RLock() + defer a.mu.RUnlock() + + for _, ep := range a.primary { + if !f(ep) { + return + } + } +} + +func (a *AddressableEndpointState) releaseAddressState(addrState *addressState) { + a.mu.Lock() + defer a.mu.Unlock() + a.releaseAddressStateLocked(addrState) +} + +// releaseAddressStateLocked removes addrState from a's address state +// (primary and endpoints list). +// +// +checklocks:a.mu +func (a *AddressableEndpointState) releaseAddressStateLocked(addrState *addressState) { + oldPrimary := a.primary + for i, s := range a.primary { + if s == addrState { + a.primary = append(a.primary[:i], a.primary[i+1:]...) + oldPrimary[len(oldPrimary)-1] = nil + break + } + } + delete(a.endpoints, addrState.addr.Address) +} + +// AddAndAcquirePermanentAddress implements AddressableEndpoint. +func (a *AddressableEndpointState) AddAndAcquirePermanentAddress(addr tcpip.AddressWithPrefix, properties AddressProperties) (AddressEndpoint, tcpip.Error) { + return a.AddAndAcquireAddress(addr, properties, Permanent) +} + +// AddAndAcquireTemporaryAddress adds a temporary address. +// +// Returns *tcpip.ErrDuplicateAddress if the address exists. +// +// The temporary address's endpoint is acquired and returned. +func (a *AddressableEndpointState) AddAndAcquireTemporaryAddress(addr tcpip.AddressWithPrefix, peb PrimaryEndpointBehavior) (AddressEndpoint, tcpip.Error) { + return a.AddAndAcquireAddress(addr, AddressProperties{PEB: peb}, Temporary) +} + +// AddAndAcquireAddress adds an address with the specified kind. +// +// Returns *tcpip.ErrDuplicateAddress if the address exists. +func (a *AddressableEndpointState) AddAndAcquireAddress(addr tcpip.AddressWithPrefix, properties AddressProperties, kind AddressKind) (AddressEndpoint, tcpip.Error) { + a.mu.Lock() + defer a.mu.Unlock() + ep, err := a.addAndAcquireAddressLocked(addr, properties, kind) + // From https://golang.org/doc/faq#nil_error: + // + // Under the covers, interfaces are implemented as two elements, a type T and + // a value V. + // + // An interface value is nil only if the V and T are both unset, (T=nil, V is + // not set), In particular, a nil interface will always hold a nil type. If we + // store a nil pointer of type *int inside an interface value, the inner type + // will be *int regardless of the value of the pointer: (T=*int, V=nil). Such + // an interface value will therefore be non-nil even when the pointer value V + // inside is nil. + // + // Since addAndAcquireAddressLocked returns a nil value with a non-nil type, + // we need to explicitly return nil below if ep is (a typed) nil. + if ep == nil { + return nil, err + } + return ep, err +} + +// addAndAcquireAddressLocked adds, acquires and returns a permanent or +// temporary address. +// +// If the addressable endpoint already has the address in a non-permanent state, +// and addAndAcquireAddressLocked is adding a permanent address, that address is +// promoted in place and its properties set to the properties provided. If the +// address already exists in any other state, then *tcpip.ErrDuplicateAddress is +// returned, regardless the kind of address that is being added. +// +// +checklocks:a.mu +func (a *AddressableEndpointState) addAndAcquireAddressLocked(addr tcpip.AddressWithPrefix, properties AddressProperties, kind AddressKind) (*addressState, tcpip.Error) { + var permanent bool + switch kind { + case PermanentExpired: + panic(fmt.Sprintf("cannot add address %s in PermanentExpired state", addr)) + case Permanent, PermanentTentative: + permanent = true + case Temporary: + default: + panic(fmt.Sprintf("unknown address kind: %d", kind)) + } + // attemptAddToPrimary is false when the address is already in the primary + // address list. + attemptAddToPrimary := true + addrState, ok := a.endpoints[addr.Address] + if ok { + if !permanent { + // We are adding a non-permanent address but the address exists. No need + // to go any further since we can only promote existing temporary/expired + // addresses to permanent. + return nil, &tcpip.ErrDuplicateAddress{} + } + + addrState.mu.RLock() + if addrState.refs.ReadRefs() == 0 { + panic(fmt.Sprintf("found an address that should have been released (ref count == 0); address = %s", addrState.addr)) + } + isPermanent := addrState.kind.IsPermanent() + addrState.mu.RUnlock() + + if isPermanent { + // We are adding a permanent address but a permanent address already + // exists. + return nil, &tcpip.ErrDuplicateAddress{} + } + + // We now promote the address. + for i, s := range a.primary { + if s == addrState { + switch properties.PEB { + case CanBePrimaryEndpoint: + // The address is already in the primary address list. + attemptAddToPrimary = false + case FirstPrimaryEndpoint: + if i == 0 { + // The address is already first in the primary address list. + attemptAddToPrimary = false + } else { + a.primary = append(a.primary[:i], a.primary[i+1:]...) + } + case NeverPrimaryEndpoint: + a.primary = append(a.primary[:i], a.primary[i+1:]...) + default: + panic(fmt.Sprintf("unrecognized primary endpoint behaviour = %d", properties.PEB)) + } + break + } + } + addrState.refs.IncRef() + } else { + addrState = &addressState{ + addressableEndpointState: a, + addr: addr, + temporary: properties.Temporary, + // Cache the subnet in addrState to avoid calls to addr.Subnet() as that + // results in allocations on every call. + subnet: addr.Subnet(), + } + addrState.refs.InitRefs() + a.endpoints[addr.Address] = addrState + // We never promote an address to temporary - it can only be added as such. + // If we are actually adding a permanent address, it is promoted below. + addrState.kind = Temporary + } + + // At this point we have an address we are either promoting from an expired or + // temporary address to permanent, promoting an expired address to temporary, + // or we are adding a new temporary or permanent address. + // + // The address MUST be write locked at this point. + addrState.mu.Lock() + defer addrState.mu.Unlock() + + if permanent { + if addrState.kind.IsPermanent() { + panic(fmt.Sprintf("only non-permanent addresses should be promoted to permanent; address = %s", addrState.addr)) + } + + // Primary addresses are biased by 1. + addrState.refs.IncRef() + addrState.kind = kind + } + addrState.configType = properties.ConfigType + lifetimes := properties.Lifetimes + lifetimes.sanitize() + addrState.lifetimes = lifetimes + addrState.disp = properties.Disp + + if attemptAddToPrimary { + switch properties.PEB { + case NeverPrimaryEndpoint: + case CanBePrimaryEndpoint: + a.primary = append(a.primary, addrState) + case FirstPrimaryEndpoint: + if cap(a.primary) == len(a.primary) { + a.primary = append([]*addressState{addrState}, a.primary...) + } else { + // Shift all the endpoints by 1 to make room for the new address at the + // front. We could have just created a new slice but this saves + // allocations when the slice has capacity for the new address. + primaryCount := len(a.primary) + a.primary = append(a.primary, nil) + if n := copy(a.primary[1:], a.primary); n != primaryCount { + panic(fmt.Sprintf("copied %d elements; expected = %d elements", n, primaryCount)) + } + a.primary[0] = addrState + } + default: + panic(fmt.Sprintf("unrecognized primary endpoint behaviour = %d", properties.PEB)) + } + } + + addrState.notifyChangedLocked() + return addrState, nil +} + +// RemovePermanentAddress implements AddressableEndpoint. +func (a *AddressableEndpointState) RemovePermanentAddress(addr tcpip.Address) tcpip.Error { + a.mu.Lock() + defer a.mu.Unlock() + return a.removePermanentAddressLocked(addr) +} + +// removePermanentAddressLocked is like RemovePermanentAddress but with locking +// requirements. +// +// +checklocks:a.mu +func (a *AddressableEndpointState) removePermanentAddressLocked(addr tcpip.Address) tcpip.Error { + addrState, ok := a.endpoints[addr] + if !ok { + return &tcpip.ErrBadLocalAddress{} + } + + return a.removePermanentEndpointLocked(addrState, AddressRemovalManualAction) +} + +// RemovePermanentEndpoint removes the passed endpoint if it is associated with +// a and permanent. +func (a *AddressableEndpointState) RemovePermanentEndpoint(ep AddressEndpoint, reason AddressRemovalReason) tcpip.Error { + addrState, ok := ep.(*addressState) + if !ok || addrState.addressableEndpointState != a { + return &tcpip.ErrInvalidEndpointState{} + } + + a.mu.Lock() + defer a.mu.Unlock() + return a.removePermanentEndpointLocked(addrState, reason) +} + +// removePermanentAddressLocked is like RemovePermanentAddress but with locking +// requirements. +// +// +checklocks:a.mu +func (a *AddressableEndpointState) removePermanentEndpointLocked(addrState *addressState, reason AddressRemovalReason) tcpip.Error { + if !addrState.GetKind().IsPermanent() { + return &tcpip.ErrBadLocalAddress{} + } + + addrState.remove(reason) + a.decAddressRefLocked(addrState) + return nil +} + +// decAddressRef decrements the address's reference count and releases it once +// the reference count hits 0. +func (a *AddressableEndpointState) decAddressRef(addrState *addressState) { + a.mu.Lock() + defer a.mu.Unlock() + a.decAddressRefLocked(addrState) +} + +// decAddressRefLocked is like decAddressRef but with locking requirements. +// +// +checklocks:a.mu +func (a *AddressableEndpointState) decAddressRefLocked(addrState *addressState) { + destroy := false + addrState.refs.DecRef(func() { + destroy = true + }) + + if !destroy { + return + } + addrState.mu.Lock() + defer addrState.mu.Unlock() + // A non-expired permanent address must not have its reference count dropped + // to 0. + if addrState.kind.IsPermanent() { + panic(fmt.Sprintf("permanent addresses should be removed through the AddressableEndpoint: addr = %s, kind = %d", addrState.addr, addrState.kind)) + } + + a.releaseAddressStateLocked(addrState) +} + +// SetDeprecated implements stack.AddressableEndpoint. +func (a *AddressableEndpointState) SetDeprecated(addr tcpip.Address, deprecated bool) tcpip.Error { + a.mu.RLock() + defer a.mu.RUnlock() + + addrState, ok := a.endpoints[addr] + if !ok { + return &tcpip.ErrBadLocalAddress{} + } + addrState.SetDeprecated(deprecated) + return nil +} + +// SetLifetimes implements stack.AddressableEndpoint. +func (a *AddressableEndpointState) SetLifetimes(addr tcpip.Address, lifetimes AddressLifetimes) tcpip.Error { + a.mu.RLock() + defer a.mu.RUnlock() + + addrState, ok := a.endpoints[addr] + if !ok { + return &tcpip.ErrBadLocalAddress{} + } + addrState.SetLifetimes(lifetimes) + return nil +} + +// MainAddress implements AddressableEndpoint. +func (a *AddressableEndpointState) MainAddress() tcpip.AddressWithPrefix { + a.mu.RLock() + defer a.mu.RUnlock() + + ep := a.acquirePrimaryAddressRLocked(tcpip.Address{}, tcpip.Address{} /* srcHint */, func(ep *addressState) bool { + switch kind := ep.GetKind(); kind { + case Permanent: + return a.networkEndpoint.Enabled() || !a.options.HiddenWhileDisabled + case PermanentTentative, PermanentExpired, Temporary: + return false + default: + panic(fmt.Sprintf("unknown address kind: %d", kind)) + } + }) + if ep == nil { + return tcpip.AddressWithPrefix{} + } + addr := ep.AddressWithPrefix() + // Note that when ep must have a ref count >=2, because its ref count + // must be >=1 in order to be found and the ref count was incremented + // when a reference was acquired. The only way for the ref count to + // drop below 2 is for the endpoint to be removed, which requires a + // write lock; so we're guaranteed to be able to decrement the ref + // count and not need to remove the endpoint from a.primary. + ep.decRefMustNotFree() + return addr +} + +// acquirePrimaryAddressRLocked returns an acquired primary address that is +// valid according to isValid. +// +// +checklocksread:a.mu +func (a *AddressableEndpointState) acquirePrimaryAddressRLocked(remoteAddr, srcHint tcpip.Address, isValid func(*addressState) bool) *addressState { + // TODO: Move this out into IPv4-specific code. + // IPv6 handles source IP selection elsewhere. We have to do source + // selection only for IPv4, in which case ep is never deprecated. Thus + // we don't have to worry about refcounts. + if remoteAddr.Len() == header.IPv4AddressSize && remoteAddr != (tcpip.Address{}) { + var best *addressState + var bestLen uint8 + for _, state := range a.primary { + if !isValid(state) { + continue + } + // Source hint takes precedent over prefix matching. + if state.addr.Address == srcHint && srcHint != (tcpip.Address{}) { + best = state + break + } + stateLen := state.addr.Address.MatchingPrefix(remoteAddr) + if best == nil || bestLen < stateLen { + best = state + bestLen = stateLen + } + } + if best != nil && best.TryIncRef() { + return best + } + } + + var deprecatedEndpoint *addressState + for _, ep := range a.primary { + if !isValid(ep) { + continue + } + + if !ep.Deprecated() { + if ep.TryIncRef() { + // ep is not deprecated, so return it immediately. + // + // If we kept track of a deprecated endpoint, decrement its reference + // count since it was incremented when we decided to keep track of it. + if deprecatedEndpoint != nil { + // Note that when deprecatedEndpoint was found, its ref count + // must have necessarily been >=1, and after incrementing it + // must be >=2. The only way for the ref count to drop below 2 is + // for the endpoint to be removed, which requires a write lock; + // so we're guaranteed to be able to decrement the ref count + // and not need to remove the endpoint from a.primary. + deprecatedEndpoint.decRefMustNotFree() + } + + return ep + } + } else if deprecatedEndpoint == nil && ep.TryIncRef() { + // We prefer an endpoint that is not deprecated, but we keep track of + // ep in case a doesn't have any non-deprecated endpoints. + // + // If we end up finding a more preferred endpoint, ep's reference count + // will be decremented. + deprecatedEndpoint = ep + } + } + + return deprecatedEndpoint +} + +// AcquireAssignedAddressOrMatching returns an address endpoint that is +// considered assigned to the addressable endpoint. +// +// If the address is an exact match with an existing address, that address is +// returned. Otherwise, if f is provided, f is called with each address and +// the address that f returns true for is returned. +// +// If there is no matching address, a temporary address will be returned if +// allowTemp is true. +// +// If readOnly is true, the address will be returned without an extra reference. +// In this case it is not safe to modify the endpoint, only read attributes like +// subnet. +// +// Regardless how the address was obtained, it will be acquired before it is +// returned. +func (a *AddressableEndpointState) AcquireAssignedAddressOrMatching(localAddr tcpip.Address, f func(AddressEndpoint) bool, allowTemp bool, tempPEB PrimaryEndpointBehavior, readOnly bool) AddressEndpoint { + lookup := func() *addressState { + if addrState, ok := a.endpoints[localAddr]; ok { + if !addrState.IsAssigned(allowTemp) { + return nil + } + + if !readOnly && !addrState.TryIncRef() { + panic(fmt.Sprintf("failed to increase the reference count for address = %s", addrState.addr)) + } + + return addrState + } + + if f != nil { + for _, addrState := range a.endpoints { + if addrState.IsAssigned(allowTemp) && f(addrState) { + if !readOnly && !addrState.TryIncRef() { + continue + } + return addrState + } + } + } + return nil + } + // Avoid exclusive lock on mu unless we need to add a new address. + a.mu.RLock() + ep := lookup() + a.mu.RUnlock() + + if ep != nil { + return ep + } + + if !allowTemp { + return nil + } + + // Acquire state lock in exclusive mode as we need to add a new temporary + // endpoint. + a.mu.Lock() + defer a.mu.Unlock() + + // Do the lookup again in case another goroutine added the address in the time + // we released and acquired the lock. + ep = lookup() + if ep != nil { + return ep + } + + // Proceed to add a new temporary endpoint. + addr := localAddr.WithPrefix() + ep, err := a.addAndAcquireAddressLocked(addr, AddressProperties{PEB: tempPEB, Temporary: true}, Temporary) + if err != nil { + // addAndAcquireAddressLocked only returns an error if the address is + // already assigned but we just checked above if the address exists so we + // expect no error. + panic(fmt.Sprintf("a.addAndAcquireAddressLocked(%s, AddressProperties{PEB: %s}, false): %s", addr, tempPEB, err)) + } + + // From https://golang.org/doc/faq#nil_error: + // + // Under the covers, interfaces are implemented as two elements, a type T and + // a value V. + // + // An interface value is nil only if the V and T are both unset, (T=nil, V is + // not set), In particular, a nil interface will always hold a nil type. If we + // store a nil pointer of type *int inside an interface value, the inner type + // will be *int regardless of the value of the pointer: (T=*int, V=nil). Such + // an interface value will therefore be non-nil even when the pointer value V + // inside is nil. + // + // Since addAndAcquireAddressLocked returns a nil value with a non-nil type, + // we need to explicitly return nil below if ep is (a typed) nil. + if ep == nil { + return nil + } + if readOnly { + if ep.addressableEndpointState == a { + // Checklocks doesn't understand that we are logically guaranteed to have + // ep.mu locked already. We need to use checklocksignore to appease the + // analyzer. + ep.addressableEndpointState.decAddressRefLocked(ep) // +checklocksignore + } else { + ep.DecRef() + } + } + return ep +} + +// AcquireAssignedAddress implements AddressableEndpoint. +func (a *AddressableEndpointState) AcquireAssignedAddress(localAddr tcpip.Address, allowTemp bool, tempPEB PrimaryEndpointBehavior, readOnly bool) AddressEndpoint { + return a.AcquireAssignedAddressOrMatching(localAddr, nil, allowTemp, tempPEB, readOnly) +} + +// AcquireOutgoingPrimaryAddress implements AddressableEndpoint. +func (a *AddressableEndpointState) AcquireOutgoingPrimaryAddress(remoteAddr tcpip.Address, srcHint tcpip.Address, allowExpired bool) AddressEndpoint { + a.mu.Lock() + defer a.mu.Unlock() + + ep := a.acquirePrimaryAddressRLocked(remoteAddr, srcHint, func(ep *addressState) bool { + return ep.IsAssigned(allowExpired) + }) + + // From https://golang.org/doc/faq#nil_error: + // + // Under the covers, interfaces are implemented as two elements, a type T and + // a value V. + // + // An interface value is nil only if the V and T are both unset, (T=nil, V is + // not set), In particular, a nil interface will always hold a nil type. If we + // store a nil pointer of type *int inside an interface value, the inner type + // will be *int regardless of the value of the pointer: (T=*int, V=nil). Such + // an interface value will therefore be non-nil even when the pointer value V + // inside is nil. + // + // Since acquirePrimaryAddressLocked returns a nil value with a non-nil type, + // we need to explicitly return nil below if ep is (a typed) nil. + if ep == nil { + return nil + } + + return ep +} + +// PrimaryAddresses implements AddressableEndpoint. +func (a *AddressableEndpointState) PrimaryAddresses() []tcpip.AddressWithPrefix { + a.mu.RLock() + defer a.mu.RUnlock() + + var addrs []tcpip.AddressWithPrefix + if a.options.HiddenWhileDisabled && !a.networkEndpoint.Enabled() { + return addrs + } + for _, ep := range a.primary { + switch kind := ep.GetKind(); kind { + // Don't include tentative, expired or temporary endpoints + // to avoid confusion and prevent the caller from using + // those. + case PermanentTentative, PermanentExpired, Temporary: + continue + case Permanent: + default: + panic(fmt.Sprintf("address %s has unknown kind %d", ep.AddressWithPrefix(), kind)) + } + + addrs = append(addrs, ep.AddressWithPrefix()) + } + + return addrs +} + +// PermanentAddresses implements AddressableEndpoint. +func (a *AddressableEndpointState) PermanentAddresses() []tcpip.AddressWithPrefix { + a.mu.RLock() + defer a.mu.RUnlock() + + var addrs []tcpip.AddressWithPrefix + for _, ep := range a.endpoints { + if !ep.GetKind().IsPermanent() { + continue + } + + addrs = append(addrs, ep.AddressWithPrefix()) + } + + return addrs +} + +// Cleanup forcefully leaves all groups and removes all permanent addresses. +func (a *AddressableEndpointState) Cleanup() { + a.mu.Lock() + defer a.mu.Unlock() + + for _, ep := range a.endpoints { + // removePermanentEndpointLocked returns *tcpip.ErrBadLocalAddress if ep is + // not a permanent address. + switch err := a.removePermanentEndpointLocked(ep, AddressRemovalInterfaceRemoved); err.(type) { + case nil, *tcpip.ErrBadLocalAddress: + default: + panic(fmt.Sprintf("unexpected error from removePermanentEndpointLocked(%s): %s", ep.addr, err)) + } + } +} + +var _ AddressEndpoint = (*addressState)(nil) + +// addressState holds state for an address. +type addressState struct { + addressableEndpointState *AddressableEndpointState + addr tcpip.AddressWithPrefix + subnet tcpip.Subnet + temporary bool + + // Lock ordering (from outer to inner lock ordering): + // + // AddressableEndpointState.mu + // addressState.mu + mu addressStateRWMutex + refs addressStateRefs + // checklocks:mu + kind AddressKind + // checklocks:mu + configType AddressConfigType + // lifetimes holds this address' lifetimes. + // + // Invariant: if lifetimes.deprecated is true, then lifetimes.PreferredUntil + // must be the zero value. Note that the converse does not need to be + // upheld! + // + // checklocks:mu + lifetimes AddressLifetimes + // The enclosing mutex must be write-locked before calling methods on the + // dispatcher. + // + // checklocks:mu + disp AddressDispatcher +} + +// AddressWithPrefix implements AddressEndpoint. +func (a *addressState) AddressWithPrefix() tcpip.AddressWithPrefix { + return a.addr +} + +// Subnet implements AddressEndpoint. +func (a *addressState) Subnet() tcpip.Subnet { + return a.subnet +} + +// GetKind implements AddressEndpoint. +func (a *addressState) GetKind() AddressKind { + a.mu.RLock() + defer a.mu.RUnlock() + return a.kind +} + +// SetKind implements AddressEndpoint. +func (a *addressState) SetKind(kind AddressKind) { + a.mu.Lock() + defer a.mu.Unlock() + + prevKind := a.kind + a.kind = kind + if kind == PermanentExpired { + a.notifyRemovedLocked(AddressRemovalManualAction) + } else if prevKind != kind && a.addressableEndpointState.networkEndpoint.Enabled() { + a.notifyChangedLocked() + } +} + +// notifyRemovedLocked notifies integrators of address removal. +// +// +checklocks:a.mu +func (a *addressState) notifyRemovedLocked(reason AddressRemovalReason) { + if disp := a.disp; disp != nil { + a.disp.OnRemoved(reason) + a.disp = nil + } +} + +func (a *addressState) remove(reason AddressRemovalReason) { + a.mu.Lock() + defer a.mu.Unlock() + + a.kind = PermanentExpired + a.notifyRemovedLocked(reason) +} + +// IsAssigned implements AddressEndpoint. +func (a *addressState) IsAssigned(allowExpired bool) bool { + switch kind := a.GetKind(); kind { + case PermanentTentative: + return false + case PermanentExpired: + return allowExpired + case Permanent, Temporary: + return true + default: + panic(fmt.Sprintf("address %s has unknown kind %d", a.AddressWithPrefix(), kind)) + } +} + +// IncRef implements AddressEndpoint. +func (a *addressState) TryIncRef() bool { + return a.refs.TryIncRef() +} + +// DecRef implements AddressEndpoint. +func (a *addressState) DecRef() { + a.addressableEndpointState.decAddressRef(a) +} + +// decRefMustNotFree decreases the reference count with the guarantee that the +// reference count will be greater than 0 after the decrement. +// +// Panics if the ref count is less than 2 after acquiring the lock in this +// function. +func (a *addressState) decRefMustNotFree() { + a.refs.DecRef(func() { + panic(fmt.Sprintf("cannot decrease addressState %s without freeing the endpoint", a.addr)) + }) +} + +// ConfigType implements AddressEndpoint. +func (a *addressState) ConfigType() AddressConfigType { + a.mu.RLock() + defer a.mu.RUnlock() + return a.configType +} + +// notifyChangedLocked notifies integrators of address property changes. +// +// +checklocks:a.mu +func (a *addressState) notifyChangedLocked() { + if a.disp == nil { + return + } + + state := AddressDisabled + if a.addressableEndpointState.networkEndpoint.Enabled() { + switch a.kind { + case Permanent: + state = AddressAssigned + case PermanentTentative: + state = AddressTentative + case Temporary, PermanentExpired: + return + default: + panic(fmt.Sprintf("unrecognized address kind = %d", a.kind)) + } + } + + a.disp.OnChanged(a.lifetimes, state) +} + +// SetDeprecated implements AddressEndpoint. +func (a *addressState) SetDeprecated(d bool) { + a.mu.Lock() + defer a.mu.Unlock() + + var changed bool + if a.lifetimes.Deprecated != d { + a.lifetimes.Deprecated = d + changed = true + } + if d { + a.lifetimes.PreferredUntil = tcpip.MonotonicTime{} + } + if changed { + a.notifyChangedLocked() + } +} + +// Deprecated implements AddressEndpoint. +func (a *addressState) Deprecated() bool { + a.mu.RLock() + defer a.mu.RUnlock() + return a.lifetimes.Deprecated +} + +// SetLifetimes implements AddressEndpoint. +func (a *addressState) SetLifetimes(lifetimes AddressLifetimes) { + a.mu.Lock() + defer a.mu.Unlock() + + lifetimes.sanitize() + + var changed bool + if a.lifetimes != lifetimes { + changed = true + } + a.lifetimes = lifetimes + if changed { + a.notifyChangedLocked() + } +} + +// Lifetimes implements AddressEndpoint. +func (a *addressState) Lifetimes() AddressLifetimes { + a.mu.RLock() + defer a.mu.RUnlock() + return a.lifetimes +} + +// Temporary implements AddressEndpoint. +func (a *addressState) Temporary() bool { + return a.temporary +} + +// RegisterDispatcher implements AddressEndpoint. +func (a *addressState) RegisterDispatcher(disp AddressDispatcher) { + a.mu.Lock() + defer a.mu.Unlock() + if disp != nil { + a.disp = disp + a.notifyChangedLocked() + } +} diff --git a/pkg/tcpip/stack/addressable_endpoint_state_mutex.go b/pkg/tcpip/stack/addressable_endpoint_state_mutex.go new file mode 100644 index 0000000..df42c81 --- /dev/null +++ b/pkg/tcpip/stack/addressable_endpoint_state_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type addressableEndpointStateRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var addressableEndpointStatelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type addressableEndpointStatelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) Lock() { + locking.AddGLock(addressableEndpointStateprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) NestedLock(i addressableEndpointStatelockNameIndex) { + locking.AddGLock(addressableEndpointStateprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(addressableEndpointStateprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) NestedUnlock(i addressableEndpointStatelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(addressableEndpointStateprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) RLock() { + locking.AddGLock(addressableEndpointStateprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(addressableEndpointStateprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *addressableEndpointStateRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var addressableEndpointStateprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func addressableEndpointStateinitLockNames() {} + +func init() { + addressableEndpointStateinitLockNames() + addressableEndpointStateprefixIndex = locking.NewMutexClass(reflect.TypeOf(addressableEndpointStateRWMutex{}), addressableEndpointStatelockNames) +} diff --git a/pkg/tcpip/stack/bridge.go b/pkg/tcpip/stack/bridge.go new file mode 100644 index 0000000..d3cc082 --- /dev/null +++ b/pkg/tcpip/stack/bridge.go @@ -0,0 +1,306 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +var _ NetworkLinkEndpoint = (*BridgeEndpoint)(nil) + +// +stateify savable +type bridgePort struct { + bridge *BridgeEndpoint + nic *nic +} + +// BridgeFDBKey is the MAC address of a device which a bridge port is associated with. +type BridgeFDBKey tcpip.LinkAddress + +// BridgeFDBEntry consists of all metadata for a FDB record. +type BridgeFDBEntry struct { + port *bridgePort +} + +// PortLinkAddress returns the mac address of the device that is bound to the bridge port. +func (e BridgeFDBEntry) PortLinkAddress() tcpip.LinkAddress { + if e.port == nil { + return "" + } + return e.port.nic.LinkAddress() +} + +// ParseHeader implements stack.LinkEndpoint. +func (p *bridgePort) ParseHeader(pkt *PacketBuffer) bool { + _, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize) + return ok +} + +// DeliverNetworkPacket implements stack.NetworkDispatcher. +func (p *bridgePort) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) { + bridge := p.bridge + eth := header.Ethernet(pkt.LinkHeader().Slice()) + updateFDB := false + bridge.mu.RLock() + // Add an entry at the bridge FDB, it maps a MAC address + // to a bridge port where the traffic is received when + // the MAC address is not multicast. + // Network packets that are sent to the learned MAC address + // will be forwarded to the bridge port that is stored in + // the FDB table. + sourceAddress := eth.SourceAddress() + if _, hasSourceFDB := bridge.fdbTable[BridgeFDBKey(sourceAddress)]; !header.IsMulticastEthernetAddress(sourceAddress) && !hasSourceFDB { + updateFDB = true + } + if entry, exist := bridge.fdbTable[BridgeFDBKey(eth.DestinationAddress())]; !exist { + // When no FDB entry is found, send the packet to all ports. + for _, port := range bridge.ports { + if p == port { + continue + } + newPkt := NewPacketBuffer(PacketBufferOptions{ + ReserveHeaderBytes: int(port.nic.MaxHeaderLength()), + Payload: pkt.ToBuffer(), + }) + port.nic.writeRawPacket(newPkt) + newPkt.DecRef() + } + } else if entry.port != p { + destPort := entry.port + newPkt := NewPacketBuffer(PacketBufferOptions{ + ReserveHeaderBytes: int(destPort.nic.MaxHeaderLength()), + Payload: pkt.ToBuffer(), + }) + destPort.nic.writeRawPacket(newPkt) + newPkt.DecRef() + } + + d := bridge.dispatcher + bridge.mu.RUnlock() + if updateFDB { + bridge.mu.Lock() + bridge.addFDBEntryLocked(eth.SourceAddress(), p, 0) + bridge.mu.Unlock() + } + if d != nil { + // The dispatcher may acquire Stack.mu in DeliverNetworkPacket(), which is + // ordered above bridge.mu. So call DeliverNetworkPacket() without holding + // bridge.mu to avoid circular locking. + d.DeliverNetworkPacket(protocol, pkt) + } +} + +func (p *bridgePort) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) { +} + +// NewBridgeEndpoint creates a new bridge endpoint. +func NewBridgeEndpoint(mtu uint32) *BridgeEndpoint { + b := &BridgeEndpoint{ + mtu: mtu, + addr: tcpip.GetRandMacAddr(), + } + b.ports = make(map[tcpip.NICID]*bridgePort) + b.fdbTable = make(map[BridgeFDBKey]BridgeFDBEntry) + return b +} + +// BridgeEndpoint is a bridge endpoint. +// +// +stateify savable +type BridgeEndpoint struct { + mu bridgeRWMutex `state:"nosave"` + // +checklocks:mu + ports map[tcpip.NICID]*bridgePort + // +checklocks:mu + dispatcher NetworkDispatcher + // +checklocks:mu + addr tcpip.LinkAddress + // +checklocks:mu + attached bool + // +checklocks:mu + mtu uint32 + // +checklocks:mu + fdbTable map[BridgeFDBKey]BridgeFDBEntry + maxHeaderLength atomicbitops.Uint32 +} + +// WritePackets implements stack.LinkEndpoint.WritePackets. +func (b *BridgeEndpoint) WritePackets(pkts PacketBufferList) (int, tcpip.Error) { + b.mu.RLock() + defer b.mu.RUnlock() + + pktsSlice := pkts.AsSlice() + n := len(pktsSlice) + for _, p := range b.ports { + for _, pkt := range pktsSlice { + // In order to properly loop back to the inbound side we must create a + // fresh packet that only contains the underlying payload with no headers + // or struct fields set. + newPkt := NewPacketBuffer(PacketBufferOptions{ + Payload: pkt.ToBuffer(), + ReserveHeaderBytes: int(p.nic.MaxHeaderLength()), + }) + newPkt.EgressRoute = pkt.EgressRoute + newPkt.NetworkProtocolNumber = pkt.NetworkProtocolNumber + p.nic.writePacket(newPkt) + newPkt.DecRef() + } + } + + return n, nil +} + +// AddNIC adds the specified NIC to the bridge. +func (b *BridgeEndpoint) AddNIC(n *nic) tcpip.Error { + b.mu.Lock() + defer b.mu.Unlock() + + port := &bridgePort{ + nic: n, + bridge: b, + } + n.NetworkLinkEndpoint.Attach(port) + b.ports[n.id] = port + + if b.maxHeaderLength.Load() < uint32(n.MaxHeaderLength()) { + b.maxHeaderLength.Store(uint32(n.MaxHeaderLength())) + } + + return nil +} + +// DelNIC remove the specified NIC from the bridge. +func (b *BridgeEndpoint) DelNIC(nic *nic) tcpip.Error { + b.mu.Lock() + defer b.mu.Unlock() + + port := b.ports[nic.id] + for k, e := range b.fdbTable { + if e.port == port { + delete(b.fdbTable, k) + } + } + delete(b.ports, nic.id) + nic.NetworkLinkEndpoint.Attach(nic) + return nil +} + +// MTU implements stack.LinkEndpoint.MTU. +func (b *BridgeEndpoint) MTU() uint32 { + b.mu.RLock() + defer b.mu.RUnlock() + if b.mtu > header.EthernetMinimumSize { + return b.mtu - header.EthernetMinimumSize + } + return 0 +} + +// SetMTU implements stack.LinkEndpoint.SetMTU. +func (b *BridgeEndpoint) SetMTU(mtu uint32) { + b.mu.Lock() + defer b.mu.Unlock() + b.mtu = mtu +} + +// MaxHeaderLength implements stack.LinkEndpoint. +func (b *BridgeEndpoint) MaxHeaderLength() uint16 { + return uint16(b.maxHeaderLength.Load()) +} + +// LinkAddress implements stack.LinkEndpoint.LinkAddress. +func (b *BridgeEndpoint) LinkAddress() tcpip.LinkAddress { + b.mu.RLock() + defer b.mu.RUnlock() + return b.addr +} + +// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. +func (b *BridgeEndpoint) SetLinkAddress(addr tcpip.LinkAddress) { + b.mu.Lock() + defer b.mu.Unlock() + b.addr = addr +} + +// Capabilities implements stack.LinkEndpoint.Capabilities. +func (b *BridgeEndpoint) Capabilities() LinkEndpointCapabilities { + return CapabilityRXChecksumOffload | CapabilitySaveRestore | CapabilityResolutionRequired +} + +// Attach implements stack.LinkEndpoint.Attach. +func (b *BridgeEndpoint) Attach(dispatcher NetworkDispatcher) { + b.mu.Lock() + defer b.mu.Unlock() + for _, p := range b.ports { + p.nic.Primary = nil + } + b.dispatcher = dispatcher + b.ports = make(map[tcpip.NICID]*bridgePort) + b.fdbTable = make(map[BridgeFDBKey]BridgeFDBEntry) +} + +// IsAttached implements stack.LinkEndpoint.IsAttached. +func (b *BridgeEndpoint) IsAttached() bool { + b.mu.RLock() + defer b.mu.RUnlock() + return b.dispatcher != nil +} + +// Wait implements stack.LinkEndpoint.Wait. +func (b *BridgeEndpoint) Wait() { +} + +// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType. +func (b *BridgeEndpoint) ARPHardwareType() header.ARPHardwareType { + return header.ARPHardwareEther +} + +// AddHeader implements stack.LinkEndpoint.AddHeader. +func (b *BridgeEndpoint) AddHeader(pkt *PacketBuffer) { +} + +// ParseHeader implements stack.LinkEndpoint.ParseHeader. +func (b *BridgeEndpoint) ParseHeader(*PacketBuffer) bool { + return true +} + +// Close implements stack.LinkEndpoint.Close. +func (b *BridgeEndpoint) Close() {} + +// SetOnCloseAction implements stack.LinkEndpoint.Close. +func (b *BridgeEndpoint) SetOnCloseAction(func()) {} + +// Add a new FDBEntry by learning. The learning happens when a packet +// is received by a bridge port, the bridge will use the port for the future +// deliveries to the NIC device. +// The addr is the key when it looks for the entry. +// +// +checklocks:b.mu +func (b *BridgeEndpoint) addFDBEntryLocked(addr tcpip.LinkAddress, source *bridgePort, flags uint64) bool { + // TODO(b/376924093): limit bridge FDB size. + b.fdbTable[BridgeFDBKey(addr)] = BridgeFDBEntry{ + port: source, + } + return true +} + +// FindFDBEntry find the FDB entry for the given address. If it doesn't exist, +// it will return an empty entry. +func (b *BridgeEndpoint) FindFDBEntry(addr tcpip.LinkAddress) BridgeFDBEntry { + b.mu.RLock() + defer b.mu.RUnlock() + return b.fdbTable[BridgeFDBKey(addr)] +} diff --git a/pkg/tcpip/stack/bridge_mutex.go b/pkg/tcpip/stack/bridge_mutex.go new file mode 100644 index 0000000..4ac0890 --- /dev/null +++ b/pkg/tcpip/stack/bridge_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type bridgeRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var bridgelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type bridgelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *bridgeRWMutex) Lock() { + locking.AddGLock(bridgeprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *bridgeRWMutex) NestedLock(i bridgelockNameIndex) { + locking.AddGLock(bridgeprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *bridgeRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(bridgeprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *bridgeRWMutex) NestedUnlock(i bridgelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(bridgeprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *bridgeRWMutex) RLock() { + locking.AddGLock(bridgeprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *bridgeRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(bridgeprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *bridgeRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *bridgeRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *bridgeRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var bridgeprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func bridgeinitLockNames() {} + +func init() { + bridgeinitLockNames() + bridgeprefixIndex = locking.NewMutexClass(reflect.TypeOf(bridgeRWMutex{}), bridgelockNames) +} diff --git a/pkg/tcpip/stack/bucket_mutex.go b/pkg/tcpip/stack/bucket_mutex.go new file mode 100644 index 0000000..fef6e33 --- /dev/null +++ b/pkg/tcpip/stack/bucket_mutex.go @@ -0,0 +1,97 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type bucketRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var bucketlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type bucketlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +const ( + bucketLockOthertuple = bucketlockNameIndex(0) +) + +// Lock locks m. +// +checklocksignore +func (m *bucketRWMutex) Lock() { + locking.AddGLock(bucketprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *bucketRWMutex) NestedLock(i bucketlockNameIndex) { + locking.AddGLock(bucketprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *bucketRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(bucketprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *bucketRWMutex) NestedUnlock(i bucketlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(bucketprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *bucketRWMutex) RLock() { + locking.AddGLock(bucketprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *bucketRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(bucketprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *bucketRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *bucketRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *bucketRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var bucketprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func bucketinitLockNames() { bucketlockNames = []string{"otherTuple"} } + +func init() { + bucketinitLockNames() + bucketprefixIndex = locking.NewMutexClass(reflect.TypeOf(bucketRWMutex{}), bucketlockNames) +} diff --git a/pkg/tcpip/stack/cleanup_endpoints_mutex.go b/pkg/tcpip/stack/cleanup_endpoints_mutex.go new file mode 100644 index 0000000..67a5bdb --- /dev/null +++ b/pkg/tcpip/stack/cleanup_endpoints_mutex.go @@ -0,0 +1,64 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type cleanupEndpointsMutex struct { + mu sync.Mutex +} + +var cleanupEndpointsprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var cleanupEndpointslockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type cleanupEndpointslockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *cleanupEndpointsMutex) Lock() { + locking.AddGLock(cleanupEndpointsprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *cleanupEndpointsMutex) NestedLock(i cleanupEndpointslockNameIndex) { + locking.AddGLock(cleanupEndpointsprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *cleanupEndpointsMutex) Unlock() { + locking.DelGLock(cleanupEndpointsprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *cleanupEndpointsMutex) NestedUnlock(i cleanupEndpointslockNameIndex) { + locking.DelGLock(cleanupEndpointsprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func cleanupEndpointsinitLockNames() {} + +func init() { + cleanupEndpointsinitLockNames() + cleanupEndpointsprefixIndex = locking.NewMutexClass(reflect.TypeOf(cleanupEndpointsMutex{}), cleanupEndpointslockNames) +} diff --git a/pkg/tcpip/stack/conn_mutex.go b/pkg/tcpip/stack/conn_mutex.go new file mode 100644 index 0000000..16ccbf4 --- /dev/null +++ b/pkg/tcpip/stack/conn_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type connRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var connlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type connlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *connRWMutex) Lock() { + locking.AddGLock(connprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *connRWMutex) NestedLock(i connlockNameIndex) { + locking.AddGLock(connprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *connRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(connprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *connRWMutex) NestedUnlock(i connlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(connprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *connRWMutex) RLock() { + locking.AddGLock(connprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *connRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(connprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *connRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *connRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *connRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var connprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func conninitLockNames() {} + +func init() { + conninitLockNames() + connprefixIndex = locking.NewMutexClass(reflect.TypeOf(connRWMutex{}), connlockNames) +} diff --git a/pkg/tcpip/stack/conn_track_mutex.go b/pkg/tcpip/stack/conn_track_mutex.go new file mode 100644 index 0000000..3e67080 --- /dev/null +++ b/pkg/tcpip/stack/conn_track_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type connTrackRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var connTracklockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type connTracklockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *connTrackRWMutex) Lock() { + locking.AddGLock(connTrackprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *connTrackRWMutex) NestedLock(i connTracklockNameIndex) { + locking.AddGLock(connTrackprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *connTrackRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(connTrackprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *connTrackRWMutex) NestedUnlock(i connTracklockNameIndex) { + m.mu.Unlock() + locking.DelGLock(connTrackprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *connTrackRWMutex) RLock() { + locking.AddGLock(connTrackprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *connTrackRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(connTrackprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *connTrackRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *connTrackRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *connTrackRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var connTrackprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func connTrackinitLockNames() {} + +func init() { + connTrackinitLockNames() + connTrackprefixIndex = locking.NewMutexClass(reflect.TypeOf(connTrackRWMutex{}), connTracklockNames) +} diff --git a/pkg/tcpip/stack/conntrack.go b/pkg/tcpip/stack/conntrack.go new file mode 100644 index 0000000..7ecf5c9 --- /dev/null +++ b/pkg/tcpip/stack/conntrack.go @@ -0,0 +1,1169 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "encoding/binary" + "fmt" + "math" + "math/rand" + "sync" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/hash/jenkins" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/transport/tcpconntrack" +) + +// Connection tracking is used to track and manipulate packets for NAT rules. +// The connection is created for a packet if it does not exist. Every +// connection contains two tuples (original and reply). The tuples are +// manipulated if there is a matching NAT rule. The packet is modified by +// looking at the tuples in each hook. +// +// Currently, only TCP tracking is supported. + +// Our hash table has 16K buckets. +const numBuckets = 1 << 14 + +const ( + establishedTimeout time.Duration = 5 * 24 * time.Hour + unestablishedTimeout time.Duration = 120 * time.Second +) + +// tuple holds a connection's identifying and manipulating data in one +// direction. It is immutable. +// +// +stateify savable +type tuple struct { + // tupleEntry is used to build an intrusive list of tuples. + tupleEntry + + // conn is the connection tracking entry this tuple belongs to. + conn *conn + + // reply is true iff the tuple's direction is opposite that of the first + // packet seen on the connection. + reply bool + + // tupleID is set at initialization and is immutable. + tupleID tupleID +} + +// tupleID uniquely identifies a trackable connection in one direction. +// +// +stateify savable +type tupleID struct { + srcAddr tcpip.Address + // The source port of a packet in the original direction is overloaded with + // the ident of an Echo Request packet. + // + // This also matches the behaviour of sending packets on Linux where the + // socket's source port value is used for the source port of outgoing packets + // for TCP/UDP and the ident field for outgoing Echo Requests on Ping sockets: + // + // IPv4: https://github.com/torvalds/linux/blob/c5c17547b778975b3d83a73c8d84e8fb5ecf3ba5/net/ipv4/ping.c#L810 + // IPv6: https://github.com/torvalds/linux/blob/c5c17547b778975b3d83a73c8d84e8fb5ecf3ba5/net/ipv6/ping.c#L133 + srcPortOrEchoRequestIdent uint16 + dstAddr tcpip.Address + // The opposite of srcPortOrEchoRequestIdent; the destination port of a packet + // in the reply direction is overloaded with the ident of an Echo Reply. + dstPortOrEchoReplyIdent uint16 + transProto tcpip.TransportProtocolNumber + netProto tcpip.NetworkProtocolNumber +} + +// reply creates the reply tupleID. +func (ti tupleID) reply() tupleID { + return tupleID{ + srcAddr: ti.dstAddr, + srcPortOrEchoRequestIdent: ti.dstPortOrEchoReplyIdent, + dstAddr: ti.srcAddr, + dstPortOrEchoReplyIdent: ti.srcPortOrEchoRequestIdent, + transProto: ti.transProto, + netProto: ti.netProto, + } +} + +type manipType int + +const ( + // manipNotPerformed indicates that NAT has not been performed. + manipNotPerformed manipType = iota + + // manipPerformed indicates that NAT was performed. + manipPerformed + + // manipPerformedNoop indicates that NAT was performed but it was a no-op. + manipPerformedNoop +) + +type finalizeResult uint32 + +const ( + // A finalizeResult must be explicitly set so we don't make use of the zero + // value. + _ finalizeResult = iota + + finalizeResultSuccess + finalizeResultConflict +) + +// conn is a tracked connection. +// +// +stateify savable +type conn struct { + ct *ConnTrack + + // original is the tuple in original direction. It is immutable. + original tuple + + // reply is the tuple in reply direction. + reply tuple + + // TODO(b/341946753): Restore when netstack is savable. + finalizeOnce sync.Once `state:"nosave"` + // Holds a finalizeResult. + finalizeResult atomicbitops.Uint32 + + mu connRWMutex `state:"nosave"` + // sourceManip indicates the source manipulation type. + // + // +checklocks:mu + sourceManip manipType + // destinationManip indicates the destination's manipulation type. + // + // +checklocks:mu + destinationManip manipType + + stateMu stateConnRWMutex `state:"nosave"` + // tcb is TCB control block. It is used to keep track of states + // of tcp connection. + // + // +checklocks:stateMu + tcb tcpconntrack.TCB + // lastUsed is the last time the connection saw a relevant packet, and + // is updated by each packet on the connection. + // + // +checklocks:stateMu + lastUsed tcpip.MonotonicTime +} + +// timedOut returns whether the connection timed out based on its state. +func (cn *conn) timedOut(now tcpip.MonotonicTime) bool { + cn.stateMu.RLock() + defer cn.stateMu.RUnlock() + if cn.tcb.State() == tcpconntrack.ResultAlive { + // Use the same default as Linux, which doesn't delete + // established connections for 5(!) days. + return now.Sub(cn.lastUsed) > establishedTimeout + } + // Use the same default as Linux, which lets connections in most states + // other than established remain for <= 120 seconds. + return now.Sub(cn.lastUsed) > unestablishedTimeout +} + +// update the connection tracking state. +func (cn *conn) update(pkt *PacketBuffer, reply bool) { + cn.stateMu.Lock() + defer cn.stateMu.Unlock() + + // Mark the connection as having been used recently so it isn't reaped. + cn.lastUsed = cn.ct.clock.NowMonotonic() + + if pkt.TransportProtocolNumber != header.TCPProtocolNumber { + return + } + + tcpHeader := header.TCP(pkt.TransportHeader().Slice()) + + // Update the state of tcb. tcb assumes it's always initialized on the + // client. However, we only need to know whether the connection is + // established or not, so the client/server distinction isn't important. + if cn.tcb.IsEmpty() { + cn.tcb.Init(tcpHeader, pkt.Data().Size()) + return + } + + if reply { + cn.tcb.UpdateStateReply(tcpHeader, pkt.Data().Size()) + } else { + cn.tcb.UpdateStateOriginal(tcpHeader, pkt.Data().Size()) + } +} + +// ConnTrack tracks all connections created for NAT rules. Most users are +// expected to only call handlePacket, insertRedirectConn, and maybeInsertNoop. +// +// ConnTrack keeps all connections in a slice of buckets, each of which holds a +// linked list of tuples. This gives us some desirable properties: +// - Each bucket has its own lock, lessening lock contention. +// - The slice is large enough that lists stay short (<10 elements on average). +// Thus traversal is fast. +// - During linked list traversal we reap expired connections. This amortizes +// the cost of reaping them and makes reapUnused faster. +// +// Locks are ordered by their location in the buckets slice. That is, a +// goroutine that locks buckets[i] can only lock buckets[j] s.t. i < j. +// +// +stateify savable +type ConnTrack struct { + // seed is a one-time random value initialized at stack startup + // and is used in the calculation of hash keys for the list of buckets. + // It is immutable. + seed uint32 + + // clock provides timing used to determine conntrack reapings. + clock tcpip.Clock + // TODO(b/341946753): Restore when netstack is savable. + rand *rand.Rand `state:"nosave"` + + mu connTrackRWMutex `state:"nosave"` + // mu protects the buckets slice, but not buckets' contents. Only take + // the write lock if you are modifying the slice or saving for S/R. + // + // +checklocks:mu + buckets []bucket +} + +// +stateify savable +type bucket struct { + mu bucketRWMutex `state:"nosave"` + // +checklocks:mu + tuples tupleList +} + +// A netAndTransHeadersFunc returns the network and transport headers found +// in an ICMP payload. The transport layer's payload will not be returned. +// +// May panic if the packet does not hold the transport header. +type netAndTransHeadersFunc func(icmpPayload []byte, minTransHdrLen int) (netHdr header.Network, transHdrBytes []byte) + +func v4NetAndTransHdr(icmpPayload []byte, minTransHdrLen int) (header.Network, []byte) { + netHdr := header.IPv4(icmpPayload) + // Do not use netHdr.Payload() as we might not hold the full packet + // in the ICMP error; Payload() panics if the buffer is smaller than + // the total length specified in the IPv4 header. + transHdr := icmpPayload[netHdr.HeaderLength():] + return netHdr, transHdr[:minTransHdrLen] +} + +func v6NetAndTransHdr(icmpPayload []byte, minTransHdrLen int) (header.Network, []byte) { + netHdr := header.IPv6(icmpPayload) + // Do not use netHdr.Payload() as we might not hold the full packet + // in the ICMP error; Payload() panics if the IP payload is smaller than + // the payload length specified in the IPv6 header. + transHdr := icmpPayload[header.IPv6MinimumSize:] + return netHdr, transHdr[:minTransHdrLen] +} + +func getEmbeddedNetAndTransHeaders(pkt *PacketBuffer, netHdrLength int, getNetAndTransHdr netAndTransHeadersFunc, transProto tcpip.TransportProtocolNumber) (header.Network, header.ChecksummableTransport, bool) { + switch transProto { + case header.TCPProtocolNumber: + if netAndTransHeader, ok := pkt.Data().PullUp(netHdrLength + header.TCPMinimumSize); ok { + netHeader, transHeaderBytes := getNetAndTransHdr(netAndTransHeader, header.TCPMinimumSize) + return netHeader, header.TCP(transHeaderBytes), true + } + case header.UDPProtocolNumber: + if netAndTransHeader, ok := pkt.Data().PullUp(netHdrLength + header.UDPMinimumSize); ok { + netHeader, transHeaderBytes := getNetAndTransHdr(netAndTransHeader, header.UDPMinimumSize) + return netHeader, header.UDP(transHeaderBytes), true + } + } + return nil, nil, false +} + +func getHeaders(pkt *PacketBuffer) (netHdr header.Network, transHdr header.Transport, isICMPError bool, ok bool) { + switch pkt.TransportProtocolNumber { + case header.TCPProtocolNumber: + if tcpHeader := header.TCP(pkt.TransportHeader().Slice()); len(tcpHeader) >= header.TCPMinimumSize { + return pkt.Network(), tcpHeader, false, true + } + return nil, nil, false, false + case header.UDPProtocolNumber: + if udpHeader := header.UDP(pkt.TransportHeader().Slice()); len(udpHeader) >= header.UDPMinimumSize { + return pkt.Network(), udpHeader, false, true + } + return nil, nil, false, false + case header.ICMPv4ProtocolNumber: + icmpHeader := header.ICMPv4(pkt.TransportHeader().Slice()) + if len(icmpHeader) < header.ICMPv4MinimumSize { + return nil, nil, false, false + } + + switch icmpType := icmpHeader.Type(); icmpType { + case header.ICMPv4Echo, header.ICMPv4EchoReply: + return pkt.Network(), icmpHeader, false, true + case header.ICMPv4DstUnreachable, header.ICMPv4TimeExceeded, header.ICMPv4ParamProblem: + default: + panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) + } + + h, ok := pkt.Data().PullUp(header.IPv4MinimumSize) + if !ok { + panic(fmt.Sprintf("should have a valid IPv4 packet; only have %d bytes, want at least %d bytes", pkt.Data().Size(), header.IPv4MinimumSize)) + } + + if header.IPv4(h).HeaderLength() > header.IPv4MinimumSize { + // TODO(https://gvisor.dev/issue/6765): Handle IPv4 options. + panic("should have dropped packets with IPv4 options") + } + + if netHdr, transHdr, ok := getEmbeddedNetAndTransHeaders(pkt, header.IPv4MinimumSize, v4NetAndTransHdr, pkt.tuple.tupleID.transProto); ok { + return netHdr, transHdr, true, true + } + return nil, nil, false, false + case header.ICMPv6ProtocolNumber: + icmpHeader := header.ICMPv6(pkt.TransportHeader().Slice()) + if len(icmpHeader) < header.ICMPv6MinimumSize { + return nil, nil, false, false + } + + switch icmpType := icmpHeader.Type(); icmpType { + case header.ICMPv6EchoRequest, header.ICMPv6EchoReply: + return pkt.Network(), icmpHeader, false, true + case header.ICMPv6DstUnreachable, header.ICMPv6PacketTooBig, header.ICMPv6TimeExceeded, header.ICMPv6ParamProblem: + default: + panic(fmt.Sprintf("unexpected ICMPv6 type = %d", icmpType)) + } + + h, ok := pkt.Data().PullUp(header.IPv6MinimumSize) + if !ok { + panic(fmt.Sprintf("should have a valid IPv6 packet; only have %d bytes, want at least %d bytes", pkt.Data().Size(), header.IPv6MinimumSize)) + } + + // We do not support extension headers in ICMP errors so the next header + // in the IPv6 packet should be a tracked protocol if we reach this point. + // + // TODO(https://gvisor.dev/issue/6789): Support extension headers. + transProto := pkt.tuple.tupleID.transProto + if got := header.IPv6(h).TransportProtocol(); got != transProto { + panic(fmt.Sprintf("got TransportProtocol() = %d, want = %d", got, transProto)) + } + + if netHdr, transHdr, ok := getEmbeddedNetAndTransHeaders(pkt, header.IPv6MinimumSize, v6NetAndTransHdr, transProto); ok { + return netHdr, transHdr, true, true + } + return nil, nil, false, false + default: + panic(fmt.Sprintf("unexpected transport protocol = %d", pkt.TransportProtocolNumber)) + } +} + +func getTupleIDForRegularPacket(netHdr header.Network, netProto tcpip.NetworkProtocolNumber, transHdr header.Transport, transProto tcpip.TransportProtocolNumber) tupleID { + return tupleID{ + srcAddr: netHdr.SourceAddress(), + srcPortOrEchoRequestIdent: transHdr.SourcePort(), + dstAddr: netHdr.DestinationAddress(), + dstPortOrEchoReplyIdent: transHdr.DestinationPort(), + transProto: transProto, + netProto: netProto, + } +} + +func getTupleIDForPacketInICMPError(pkt *PacketBuffer, getNetAndTransHdr netAndTransHeadersFunc, netProto tcpip.NetworkProtocolNumber, netLen int, transProto tcpip.TransportProtocolNumber) (tupleID, bool) { + if netHdr, transHdr, ok := getEmbeddedNetAndTransHeaders(pkt, netLen, getNetAndTransHdr, transProto); ok { + return tupleID{ + srcAddr: netHdr.DestinationAddress(), + srcPortOrEchoRequestIdent: transHdr.DestinationPort(), + dstAddr: netHdr.SourceAddress(), + dstPortOrEchoReplyIdent: transHdr.SourcePort(), + transProto: transProto, + netProto: netProto, + }, true + } + + return tupleID{}, false +} + +type getTupleIDDisposition int + +const ( + getTupleIDNotOK getTupleIDDisposition = iota + getTupleIDOKAndAllowNewConn + getTupleIDOKAndDontAllowNewConn +) + +func getTupleIDForEchoPacket(pkt *PacketBuffer, ident uint16, request bool) tupleID { + netHdr := pkt.Network() + tid := tupleID{ + srcAddr: netHdr.SourceAddress(), + dstAddr: netHdr.DestinationAddress(), + transProto: pkt.TransportProtocolNumber, + netProto: pkt.NetworkProtocolNumber, + } + + if request { + tid.srcPortOrEchoRequestIdent = ident + } else { + tid.dstPortOrEchoReplyIdent = ident + } + + return tid +} + +func getTupleID(pkt *PacketBuffer) (tupleID, getTupleIDDisposition) { + switch pkt.TransportProtocolNumber { + case header.TCPProtocolNumber: + if transHeader := header.TCP(pkt.TransportHeader().Slice()); len(transHeader) >= header.TCPMinimumSize { + return getTupleIDForRegularPacket(pkt.Network(), pkt.NetworkProtocolNumber, transHeader, pkt.TransportProtocolNumber), getTupleIDOKAndAllowNewConn + } + case header.UDPProtocolNumber: + if transHeader := header.UDP(pkt.TransportHeader().Slice()); len(transHeader) >= header.UDPMinimumSize { + return getTupleIDForRegularPacket(pkt.Network(), pkt.NetworkProtocolNumber, transHeader, pkt.TransportProtocolNumber), getTupleIDOKAndAllowNewConn + } + case header.ICMPv4ProtocolNumber: + icmp := header.ICMPv4(pkt.TransportHeader().Slice()) + if len(icmp) < header.ICMPv4MinimumSize { + return tupleID{}, getTupleIDNotOK + } + + switch icmp.Type() { + case header.ICMPv4Echo: + return getTupleIDForEchoPacket(pkt, icmp.Ident(), true /* request */), getTupleIDOKAndAllowNewConn + case header.ICMPv4EchoReply: + // Do not create a new connection in response to a reply packet as only + // the first packet of a connection should create a conntrack entry but + // a reply is never the first packet sent for a connection. + return getTupleIDForEchoPacket(pkt, icmp.Ident(), false /* request */), getTupleIDOKAndDontAllowNewConn + case header.ICMPv4DstUnreachable, header.ICMPv4TimeExceeded, header.ICMPv4ParamProblem: + default: + // Unsupported ICMP type for NAT-ing. + return tupleID{}, getTupleIDNotOK + } + + h, ok := pkt.Data().PullUp(header.IPv4MinimumSize) + if !ok { + return tupleID{}, getTupleIDNotOK + } + + ipv4 := header.IPv4(h) + if ipv4.HeaderLength() > header.IPv4MinimumSize { + // TODO(https://gvisor.dev/issue/6765): Handle IPv4 options. + return tupleID{}, getTupleIDNotOK + } + + if tid, ok := getTupleIDForPacketInICMPError(pkt, v4NetAndTransHdr, header.IPv4ProtocolNumber, header.IPv4MinimumSize, ipv4.TransportProtocol()); ok { + // Do not create a new connection in response to an ICMP error. + return tid, getTupleIDOKAndDontAllowNewConn + } + case header.ICMPv6ProtocolNumber: + icmp := header.ICMPv6(pkt.TransportHeader().Slice()) + if len(icmp) < header.ICMPv6MinimumSize { + return tupleID{}, getTupleIDNotOK + } + + switch icmp.Type() { + case header.ICMPv6EchoRequest: + return getTupleIDForEchoPacket(pkt, icmp.Ident(), true /* request */), getTupleIDOKAndAllowNewConn + case header.ICMPv6EchoReply: + // Do not create a new connection in response to a reply packet as only + // the first packet of a connection should create a conntrack entry but + // a reply is never the first packet sent for a connection. + return getTupleIDForEchoPacket(pkt, icmp.Ident(), false /* request */), getTupleIDOKAndDontAllowNewConn + case header.ICMPv6DstUnreachable, header.ICMPv6PacketTooBig, header.ICMPv6TimeExceeded, header.ICMPv6ParamProblem: + default: + return tupleID{}, getTupleIDNotOK + } + + h, ok := pkt.Data().PullUp(header.IPv6MinimumSize) + if !ok { + return tupleID{}, getTupleIDNotOK + } + + // TODO(https://gvisor.dev/issue/6789): Handle extension headers. + if tid, ok := getTupleIDForPacketInICMPError(pkt, v6NetAndTransHdr, header.IPv6ProtocolNumber, header.IPv6MinimumSize, header.IPv6(h).TransportProtocol()); ok { + // Do not create a new connection in response to an ICMP error. + return tid, getTupleIDOKAndDontAllowNewConn + } + } + + return tupleID{}, getTupleIDNotOK +} + +func (ct *ConnTrack) init() { + ct.mu.Lock() + defer ct.mu.Unlock() + ct.buckets = make([]bucket, numBuckets) +} + +// getConnAndUpdate attempts to get a connection or creates one if no +// connection exists for the packet and packet's protocol is trackable. +// +// If the packet's protocol is trackable, the connection's state is updated to +// match the contents of the packet. +func (ct *ConnTrack) getConnAndUpdate(pkt *PacketBuffer, skipChecksumValidation bool) *tuple { + // Get or (maybe) create a connection. + t := func() *tuple { + var allowNewConn bool + tid, res := getTupleID(pkt) + switch res { + case getTupleIDNotOK: + return nil + case getTupleIDOKAndAllowNewConn: + allowNewConn = true + case getTupleIDOKAndDontAllowNewConn: + allowNewConn = false + default: + panic(fmt.Sprintf("unhandled %[1]T = %[1]d", res)) + } + + // Just skip bad packets. They'll be rejected later by the appropriate + // protocol package. + switch pkt.TransportProtocolNumber { + case header.TCPProtocolNumber: + _, csumValid, ok := header.TCPValid( + header.TCP(pkt.TransportHeader().Slice()), + func() uint16 { return pkt.Data().Checksum() }, + uint16(pkt.Data().Size()), + tid.srcAddr, + tid.dstAddr, + pkt.RXChecksumValidated || skipChecksumValidation) + if !csumValid || !ok { + return nil + } + case header.UDPProtocolNumber: + lengthValid, csumValid := header.UDPValid( + header.UDP(pkt.TransportHeader().Slice()), + func() uint16 { return pkt.Data().Checksum() }, + uint16(pkt.Data().Size()), + pkt.NetworkProtocolNumber, + tid.srcAddr, + tid.dstAddr, + pkt.RXChecksumValidated || skipChecksumValidation) + if !lengthValid || !csumValid { + return nil + } + } + + ct.mu.RLock() + bkt := &ct.buckets[ct.bucket(tid)] + ct.mu.RUnlock() + + now := ct.clock.NowMonotonic() + if t := bkt.connForTID(tid, now); t != nil { + return t + } + + if !allowNewConn { + return nil + } + + bkt.mu.Lock() + defer bkt.mu.Unlock() + + // Make sure a connection wasn't added between when we last checked the + // bucket and acquired the bucket's write lock. + if t := bkt.connForTIDRLocked(tid, now); t != nil { + return t + } + + // This is the first packet we're seeing for the connection. Create an entry + // for this new connection. + conn := &conn{ + ct: ct, + original: tuple{tupleID: tid}, + reply: tuple{tupleID: tid.reply(), reply: true}, + lastUsed: now, + } + conn.original.conn = conn + conn.reply.conn = conn + + // For now, we only map an entry for the packet's original tuple as NAT may be + // performed on this connection. Until the packet goes through all the hooks + // and its final address/port is known, we cannot know what the response + // packet's addresses/ports will look like. + // + // This is okay because the destination cannot send its response until it + // receives the packet; the packet will only be received once all the hooks + // have been performed. + // + // See (*conn).finalize. + bkt.tuples.PushFront(&conn.original) + return &conn.original + }() + if t != nil { + t.conn.update(pkt, t.reply) + } + return t +} + +func (ct *ConnTrack) connForTID(tid tupleID) *tuple { + ct.mu.RLock() + bkt := &ct.buckets[ct.bucket(tid)] + ct.mu.RUnlock() + + return bkt.connForTID(tid, ct.clock.NowMonotonic()) +} + +func (bkt *bucket) connForTID(tid tupleID, now tcpip.MonotonicTime) *tuple { + bkt.mu.RLock() + defer bkt.mu.RUnlock() + return bkt.connForTIDRLocked(tid, now) +} + +// +checklocksread:bkt.mu +func (bkt *bucket) connForTIDRLocked(tid tupleID, now tcpip.MonotonicTime) *tuple { + for other := bkt.tuples.Front(); other != nil; other = other.Next() { + if tid == other.tupleID && !other.conn.timedOut(now) { + return other + } + } + return nil +} + +func (ct *ConnTrack) finalize(cn *conn) finalizeResult { + ct.mu.RLock() + buckets := ct.buckets + ct.mu.RUnlock() + + { + tid := cn.reply.tupleID + id := ct.bucketWithTableLength(tid, len(buckets)) + + bkt := &buckets[id] + bkt.mu.Lock() + t := bkt.connForTIDRLocked(tid, ct.clock.NowMonotonic()) + if t == nil { + bkt.tuples.PushFront(&cn.reply) + bkt.mu.Unlock() + return finalizeResultSuccess + } + bkt.mu.Unlock() + + if t.conn == cn { + // We already have an entry for the reply tuple. + // + // This can occur when the source address/port is the same as the + // destination address/port. In this scenario, tid == tid.reply(). + return finalizeResultSuccess + } + } + + // Another connection for the reply already exists. Remove the original and + // let the caller know we failed. + // + // TODO(https://gvisor.dev/issue/6850): Investigate handling this clash + // better. + + tid := cn.original.tupleID + id := ct.bucketWithTableLength(tid, len(buckets)) + bkt := &buckets[id] + bkt.mu.Lock() + defer bkt.mu.Unlock() + bkt.tuples.Remove(&cn.original) + return finalizeResultConflict +} + +func (cn *conn) getFinalizeResult() finalizeResult { + return finalizeResult(cn.finalizeResult.Load()) +} + +// finalize attempts to finalize the connection and returns true iff the +// connection was successfully finalized. +// +// If the connection failed to finalize, the caller should drop the packet +// associated with the connection. +// +// If multiple goroutines attempt to finalize at the same time, only one +// goroutine will perform the work to finalize the connection, but all +// goroutines will block until the finalizing goroutine finishes finalizing. +func (cn *conn) finalize() bool { + cn.finalizeOnce.Do(func() { + cn.finalizeResult.Store(uint32(cn.ct.finalize(cn))) + }) + + switch res := cn.getFinalizeResult(); res { + case finalizeResultSuccess: + return true + case finalizeResultConflict: + return false + default: + panic(fmt.Sprintf("unhandled result = %d", res)) + } +} + +// If NAT has not been configured for this connection, either mark the +// connection as configured for "no-op NAT", in the case of DNAT, or, in the +// case of SNAT, perform source port remapping so that source ports used by +// locally-generated traffic do not conflict with ports occupied by existing NAT +// bindings. +// +// Note that in the typical case this is also a no-op, because `snatAction` +// will do nothing if the original tuple is already unique. +func (cn *conn) maybePerformNoopNAT(pkt *PacketBuffer, hook Hook, r *Route, dnat bool) { + cn.mu.Lock() + var manip *manipType + if dnat { + manip = &cn.destinationManip + } else { + manip = &cn.sourceManip + } + if *manip != manipNotPerformed { + cn.mu.Unlock() + _ = cn.handlePacket(pkt, hook, r) + return + } + if dnat { + *manip = manipPerformedNoop + cn.mu.Unlock() + _ = cn.handlePacket(pkt, hook, r) + return + } + cn.mu.Unlock() + + // At this point, we know that NAT has not yet been performed on this + // connection, and the DNAT case has been handled with a no-op. For SNAT, we + // simply perform source port remapping to ensure that source ports for + // locally generated traffic do not clash with ports used by existing NAT + // bindings. + _, _ = snatAction(pkt, hook, r, 0, tcpip.Address{}, true /* changePort */, false /* changeAddress */) +} + +type portOrIdentRange struct { + start uint16 + size uint32 +} + +// performNAT setups up the connection for the specified NAT and rewrites the +// packet. +// +// If NAT has already been performed on the connection, then the packet will +// be rewritten with the NAT performed on the connection, ignoring the passed +// address and port range. +// +// Generally, only the first packet of a connection reaches this method; other +// packets will be manipulated without needing to modify the connection. +func (cn *conn) performNAT(pkt *PacketBuffer, hook Hook, r *Route, portsOrIdents portOrIdentRange, natAddress tcpip.Address, dnat, changePort, changeAddress bool) { + lastPortOrIdent := func() uint16 { + lastPortOrIdent := uint32(portsOrIdents.start) + portsOrIdents.size - 1 + if lastPortOrIdent > math.MaxUint16 { + panic(fmt.Sprintf("got lastPortOrIdent = %d, want <= MaxUint16(=%d); portsOrIdents=%#v", lastPortOrIdent, math.MaxUint16, portsOrIdents)) + } + return uint16(lastPortOrIdent) + }() + + // Make sure the packet is re-written after performing NAT. + defer func() { + // handlePacket returns true if the packet may skip the NAT table as the + // connection is already NATed, but if we reach this point we must be in the + // NAT table, so the return value is useless for us. + _ = cn.handlePacket(pkt, hook, r) + }() + + cn.mu.Lock() + defer cn.mu.Unlock() + + var manip *manipType + var address *tcpip.Address + var portOrIdent *uint16 + if dnat { + manip = &cn.destinationManip + address = &cn.reply.tupleID.srcAddr + portOrIdent = &cn.reply.tupleID.srcPortOrEchoRequestIdent + } else { + manip = &cn.sourceManip + address = &cn.reply.tupleID.dstAddr + portOrIdent = &cn.reply.tupleID.dstPortOrEchoReplyIdent + } + + if *manip != manipNotPerformed { + return + } + *manip = manipPerformed + if changeAddress { + *address = natAddress + } + + // Everything below here is port-fiddling. + if !changePort { + return + } + + // Does the current port/ident fit in the range? + if portsOrIdents.start <= *portOrIdent && *portOrIdent <= lastPortOrIdent { + // Yes, is the current reply tuple unique? + // + // Or, does the reply tuple refer to the same connection as the current one that + // we are NATing? This would apply, for example, to a self-connected socket, + // where the original and reply tuples are identical. + other := cn.ct.connForTID(cn.reply.tupleID) + if other == nil || other.conn == cn { + // Yes! No need to change the port. + return + } + } + + // Try our best to find a port/ident that results in a unique reply tuple. + // + // We limit the number of attempts to find a unique tuple to not waste a lot + // of time looking for a unique tuple. + // + // Matches linux behaviour introduced in + // https://github.com/torvalds/linux/commit/a504b703bb1da526a01593da0e4be2af9d9f5fa8. + const maxAttemptsForInitialRound uint32 = 128 + const minAttemptsToContinue = 16 + + allowedInitialAttempts := maxAttemptsForInitialRound + if allowedInitialAttempts > portsOrIdents.size { + allowedInitialAttempts = portsOrIdents.size + } + + for maxAttempts := allowedInitialAttempts; ; maxAttempts /= 2 { + // Start reach round with a random initial port/ident offset. + randOffset := cn.ct.rand.Uint32() + + for i := uint32(0); i < maxAttempts; i++ { + newPortOrIdentU32 := uint32(portsOrIdents.start) + (randOffset+i)%portsOrIdents.size + if newPortOrIdentU32 > math.MaxUint16 { + panic(fmt.Sprintf("got newPortOrIdentU32 = %d, want <= MaxUint16(=%d); portsOrIdents=%#v, randOffset=%d", newPortOrIdentU32, math.MaxUint16, portsOrIdents, randOffset)) + } + + *portOrIdent = uint16(newPortOrIdentU32) + + if other := cn.ct.connForTID(cn.reply.tupleID); other == nil { + // We found a unique tuple! + return + } + } + + if maxAttempts == portsOrIdents.size { + // We already tried all the ports/idents in the range so no need to keep + // trying. + return + } + + if maxAttempts < minAttemptsToContinue { + return + } + } + + // We did not find a unique tuple, use the last used port anyways. + // TODO(https://gvisor.dev/issue/6850): Handle not finding a unique tuple + // better (e.g. remove the connection and drop the packet). +} + +// handlePacket attempts to handle a packet and perform NAT if the connection +// has had NAT performed on it. +// +// Returns true if the packet can skip the NAT table. +func (cn *conn) handlePacket(pkt *PacketBuffer, hook Hook, rt *Route) bool { + netHdr, transHdr, isICMPError, ok := getHeaders(pkt) + if !ok { + return false + } + + fullChecksum := false + updatePseudoHeader := false + natDone := &pkt.snatDone + dnat := false + switch hook { + case Prerouting: + // Packet came from outside the stack so it must have a checksum set + // already. + fullChecksum = true + updatePseudoHeader = true + + natDone = &pkt.dnatDone + dnat = true + case Input: + case Forward: + panic("should not handle packet in the forwarding hook") + case Output: + natDone = &pkt.dnatDone + dnat = true + fallthrough + case Postrouting: + if pkt.TransportProtocolNumber == header.TCPProtocolNumber && pkt.GSOOptions.Type != GSONone && pkt.GSOOptions.NeedsCsum { + updatePseudoHeader = true + } else if rt.RequiresTXTransportChecksum() { + fullChecksum = true + updatePseudoHeader = true + } + default: + panic(fmt.Sprintf("unrecognized hook = %d", hook)) + } + + if *natDone { + panic(fmt.Sprintf("packet already had NAT(dnat=%t) performed at hook=%s; pkt=%#v", dnat, hook, pkt)) + } + + // TODO(gvisor.dev/issue/5748): TCP checksums on inbound packets should be + // validated if checksum offloading is off. It may require IP defrag if the + // packets are fragmented. + + reply := pkt.tuple.reply + + tid, manip := func() (tupleID, manipType) { + cn.mu.RLock() + defer cn.mu.RUnlock() + + if reply { + tid := cn.original.tupleID + + if dnat { + return tid, cn.sourceManip + } + return tid, cn.destinationManip + } + + tid := cn.reply.tupleID + if dnat { + return tid, cn.destinationManip + } + return tid, cn.sourceManip + }() + switch manip { + case manipNotPerformed: + return false + case manipPerformedNoop: + *natDone = true + return true + case manipPerformed: + default: + panic(fmt.Sprintf("unhandled manip = %d", manip)) + } + + newPort := tid.dstPortOrEchoReplyIdent + newAddr := tid.dstAddr + if dnat { + newPort = tid.srcPortOrEchoRequestIdent + newAddr = tid.srcAddr + } + + rewritePacket( + netHdr, + transHdr, + !dnat != isICMPError, + fullChecksum, + updatePseudoHeader, + newPort, + newAddr, + ) + + *natDone = true + + if !isICMPError { + return true + } + + // We performed NAT on (erroneous) packet that triggered an ICMP response, but + // not the ICMP packet itself. + switch pkt.TransportProtocolNumber { + case header.ICMPv4ProtocolNumber: + icmp := header.ICMPv4(pkt.TransportHeader().Slice()) + // TODO(https://gvisor.dev/issue/6788): Incrementally update ICMP checksum. + icmp.SetChecksum(0) + icmp.SetChecksum(header.ICMPv4Checksum(icmp, pkt.Data().Checksum())) + + network := header.IPv4(pkt.NetworkHeader().Slice()) + if dnat { + network.SetDestinationAddressWithChecksumUpdate(tid.srcAddr) + } else { + network.SetSourceAddressWithChecksumUpdate(tid.dstAddr) + } + case header.ICMPv6ProtocolNumber: + network := header.IPv6(pkt.NetworkHeader().Slice()) + srcAddr := network.SourceAddress() + dstAddr := network.DestinationAddress() + if dnat { + dstAddr = tid.srcAddr + } else { + srcAddr = tid.dstAddr + } + + icmp := header.ICMPv6(pkt.TransportHeader().Slice()) + // TODO(https://gvisor.dev/issue/6788): Incrementally update ICMP checksum. + icmp.SetChecksum(0) + payload := pkt.Data() + icmp.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmp, + Src: srcAddr, + Dst: dstAddr, + PayloadCsum: payload.Checksum(), + PayloadLen: payload.Size(), + })) + + if dnat { + network.SetDestinationAddress(dstAddr) + } else { + network.SetSourceAddress(srcAddr) + } + } + + return true +} + +// bucket gets the conntrack bucket for a tupleID. +// +checklocksread:ct.mu +func (ct *ConnTrack) bucket(id tupleID) int { + return ct.bucketWithTableLength(id, len(ct.buckets)) +} + +func (ct *ConnTrack) bucketWithTableLength(id tupleID, tableLength int) int { + h := jenkins.Sum32(ct.seed) + h.Write(id.srcAddr.AsSlice()) + h.Write(id.dstAddr.AsSlice()) + shortBuf := make([]byte, 2) + binary.LittleEndian.PutUint16(shortBuf, id.srcPortOrEchoRequestIdent) + h.Write([]byte(shortBuf)) + binary.LittleEndian.PutUint16(shortBuf, id.dstPortOrEchoReplyIdent) + h.Write([]byte(shortBuf)) + binary.LittleEndian.PutUint16(shortBuf, uint16(id.transProto)) + h.Write([]byte(shortBuf)) + binary.LittleEndian.PutUint16(shortBuf, uint16(id.netProto)) + h.Write([]byte(shortBuf)) + return int(h.Sum32()) % tableLength +} + +// reapUnused deletes timed out entries from the conntrack map. The rules for +// reaping are: +// - Each call to reapUnused traverses a fraction of the conntrack table. +// Specifically, it traverses len(ct.buckets)/fractionPerReaping. +// - After reaping, reapUnused decides when it should next run based on the +// ratio of expired connections to examined connections. If the ratio is +// greater than maxExpiredPct, it schedules the next run quickly. Otherwise it +// slightly increases the interval between runs. +// - maxFullTraversal caps the time it takes to traverse the entire table. +// +// reapUnused returns the next bucket that should be checked and the time after +// which it should be called again. +func (ct *ConnTrack) reapUnused(start int, prevInterval time.Duration) (int, time.Duration) { + const fractionPerReaping = 128 + const maxExpiredPct = 50 + const maxFullTraversal = 60 * time.Second + const minInterval = 10 * time.Millisecond + const maxInterval = maxFullTraversal / fractionPerReaping + + now := ct.clock.NowMonotonic() + checked := 0 + expired := 0 + var idx int + ct.mu.RLock() + defer ct.mu.RUnlock() + for i := 0; i < len(ct.buckets)/fractionPerReaping; i++ { + idx = (i + start) % len(ct.buckets) + bkt := &ct.buckets[idx] + bkt.mu.Lock() + for tuple := bkt.tuples.Front(); tuple != nil; { + // reapTupleLocked updates tuple's next pointer so we grab it here. + nextTuple := tuple.Next() + + checked++ + if ct.reapTupleLocked(tuple, idx, bkt, now) { + expired++ + } + + tuple = nextTuple + } + bkt.mu.Unlock() + } + // We already checked buckets[idx]. + idx++ + + // If half or more of the connections are expired, the table has gotten + // stale. Reschedule quickly. + expiredPct := 0 + if checked != 0 { + expiredPct = expired * 100 / checked + } + if expiredPct > maxExpiredPct { + return idx, minInterval + } + if interval := prevInterval + minInterval; interval <= maxInterval { + // Increment the interval between runs. + return idx, interval + } + // We've hit the maximum interval. + return idx, maxInterval +} + +// reapTupleLocked tries to remove tuple and its reply from the table. It +// returns whether the tuple's connection has timed out. +// +// Precondition: ct.mu is read locked and bkt.mu is write locked. +// +checklocksread:ct.mu +// +checklocks:bkt.mu +func (ct *ConnTrack) reapTupleLocked(reapingTuple *tuple, bktID int, bkt *bucket, now tcpip.MonotonicTime) bool { + if !reapingTuple.conn.timedOut(now) { + return false + } + + var otherTuple *tuple + if reapingTuple.reply { + otherTuple = &reapingTuple.conn.original + } else { + otherTuple = &reapingTuple.conn.reply + } + + otherTupleBktID := ct.bucket(otherTuple.tupleID) + replyTupleInserted := reapingTuple.conn.getFinalizeResult() == finalizeResultSuccess + + // To maintain lock order, we can only reap both tuples if the tuple for the + // other direction appears later in the table. + if bktID > otherTupleBktID && replyTupleInserted { + return true + } + + bkt.tuples.Remove(reapingTuple) + + if !replyTupleInserted { + // The other tuple is the reply which has not yet been inserted. + return true + } + + // Reap the other connection. + if bktID == otherTupleBktID { + // Don't re-lock if both tuples are in the same bucket. + bkt.tuples.Remove(otherTuple) + } else { + otherTupleBkt := &ct.buckets[otherTupleBktID] + otherTupleBkt.mu.NestedLock(bucketLockOthertuple) + otherTupleBkt.tuples.Remove(otherTuple) + otherTupleBkt.mu.NestedUnlock(bucketLockOthertuple) + } + + return true +} + +func (ct *ConnTrack) originalDst(epID TransportEndpointID, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber) (tcpip.Address, uint16, tcpip.Error) { + // Lookup the connection. The reply's original destination + // describes the original address. + tid := tupleID{ + srcAddr: epID.LocalAddress, + srcPortOrEchoRequestIdent: epID.LocalPort, + dstAddr: epID.RemoteAddress, + dstPortOrEchoReplyIdent: epID.RemotePort, + transProto: transProto, + netProto: netProto, + } + t := ct.connForTID(tid) + if t == nil { + // Not a tracked connection. + return tcpip.Address{}, 0, &tcpip.ErrNotConnected{} + } + + t.conn.mu.RLock() + defer t.conn.mu.RUnlock() + if t.conn.destinationManip == manipNotPerformed { + // Unmanipulated destination. + return tcpip.Address{}, 0, &tcpip.ErrInvalidOptionValue{} + } + + id := t.conn.original.tupleID + return id.dstAddr, id.dstPortOrEchoReplyIdent, nil +} diff --git a/pkg/tcpip/stack/endpoints_by_nic_mutex.go b/pkg/tcpip/stack/endpoints_by_nic_mutex.go new file mode 100644 index 0000000..dee401b --- /dev/null +++ b/pkg/tcpip/stack/endpoints_by_nic_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointsByNICRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointsByNIClockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointsByNIClockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointsByNICRWMutex) Lock() { + locking.AddGLock(endpointsByNICprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointsByNICRWMutex) NestedLock(i endpointsByNIClockNameIndex) { + locking.AddGLock(endpointsByNICprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointsByNICRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointsByNICprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointsByNICRWMutex) NestedUnlock(i endpointsByNIClockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointsByNICprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointsByNICRWMutex) RLock() { + locking.AddGLock(endpointsByNICprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointsByNICRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointsByNICprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointsByNICRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointsByNICRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointsByNICRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointsByNICprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointsByNICinitLockNames() {} + +func init() { + endpointsByNICinitLockNames() + endpointsByNICprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointsByNICRWMutex{}), endpointsByNIClockNames) +} diff --git a/pkg/tcpip/stack/gro/gro.go b/pkg/tcpip/stack/gro/gro.go new file mode 100644 index 0000000..55a11ed --- /dev/null +++ b/pkg/tcpip/stack/gro/gro.go @@ -0,0 +1,603 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gro implements generic receive offload. +package gro + +import ( + "bytes" + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// There is room for improvement to the GRO engine: +// - We should save those headers in +// PacketBuffers so they don't have to be re-parsed later. +// - We still see the occasional SACK block in the zero-loss +// benchmark, which should not happen. +// - Some dispatchers, e.g. XDP and RecvMmsg, can receive +// multiple packets at a time. Even if the GRO interval is 0, there is an +// opportunity for coalescing. +// - We could pass a packet list up the stack to reduce traversals up the +// stack. + +const ( + // groNBuckets is the number of GRO buckets. + groNBuckets = 8 + + groNBucketsMask = groNBuckets - 1 + + // groBucketSize is the size of each GRO bucket. + groBucketSize = 8 + + // groMaxPacketSize is the maximum size of a GRO'd packet. + groMaxPacketSize = 1 << 16 // 65KB. +) + +// A groBucket holds packets that are undergoing GRO. +// +// +stateify savable +type groBucket struct { + // count is the number of packets in the bucket. + count int + + // packets is the linked list of packets. + packets groPacketList + + // packetsPrealloc and allocIdxs are used to preallocate and reuse + // groPacket structs and avoid allocation. + packetsPrealloc [groBucketSize]groPacket + + allocIdxs [groBucketSize]int +} + +func (gb *groBucket) full() bool { + return gb.count == groBucketSize +} + +// insert inserts pkt into the bucket. +func (gb *groBucket) insert(pkt *stack.PacketBuffer, ipHdr []byte, tcpHdr header.TCP) { + groPkt := &gb.packetsPrealloc[gb.allocIdxs[gb.count]] + *groPkt = groPacket{ + pkt: pkt, + ipHdr: ipHdr, + tcpHdr: tcpHdr, + initialLength: pkt.Data().Size(), // pkt.Data() contains network header. + idx: groPkt.idx, + } + gb.count++ + gb.packets.PushBack(groPkt) +} + +// removeOldest removes the oldest packet from gb and returns the contained +// PacketBuffer. gb must not be empty. +func (gb *groBucket) removeOldest() *stack.PacketBuffer { + pkt := gb.packets.Front() + gb.packets.Remove(pkt) + gb.count-- + gb.allocIdxs[gb.count] = pkt.idx + ret := pkt.pkt + pkt.reset() + return ret +} + +// removeOne removes a packet from gb. It also resets pkt to its zero value. +func (gb *groBucket) removeOne(pkt *groPacket) { + gb.packets.Remove(pkt) + gb.count-- + gb.allocIdxs[gb.count] = pkt.idx + pkt.reset() +} + +// findGROPacket4 returns the groPkt that matches ipHdr and tcpHdr, or nil if +// none exists. It also returns whether the groPkt should be flushed based on +// differences between the two headers. +func (gb *groBucket) findGROPacket4(pkt *stack.PacketBuffer, ipHdr header.IPv4, tcpHdr header.TCP) (*groPacket, bool) { + for groPkt := gb.packets.Front(); groPkt != nil; groPkt = groPkt.Next() { + // Do the addresses match? + groIPHdr := header.IPv4(groPkt.ipHdr) + if ipHdr.SourceAddress() != groIPHdr.SourceAddress() || ipHdr.DestinationAddress() != groIPHdr.DestinationAddress() { + continue + } + + // Do the ports match? + if tcpHdr.SourcePort() != groPkt.tcpHdr.SourcePort() || tcpHdr.DestinationPort() != groPkt.tcpHdr.DestinationPort() { + continue + } + + // We've found a packet of the same flow. + + // IP checks. + TOS, _ := ipHdr.TOS() + groTOS, _ := groIPHdr.TOS() + if ipHdr.TTL() != groIPHdr.TTL() || TOS != groTOS { + return groPkt, true + } + + // TCP checks. + if shouldFlushTCP(groPkt, tcpHdr) { + return groPkt, true + } + + // There's an upper limit on coalesced packet size. + if pkt.Data().Size()-header.IPv4MinimumSize-int(tcpHdr.DataOffset())+groPkt.pkt.Data().Size() >= groMaxPacketSize { + return groPkt, true + } + + return groPkt, false + } + + return nil, false +} + +// findGROPacket6 returns the groPkt that matches ipHdr and tcpHdr, or nil if +// none exists. It also returns whether the groPkt should be flushed based on +// differences between the two headers. +func (gb *groBucket) findGROPacket6(pkt *stack.PacketBuffer, ipHdr header.IPv6, tcpHdr header.TCP) (*groPacket, bool) { + for groPkt := gb.packets.Front(); groPkt != nil; groPkt = groPkt.Next() { + // Do the addresses match? + groIPHdr := header.IPv6(groPkt.ipHdr) + if ipHdr.SourceAddress() != groIPHdr.SourceAddress() || ipHdr.DestinationAddress() != groIPHdr.DestinationAddress() { + continue + } + + // Need to check that headers are the same except: + // - Traffic class, a difference of which causes a flush. + // - Hop limit, a difference of which causes a flush. + // - Length, which is checked later. + // - Version, which is checked by an earlier call to IsValid(). + trafficClass, flowLabel := ipHdr.TOS() + groTrafficClass, groFlowLabel := groIPHdr.TOS() + if flowLabel != groFlowLabel || ipHdr.NextHeader() != groIPHdr.NextHeader() { + continue + } + // Unlike IPv4, IPv6 packets with extension headers can be coalesced. + if !bytes.Equal(ipHdr[header.IPv6MinimumSize:], groIPHdr[header.IPv6MinimumSize:]) { + continue + } + + // Do the ports match? + if tcpHdr.SourcePort() != groPkt.tcpHdr.SourcePort() || tcpHdr.DestinationPort() != groPkt.tcpHdr.DestinationPort() { + continue + } + + // We've found a packet of the same flow. + + // TCP checks. + if shouldFlushTCP(groPkt, tcpHdr) { + return groPkt, true + } + + // Do the traffic class and hop limit match? + if trafficClass != groTrafficClass || ipHdr.HopLimit() != groIPHdr.HopLimit() { + return groPkt, true + } + + // This limit is artificial for IPv6 -- we could allow even + // larger packets via jumbograms. + if pkt.Data().Size()-len(ipHdr)-int(tcpHdr.DataOffset())+groPkt.pkt.Data().Size() >= groMaxPacketSize { + return groPkt, true + } + + return groPkt, false + } + + return nil, false +} + +func (gb *groBucket) found(gd *GRO, groPkt *groPacket, flushGROPkt bool, pkt *stack.PacketBuffer, ipHdr []byte, tcpHdr header.TCP, updateIPHdr func([]byte, int)) { + // Flush groPkt or merge the packets. + pktSize := pkt.Data().Size() + flags := tcpHdr.Flags() + dataOff := tcpHdr.DataOffset() + tcpPayloadSize := pkt.Data().Size() - len(ipHdr) - int(dataOff) + if flushGROPkt { + // Flush the existing GRO packet. + pkt := groPkt.pkt + gb.removeOne(groPkt) + gd.handlePacket(pkt) + pkt.DecRef() + groPkt = nil + } else if groPkt != nil { + // Merge pkt in to GRO packet. + pkt.Data().TrimFront(len(ipHdr) + int(dataOff)) + groPkt.pkt.Data().Merge(pkt.Data()) + // Update the IP total length. + updateIPHdr(groPkt.ipHdr, tcpPayloadSize) + // Add flags from the packet to the GRO packet. + groPkt.tcpHdr.SetFlags(uint8(groPkt.tcpHdr.Flags() | (flags & (header.TCPFlagFin | header.TCPFlagPsh)))) + + pkt = nil + } + + // Flush if the packet isn't the same size as the previous packets or + // if certain flags are set. The reason for checking size equality is: + // - If the packet is smaller than the others, this is likely the end + // of some message. Peers will send MSS-sized packets until they have + // insufficient data to do so. + // - If the packet is larger than the others, this packet is either + // malformed, a local GSO packet, or has already been handled by host + // GRO. + flush := header.TCPFlags(flags)&(header.TCPFlagUrg|header.TCPFlagPsh|header.TCPFlagRst|header.TCPFlagSyn|header.TCPFlagFin) != 0 + flush = flush || tcpPayloadSize == 0 + if groPkt != nil { + flush = flush || pktSize != groPkt.initialLength + } + + switch { + case flush && groPkt != nil: + // A merge occurred and we need to flush groPkt. + pkt := groPkt.pkt + gb.removeOne(groPkt) + gd.handlePacket(pkt) + pkt.DecRef() + case flush && groPkt == nil: + // No merge occurred and the incoming packet needs to be flushed. + gd.handlePacket(pkt) + case !flush && groPkt == nil: + // New flow and we don't need to flush. Insert pkt into GRO. + if gb.full() { + // Head is always the oldest packet + toFlush := gb.removeOldest() + gb.insert(pkt.IncRef(), ipHdr, tcpHdr) + gd.handlePacket(toFlush) + toFlush.DecRef() + } else { + gb.insert(pkt.IncRef(), ipHdr, tcpHdr) + } + default: + // A merge occurred and we don't need to flush anything. + } +} + +// A groPacket is packet undergoing GRO. It may be several packets coalesced +// together. +// +// +stateify savable +type groPacket struct { + // groPacketEntry is an intrusive list. + groPacketEntry + + // pkt is the coalesced packet. + pkt *stack.PacketBuffer + + // ipHdr is the IP (v4 or v6) header for the coalesced packet. + ipHdr []byte + + // tcpHdr is the TCP header for the coalesced packet. + tcpHdr header.TCP + + // initialLength is the length of the first packet in the flow. It is + // used as a best-effort guess at MSS: senders will send MSS-sized + // packets until they run out of data, so we coalesce as long as + // packets are the same size. + initialLength int + + // idx is the groPacket's index in its bucket packetsPrealloc. It is + // immutable. + idx int +} + +// reset resets all mutable fields of the groPacket. +func (pk *groPacket) reset() { + *pk = groPacket{ + idx: pk.idx, + } +} + +// payloadSize is the payload size of the coalesced packet, which does not +// include the network or transport headers. +func (pk *groPacket) payloadSize() int { + return pk.pkt.Data().Size() - len(pk.ipHdr) - int(pk.tcpHdr.DataOffset()) +} + +// GRO coalesces incoming packets to increase throughput. +// +// +stateify savable +type GRO struct { + enabled bool + buckets [groNBuckets]groBucket + + Dispatcher stack.NetworkDispatcher +} + +// Init initializes GRO. +func (gd *GRO) Init(enabled bool) { + gd.enabled = enabled + for i := range gd.buckets { + bucket := &gd.buckets[i] + for j := range bucket.packetsPrealloc { + bucket.allocIdxs[j] = j + bucket.packetsPrealloc[j].idx = j + } + } +} + +// Enqueue the packet in GRO. This does not flush packets; Flush() must be +// called explicitly for that. +// +// pkt.NetworkProtocolNumber and pkt.RXChecksumValidated must be set. +func (gd *GRO) Enqueue(pkt *stack.PacketBuffer) { + if !gd.enabled { + gd.handlePacket(pkt) + return + } + + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + gd.dispatch4(pkt) + case header.IPv6ProtocolNumber: + gd.dispatch6(pkt) + default: + gd.handlePacket(pkt) + } +} + +func (gd *GRO) dispatch4(pkt *stack.PacketBuffer) { + // Immediately get the IPv4 and TCP headers. We need a way to hash the + // packet into its bucket, which requires addresses and ports. Linux + // simply gets a hash passed by hardware, but we're not so lucky. + + // We only GRO TCP packets. The check for the transport protocol number + // is done below so that we can PullUp both the IP and TCP headers + // together. + hdrBytes, ok := pkt.Data().PullUp(header.IPv4MinimumSize + header.TCPMinimumSize) + if !ok { + gd.handlePacket(pkt) + return + } + ipHdr := header.IPv4(hdrBytes) + + // We don't handle fragments. That should be the vast majority of + // traffic, and simplifies handling. + if ipHdr.FragmentOffset() != 0 || ipHdr.Flags()&header.IPv4FlagMoreFragments != 0 { + gd.handlePacket(pkt) + return + } + + // We only handle TCP packets without IP options. + if ipHdr.HeaderLength() != header.IPv4MinimumSize || tcpip.TransportProtocolNumber(ipHdr.Protocol()) != header.TCPProtocolNumber { + gd.handlePacket(pkt) + return + } + tcpHdr := header.TCP(hdrBytes[header.IPv4MinimumSize:]) + ipHdr = ipHdr[:header.IPv4MinimumSize] + dataOff := tcpHdr.DataOffset() + if dataOff < header.TCPMinimumSize { + // Malformed packet: will be handled further up the stack. + gd.handlePacket(pkt) + return + } + hdrBytes, ok = pkt.Data().PullUp(header.IPv4MinimumSize + int(dataOff)) + if !ok { + // Malformed packet: will be handled further up the stack. + gd.handlePacket(pkt) + return + } + + tcpHdr = header.TCP(hdrBytes[header.IPv4MinimumSize:]) + + // If either checksum is bad, flush the packet. Since we don't know + // what bits were flipped, we can't identify this packet with a flow. + if !pkt.RXChecksumValidated { + if !ipHdr.IsValid(pkt.Data().Size()) || !ipHdr.IsChecksumValid() { + gd.handlePacket(pkt) + return + } + payloadChecksum := pkt.Data().ChecksumAtOffset(header.IPv4MinimumSize + int(dataOff)) + tcpPayloadSize := pkt.Data().Size() - header.IPv4MinimumSize - int(dataOff) + if !tcpHdr.IsChecksumValid(ipHdr.SourceAddress(), ipHdr.DestinationAddress(), payloadChecksum, uint16(tcpPayloadSize)) { + gd.handlePacket(pkt) + return + } + // We've validated the checksum, no reason for others to do it + // again. + pkt.RXChecksumValidated = true + } + + // Now we can get the bucket for the packet. + bucket := &gd.buckets[gd.bucketForPacket4(ipHdr, tcpHdr)&groNBucketsMask] + groPkt, flushGROPkt := bucket.findGROPacket4(pkt, ipHdr, tcpHdr) + bucket.found(gd, groPkt, flushGROPkt, pkt, ipHdr, tcpHdr, updateIPv4Hdr) +} + +func (gd *GRO) dispatch6(pkt *stack.PacketBuffer) { + // Immediately get the IPv6 and TCP headers. We need a way to hash the + // packet into its bucket, which requires addresses and ports. Linux + // simply gets a hash passed by hardware, but we're not so lucky. + + hdrBytes, ok := pkt.Data().PullUp(header.IPv6MinimumSize) + if !ok { + gd.handlePacket(pkt) + return + } + ipHdr := header.IPv6(hdrBytes) + + // Getting the IP header (+ extension headers) size is a bit of a pain + // on IPv6. + transProto := tcpip.TransportProtocolNumber(ipHdr.NextHeader()) + buf := pkt.Data().ToBuffer() + buf.TrimFront(header.IPv6MinimumSize) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(transProto), buf) + ipHdrSize := int(header.IPv6MinimumSize) + for { + transProto = tcpip.TransportProtocolNumber(it.NextHeaderIdentifier()) + extHdr, done, err := it.Next() + if err != nil { + gd.handlePacket(pkt) + return + } + if done { + break + } + switch extHdr.(type) { + // We can GRO these, so just skip over them. + case header.IPv6HopByHopOptionsExtHdr: + case header.IPv6RoutingExtHdr: + case header.IPv6DestinationOptionsExtHdr: + case header.IPv6ExperimentExtHdr: + default: + // This is either a TCP header or something we can't handle. + ipHdrSize = int(it.HeaderOffset()) + done = true + } + extHdr.Release() + if done { + break + } + } + + hdrBytes, ok = pkt.Data().PullUp(ipHdrSize + header.TCPMinimumSize) + if !ok { + gd.handlePacket(pkt) + return + } + ipHdr = header.IPv6(hdrBytes[:ipHdrSize]) + + // We only handle TCP packets. + if transProto != header.TCPProtocolNumber { + gd.handlePacket(pkt) + return + } + tcpHdr := header.TCP(hdrBytes[ipHdrSize:]) + dataOff := tcpHdr.DataOffset() + if dataOff < header.TCPMinimumSize { + // Malformed packet: will be handled further up the stack. + gd.handlePacket(pkt) + return + } + + hdrBytes, ok = pkt.Data().PullUp(ipHdrSize + int(dataOff)) + if !ok { + // Malformed packet: will be handled further up the stack. + gd.handlePacket(pkt) + return + } + tcpHdr = header.TCP(hdrBytes[ipHdrSize:]) + + // If either checksum is bad, flush the packet. Since we don't know + // what bits were flipped, we can't identify this packet with a flow. + if !pkt.RXChecksumValidated { + if !ipHdr.IsValid(pkt.Data().Size()) { + gd.handlePacket(pkt) + return + } + payloadChecksum := pkt.Data().ChecksumAtOffset(ipHdrSize + int(dataOff)) + tcpPayloadSize := pkt.Data().Size() - ipHdrSize - int(dataOff) + if !tcpHdr.IsChecksumValid(ipHdr.SourceAddress(), ipHdr.DestinationAddress(), payloadChecksum, uint16(tcpPayloadSize)) { + gd.handlePacket(pkt) + return + } + // We've validated the checksum, no reason for others to do it + // again. + pkt.RXChecksumValidated = true + } + + // Now we can get the bucket for the packet. + bucket := &gd.buckets[gd.bucketForPacket6(ipHdr, tcpHdr)&groNBucketsMask] + groPkt, flushGROPkt := bucket.findGROPacket6(pkt, ipHdr, tcpHdr) + bucket.found(gd, groPkt, flushGROPkt, pkt, ipHdr, tcpHdr, updateIPv6Hdr) +} + +func (gd *GRO) bucketForPacket4(ipHdr header.IPv4, tcpHdr header.TCP) int { + // It would be better to use jenkins or checksum. + var sum int + srcAddr := ipHdr.SourceAddress() + for _, val := range srcAddr.AsSlice() { + sum += int(val) + } + dstAddr := ipHdr.DestinationAddress() + for _, val := range dstAddr.AsSlice() { + sum += int(val) + } + sum += int(tcpHdr.SourcePort()) + sum += int(tcpHdr.DestinationPort()) + return sum +} + +func (gd *GRO) bucketForPacket6(ipHdr header.IPv6, tcpHdr header.TCP) int { + // It would be better to use jenkins or checksum. + var sum int + srcAddr := ipHdr.SourceAddress() + for _, val := range srcAddr.AsSlice() { + sum += int(val) + } + dstAddr := ipHdr.DestinationAddress() + for _, val := range dstAddr.AsSlice() { + sum += int(val) + } + sum += int(tcpHdr.SourcePort()) + sum += int(tcpHdr.DestinationPort()) + return sum +} + +// Flush sends all packets up the stack. +func (gd *GRO) Flush() { + for i := range gd.buckets { + for groPkt := gd.buckets[i].packets.Front(); groPkt != nil; groPkt = groPkt.Next() { + pkt := groPkt.pkt + gd.buckets[i].removeOne(groPkt) + gd.handlePacket(pkt) + pkt.DecRef() + } + } +} + +func (gd *GRO) handlePacket(pkt *stack.PacketBuffer) { + gd.Dispatcher.DeliverNetworkPacket(pkt.NetworkProtocolNumber, pkt) +} + +// String implements fmt.Stringer. +func (gd *GRO) String() string { + ret := "GRO state: \n" + for i := range gd.buckets { + bucket := &gd.buckets[i] + ret += fmt.Sprintf("bucket %d: %d packets: ", i, bucket.count) + for groPkt := bucket.packets.Front(); groPkt != nil; groPkt = groPkt.Next() { + ret += fmt.Sprintf("%d, ", groPkt.pkt.Data().Size()) + } + ret += "\n" + } + return ret +} + +// shouldFlushTCP returns whether the TCP headers indicate that groPkt should +// be flushed +func shouldFlushTCP(groPkt *groPacket, tcpHdr header.TCP) bool { + flags := tcpHdr.Flags() + groPktFlags := groPkt.tcpHdr.Flags() + dataOff := tcpHdr.DataOffset() + if flags&header.TCPFlagCwr != 0 || // Is congestion control occurring? + (flags^groPktFlags)&^(header.TCPFlagCwr|header.TCPFlagFin|header.TCPFlagPsh) != 0 || // Do the flags differ besides CRW, FIN, and PSH? + tcpHdr.AckNumber() != groPkt.tcpHdr.AckNumber() || // Do the ACKs match? + dataOff != groPkt.tcpHdr.DataOffset() || // Are the TCP headers the same length? + groPkt.tcpHdr.SequenceNumber()+uint32(groPkt.payloadSize()) != tcpHdr.SequenceNumber() { // Does the incoming packet match the expected sequence number? + return true + } + // The options, including timestamps, must be identical. + return !bytes.Equal(tcpHdr[header.TCPMinimumSize:], groPkt.tcpHdr[header.TCPMinimumSize:]) +} + +func updateIPv4Hdr(ipHdrBytes []byte, newBytes int) { + ipHdr := header.IPv4(ipHdrBytes) + ipHdr.SetTotalLength(ipHdr.TotalLength() + uint16(newBytes)) +} + +func updateIPv6Hdr(ipHdrBytes []byte, newBytes int) { + ipHdr := header.IPv6(ipHdrBytes) + ipHdr.SetPayloadLength(ipHdr.PayloadLength() + uint16(newBytes)) +} diff --git a/pkg/tcpip/stack/gro/gro_packet_list.go b/pkg/tcpip/stack/gro/gro_packet_list.go new file mode 100644 index 0000000..3241d56 --- /dev/null +++ b/pkg/tcpip/stack/gro/gro_packet_list.go @@ -0,0 +1,239 @@ +package gro + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type groPacketElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (groPacketElementMapper) linkerFor(elem *groPacket) *groPacket { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type groPacketList struct { + head *groPacket + tail *groPacket +} + +// Reset resets list l to the empty state. +func (l *groPacketList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *groPacketList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *groPacketList) Front() *groPacket { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *groPacketList) Back() *groPacket { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *groPacketList) Len() (count int) { + for e := l.Front(); e != nil; e = (groPacketElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *groPacketList) PushFront(e *groPacket) { + linker := groPacketElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + groPacketElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *groPacketList) PushFrontList(m *groPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + groPacketElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + groPacketElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *groPacketList) PushBack(e *groPacket) { + linker := groPacketElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + groPacketElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *groPacketList) PushBackList(m *groPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + groPacketElementMapper{}.linkerFor(l.tail).SetNext(m.head) + groPacketElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *groPacketList) InsertAfter(b, e *groPacket) { + bLinker := groPacketElementMapper{}.linkerFor(b) + eLinker := groPacketElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + groPacketElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *groPacketList) InsertBefore(a, e *groPacket) { + aLinker := groPacketElementMapper{}.linkerFor(a) + eLinker := groPacketElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + groPacketElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *groPacketList) Remove(e *groPacket) { + linker := groPacketElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + groPacketElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + groPacketElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type groPacketEntry struct { + next *groPacket + prev *groPacket +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *groPacketEntry) Next() *groPacket { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *groPacketEntry) Prev() *groPacket { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *groPacketEntry) SetNext(elem *groPacket) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *groPacketEntry) SetPrev(elem *groPacket) { + e.prev = elem +} diff --git a/pkg/tcpip/stack/gro/gro_state_autogen.go b/pkg/tcpip/stack/gro/gro_state_autogen.go new file mode 100644 index 0000000..684f7c8 --- /dev/null +++ b/pkg/tcpip/stack/gro/gro_state_autogen.go @@ -0,0 +1,178 @@ +// automatically generated by stateify. + +package gro + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (gb *groBucket) StateTypeName() string { + return "pkg/tcpip/stack/gro.groBucket" +} + +func (gb *groBucket) StateFields() []string { + return []string{ + "count", + "packets", + "packetsPrealloc", + "allocIdxs", + } +} + +func (gb *groBucket) beforeSave() {} + +// +checklocksignore +func (gb *groBucket) StateSave(stateSinkObject state.Sink) { + gb.beforeSave() + stateSinkObject.Save(0, &gb.count) + stateSinkObject.Save(1, &gb.packets) + stateSinkObject.Save(2, &gb.packetsPrealloc) + stateSinkObject.Save(3, &gb.allocIdxs) +} + +func (gb *groBucket) afterLoad(context.Context) {} + +// +checklocksignore +func (gb *groBucket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &gb.count) + stateSourceObject.Load(1, &gb.packets) + stateSourceObject.Load(2, &gb.packetsPrealloc) + stateSourceObject.Load(3, &gb.allocIdxs) +} + +func (pk *groPacket) StateTypeName() string { + return "pkg/tcpip/stack/gro.groPacket" +} + +func (pk *groPacket) StateFields() []string { + return []string{ + "groPacketEntry", + "pkt", + "ipHdr", + "tcpHdr", + "initialLength", + "idx", + } +} + +func (pk *groPacket) beforeSave() {} + +// +checklocksignore +func (pk *groPacket) StateSave(stateSinkObject state.Sink) { + pk.beforeSave() + stateSinkObject.Save(0, &pk.groPacketEntry) + stateSinkObject.Save(1, &pk.pkt) + stateSinkObject.Save(2, &pk.ipHdr) + stateSinkObject.Save(3, &pk.tcpHdr) + stateSinkObject.Save(4, &pk.initialLength) + stateSinkObject.Save(5, &pk.idx) +} + +func (pk *groPacket) afterLoad(context.Context) {} + +// +checklocksignore +func (pk *groPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &pk.groPacketEntry) + stateSourceObject.Load(1, &pk.pkt) + stateSourceObject.Load(2, &pk.ipHdr) + stateSourceObject.Load(3, &pk.tcpHdr) + stateSourceObject.Load(4, &pk.initialLength) + stateSourceObject.Load(5, &pk.idx) +} + +func (gd *GRO) StateTypeName() string { + return "pkg/tcpip/stack/gro.GRO" +} + +func (gd *GRO) StateFields() []string { + return []string{ + "enabled", + "buckets", + "Dispatcher", + } +} + +func (gd *GRO) beforeSave() {} + +// +checklocksignore +func (gd *GRO) StateSave(stateSinkObject state.Sink) { + gd.beforeSave() + stateSinkObject.Save(0, &gd.enabled) + stateSinkObject.Save(1, &gd.buckets) + stateSinkObject.Save(2, &gd.Dispatcher) +} + +func (gd *GRO) afterLoad(context.Context) {} + +// +checklocksignore +func (gd *GRO) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &gd.enabled) + stateSourceObject.Load(1, &gd.buckets) + stateSourceObject.Load(2, &gd.Dispatcher) +} + +func (l *groPacketList) StateTypeName() string { + return "pkg/tcpip/stack/gro.groPacketList" +} + +func (l *groPacketList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *groPacketList) beforeSave() {} + +// +checklocksignore +func (l *groPacketList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *groPacketList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *groPacketList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *groPacketEntry) StateTypeName() string { + return "pkg/tcpip/stack/gro.groPacketEntry" +} + +func (e *groPacketEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *groPacketEntry) beforeSave() {} + +// +checklocksignore +func (e *groPacketEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *groPacketEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *groPacketEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*groBucket)(nil)) + state.Register((*groPacket)(nil)) + state.Register((*GRO)(nil)) + state.Register((*groPacketList)(nil)) + state.Register((*groPacketEntry)(nil)) +} diff --git a/pkg/tcpip/stack/headertype_string.go b/pkg/tcpip/stack/headertype_string.go new file mode 100644 index 0000000..cd80de0 --- /dev/null +++ b/pkg/tcpip/stack/headertype_string.go @@ -0,0 +1,40 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by "stringer -type headerType ."; DO NOT EDIT. + +package stack + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[virtioNetHeader-0] + _ = x[linkHeader-1] + _ = x[networkHeader-2] + _ = x[transportHeader-3] + _ = x[numHeaderType-4] +} + +const _headerType_name = "virtioNetHeaderlinkHeadernetworkHeadertransportHeadernumHeaderType" + +var _headerType_index = [...]uint8{0, 10, 23, 38, 51} + +func (i headerType) String() string { + if i < 0 || i >= headerType(len(_headerType_index)-1) { + return "headerType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _headerType_name[_headerType_index[i]:_headerType_index[i+1]] +} diff --git a/pkg/tcpip/stack/hook_string.go b/pkg/tcpip/stack/hook_string.go new file mode 100644 index 0000000..3dc8a7b --- /dev/null +++ b/pkg/tcpip/stack/hook_string.go @@ -0,0 +1,41 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by "stringer -type Hook ."; DO NOT EDIT. + +package stack + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Prerouting-0] + _ = x[Input-1] + _ = x[Forward-2] + _ = x[Output-3] + _ = x[Postrouting-4] + _ = x[NumHooks-5] +} + +const _Hook_name = "PreroutingInputForwardOutputPostroutingNumHooks" + +var _Hook_index = [...]uint8{0, 10, 15, 22, 28, 39, 47} + +func (i Hook) String() string { + if i >= Hook(len(_Hook_index)-1) { + return "Hook(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Hook_name[_Hook_index[i]:_Hook_index[i+1]] +} diff --git a/pkg/tcpip/stack/icmp_rate_limit.go b/pkg/tcpip/stack/icmp_rate_limit.go new file mode 100644 index 0000000..1b3c3b1 --- /dev/null +++ b/pkg/tcpip/stack/icmp_rate_limit.go @@ -0,0 +1,75 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "golang.org/x/time/rate" +) + +const ( + // icmpLimit is the default maximum number of ICMP messages permitted by this + // rate limiter. + icmpLimit = 1000 + + // icmpBurst is the default number of ICMP messages that can be sent in a single + // burst. + icmpBurst = 50 +) + +// ICMPRateLimiter is a global rate limiter that controls the generation of +// ICMP messages generated by the stack. +// +// +stateify savable +type ICMPRateLimiter struct { + // TODO(b/341946753): Restore when netstack is savable. + limiter *rate.Limiter `state:"nosave"` + clock tcpip.Clock +} + +// NewICMPRateLimiter returns a global rate limiter for controlling the rate +// at which ICMP messages are generated by the stack. The returned limiter +// does not apply limits to any ICMP types by default. +func NewICMPRateLimiter(clock tcpip.Clock) *ICMPRateLimiter { + return &ICMPRateLimiter{ + clock: clock, + limiter: rate.NewLimiter(icmpLimit, icmpBurst), + } +} + +// SetLimit sets a new Limit for the limiter. +func (l *ICMPRateLimiter) SetLimit(limit rate.Limit) { + l.limiter.SetLimitAt(l.clock.Now(), limit) +} + +// Limit returns the maximum overall event rate. +func (l *ICMPRateLimiter) Limit() rate.Limit { + return l.limiter.Limit() +} + +// SetBurst sets a new burst size for the limiter. +func (l *ICMPRateLimiter) SetBurst(burst int) { + l.limiter.SetBurstAt(l.clock.Now(), burst) +} + +// Burst returns the maximum burst size. +func (l *ICMPRateLimiter) Burst() int { + return l.limiter.Burst() +} + +// Allow reports whether one ICMP message may be sent now. +func (l *ICMPRateLimiter) Allow() bool { + return l.limiter.AllowN(l.clock.Now(), 1) +} diff --git a/pkg/tcpip/stack/iptables.go b/pkg/tcpip/stack/iptables.go new file mode 100644 index 0000000..884abfd --- /dev/null +++ b/pkg/tcpip/stack/iptables.go @@ -0,0 +1,716 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "context" + "fmt" + "math/rand" + "reflect" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +// TableID identifies a specific table. +type TableID int + +// Each value identifies a specific table. +const ( + NATID TableID = iota + MangleID + FilterID + NumTables +) + +// HookUnset indicates that there is no hook set for an entrypoint or +// underflow. +const HookUnset = -1 + +// reaperDelay is how long to wait before starting to reap connections. +const reaperDelay = 5 * time.Second + +// DefaultTables returns a default set of tables. Each chain is set to accept +// all packets. +func DefaultTables(clock tcpip.Clock, rand *rand.Rand) *IPTables { + return &IPTables{ + v4Tables: [NumTables]Table{ + NATID: { + Rules: []Rule{ + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &ErrorTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: 0, + Input: 1, + Forward: HookUnset, + Output: 2, + Postrouting: 3, + }, + Underflows: [NumHooks]int{ + Prerouting: 0, + Input: 1, + Forward: HookUnset, + Output: 2, + Postrouting: 3, + }, + }, + MangleID: { + Rules: []Rule{ + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &ErrorTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: 0, + Output: 1, + }, + Underflows: [NumHooks]int{ + Prerouting: 0, + Input: HookUnset, + Forward: HookUnset, + Output: 1, + Postrouting: HookUnset, + }, + }, + FilterID: { + Rules: []Rule{ + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &ErrorTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: HookUnset, + Input: 0, + Forward: 1, + Output: 2, + Postrouting: HookUnset, + }, + Underflows: [NumHooks]int{ + Prerouting: HookUnset, + Input: 0, + Forward: 1, + Output: 2, + Postrouting: HookUnset, + }, + }, + }, + v6Tables: [NumTables]Table{ + NATID: { + Rules: []Rule{ + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &ErrorTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: 0, + Input: 1, + Forward: HookUnset, + Output: 2, + Postrouting: 3, + }, + Underflows: [NumHooks]int{ + Prerouting: 0, + Input: 1, + Forward: HookUnset, + Output: 2, + Postrouting: 3, + }, + }, + MangleID: { + Rules: []Rule{ + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &ErrorTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: 0, + Output: 1, + }, + Underflows: [NumHooks]int{ + Prerouting: 0, + Input: HookUnset, + Forward: HookUnset, + Output: 1, + Postrouting: HookUnset, + }, + }, + FilterID: { + Rules: []Rule{ + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &ErrorTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: HookUnset, + Input: 0, + Forward: 1, + Output: 2, + Postrouting: HookUnset, + }, + Underflows: [NumHooks]int{ + Prerouting: HookUnset, + Input: 0, + Forward: 1, + Output: 2, + Postrouting: HookUnset, + }, + }, + }, + connections: ConnTrack{ + seed: rand.Uint32(), + clock: clock, + rand: rand, + }, + } +} + +// EmptyFilterTable returns a Table with no rules and the filter table chains +// mapped to HookUnset. +func EmptyFilterTable() Table { + return Table{ + Rules: []Rule{}, + BuiltinChains: [NumHooks]int{ + Prerouting: HookUnset, + Postrouting: HookUnset, + }, + Underflows: [NumHooks]int{ + Prerouting: HookUnset, + Postrouting: HookUnset, + }, + } +} + +// EmptyNATTable returns a Table with no rules and the filter table chains +// mapped to HookUnset. +func EmptyNATTable() Table { + return Table{ + Rules: []Rule{}, + BuiltinChains: [NumHooks]int{ + Forward: HookUnset, + }, + Underflows: [NumHooks]int{ + Forward: HookUnset, + }, + } +} + +// GetTable returns a table with the given id and IP version. It panics when an +// invalid id is provided. +func (it *IPTables) GetTable(id TableID, ipv6 bool) Table { + it.mu.RLock() + defer it.mu.RUnlock() + return it.getTableRLocked(id, ipv6) +} + +// +checklocksread:it.mu +func (it *IPTables) getTableRLocked(id TableID, ipv6 bool) Table { + if ipv6 { + return it.v6Tables[id] + } + return it.v4Tables[id] +} + +// ReplaceTable replaces or inserts table by name. It panics when an invalid id +// is provided. +func (it *IPTables) ReplaceTable(id TableID, table Table, ipv6 bool) { + it.replaceTable(id, table, ipv6, false /* force */) +} + +// ForceReplaceTable replaces or inserts table by name. It panics when an invalid id +// is provided. It enables iptables even when the inserted table is all +// conditionless ACCEPT, skipping our optimization that disables iptables until +// they're modified. +func (it *IPTables) ForceReplaceTable(id TableID, table Table, ipv6 bool) { + it.replaceTable(id, table, ipv6, true /* force */) +} + +func (it *IPTables) replaceTable(id TableID, table Table, ipv6, force bool) { + it.mu.Lock() + defer it.mu.Unlock() + + // If iptables is being enabled, initialize the conntrack table and + // reaper. + if !it.modified { + // Don't do anything if the table is identical. + if ((ipv6 && reflect.DeepEqual(table, it.v6Tables[id])) || (!ipv6 && reflect.DeepEqual(table, it.v4Tables[id]))) && !force { + return + } + + it.connections.init() + it.startReaper(reaperDelay) + } + it.modified = true + if ipv6 { + it.v6Tables[id] = table + } else { + it.v4Tables[id] = table + } +} + +// A chainVerdict is what a table decides should be done with a packet. +type chainVerdict int + +const ( + // chainAccept indicates the packet should continue through netstack. + chainAccept chainVerdict = iota + + // chainDrop indicates the packet should be dropped. + chainDrop + + // chainReturn indicates the packet should return to the calling chain + // or the underflow rule of a builtin chain. + chainReturn +) + +type checkTable struct { + fn checkTableFn + tableID TableID + table Table +} + +// shouldSkipOrPopulateTables returns true iff IPTables should be skipped. +// +// If IPTables should not be skipped, tables will be updated with the +// specified table. +// +// This is called in the hot path even when iptables are disabled, so we ensure +// it does not allocate. We check recursively for heap allocations, but not for: +// - Stack splitting, which can allocate. +// - Calls to interfaces, which can allocate. +// - Calls to dynamic functions, which can allocate. +// +// +checkescape:hard +func (it *IPTables) shouldSkipOrPopulateTables(tables []checkTable, pkt *PacketBuffer) bool { + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber, header.IPv6ProtocolNumber: + default: + // IPTables only supports IPv4/IPv6. + return true + } + + it.mu.RLock() + defer it.mu.RUnlock() + + if !it.modified { + // Many users never configure iptables. Spare them the cost of rule + // traversal if rules have never been set. + return true + } + + for i := range tables { + table := &tables[i] + table.table = it.getTableRLocked(table.tableID, pkt.NetworkProtocolNumber == header.IPv6ProtocolNumber) + } + return false +} + +// CheckPrerouting performs the prerouting hook on the packet. +// +// Returns true iff the packet may continue traversing the stack; the packet +// must be dropped if false is returned. +// +// Precondition: The packet's network and transport header must be set. +// +// This is called in the hot path even when iptables are disabled, so we ensure +// that it does not allocate. Note that called functions (e.g. +// getConnAndUpdate) can allocate. +// +checkescape +func (it *IPTables) CheckPrerouting(pkt *PacketBuffer, addressEP AddressableEndpoint, inNicName string) bool { + tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: check, + tableID: MangleID, + }, + { + fn: checkNAT, + tableID: NATID, + }, + } + + if it.shouldSkipOrPopulateTables(tables[:], pkt) { + return true + } + + pkt.tuple = it.connections.getConnAndUpdate(pkt, false /* skipChecksumValidation */) + + for _, table := range tables { + if !table.fn(it, table.table, Prerouting, pkt, nil /* route */, addressEP, inNicName, "" /* outNicName */) { + return false + } + } + + return true +} + +// CheckInput performs the input hook on the packet. +// +// Returns true iff the packet may continue traversing the stack; the packet +// must be dropped if false is returned. +// +// Precondition: The packet's network and transport header must be set. +// +// This is called in the hot path even when iptables are disabled, so we ensure +// that it does not allocate. Note that called functions (e.g. +// getConnAndUpdate) can allocate. +// +checkescape +func (it *IPTables) CheckInput(pkt *PacketBuffer, inNicName string) bool { + tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: checkNAT, + tableID: NATID, + }, + { + fn: check, + tableID: FilterID, + }, + } + + if it.shouldSkipOrPopulateTables(tables[:], pkt) { + return true + } + + for _, table := range tables { + if !table.fn(it, table.table, Input, pkt, nil /* route */, nil /* addressEP */, inNicName, "" /* outNicName */) { + return false + } + } + + if t := pkt.tuple; t != nil { + pkt.tuple = nil + return t.conn.finalize() + } + return true +} + +// CheckForward performs the forward hook on the packet. +// +// Returns true iff the packet may continue traversing the stack; the packet +// must be dropped if false is returned. +// +// Precondition: The packet's network and transport header must be set. +// +// This is called in the hot path even when iptables are disabled, so we ensure +// that it does not allocate. Note that called functions (e.g. +// getConnAndUpdate) can allocate. +// +checkescape +func (it *IPTables) CheckForward(pkt *PacketBuffer, inNicName, outNicName string) bool { + tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: check, + tableID: FilterID, + }, + } + + if it.shouldSkipOrPopulateTables(tables[:], pkt) { + return true + } + + for _, table := range tables { + if !table.fn(it, table.table, Forward, pkt, nil /* route */, nil /* addressEP */, inNicName, outNicName) { + return false + } + } + + return true +} + +// CheckOutput performs the output hook on the packet. +// +// Returns true iff the packet may continue traversing the stack; the packet +// must be dropped if false is returned. +// +// Precondition: The packet's network and transport header must be set. +// +// This is called in the hot path even when iptables are disabled, so we ensure +// that it does not allocate. Note that called functions (e.g. +// getConnAndUpdate) can allocate. +// +checkescape +func (it *IPTables) CheckOutput(pkt *PacketBuffer, r *Route, outNicName string) bool { + tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: check, + tableID: MangleID, + }, + { + fn: checkNAT, + tableID: NATID, + }, + { + fn: check, + tableID: FilterID, + }, + } + + if it.shouldSkipOrPopulateTables(tables[:], pkt) { + return true + } + + // We don't need to validate the checksum in the Output path: we can assume + // we calculate it correctly, plus checksumming may be deferred due to GSO. + pkt.tuple = it.connections.getConnAndUpdate(pkt, true /* skipChecksumValidation */) + + for _, table := range tables { + if !table.fn(it, table.table, Output, pkt, r, nil /* addressEP */, "" /* inNicName */, outNicName) { + return false + } + } + + return true +} + +// CheckPostrouting performs the postrouting hook on the packet. +// +// Returns true iff the packet may continue traversing the stack; the packet +// must be dropped if false is returned. +// +// Precondition: The packet's network and transport header must be set. +// +// This is called in the hot path even when iptables are disabled, so we ensure +// that it does not allocate. Note that called functions (e.g. +// getConnAndUpdate) can allocate. +// +checkescape +func (it *IPTables) CheckPostrouting(pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, outNicName string) bool { + tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: check, + tableID: MangleID, + }, + { + fn: checkNAT, + tableID: NATID, + }, + } + + if it.shouldSkipOrPopulateTables(tables[:], pkt) { + return true + } + + for _, table := range tables { + if !table.fn(it, table.table, Postrouting, pkt, r, addressEP, "" /* inNicName */, outNicName) { + return false + } + } + + if t := pkt.tuple; t != nil { + pkt.tuple = nil + return t.conn.finalize() + } + return true +} + +// Note: this used to omit the *IPTables parameter, but doing so caused +// unnecessary allocations. +type checkTableFn func(it *IPTables, table Table, hook Hook, pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) bool + +func checkNAT(it *IPTables, table Table, hook Hook, pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) bool { + return it.checkNAT(table, hook, pkt, r, addressEP, inNicName, outNicName) +} + +// checkNAT runs the packet through the NAT table. +// +// See check. +func (it *IPTables) checkNAT(table Table, hook Hook, pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) bool { + t := pkt.tuple + if t != nil && t.conn.handlePacket(pkt, hook, r) { + return true + } + + if !it.check(table, hook, pkt, r, addressEP, inNicName, outNicName) { + return false + } + + if t == nil { + return true + } + + dnat, natDone := func() (bool, bool) { + switch hook { + case Prerouting, Output: + return true, pkt.dnatDone + case Input, Postrouting: + return false, pkt.snatDone + case Forward: + panic("should not attempt NAT in forwarding") + default: + panic(fmt.Sprintf("unhandled hook = %d", hook)) + } + }() + + // Make sure the connection is NATed. + // + // If the packet was already NATed, the connection must be NATed. + if !natDone { + t.conn.maybePerformNoopNAT(pkt, hook, r, dnat) + } + + return true +} + +func check(it *IPTables, table Table, hook Hook, pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) bool { + return it.check(table, hook, pkt, r, addressEP, inNicName, outNicName) +} + +// check runs the packet through the rules in the specified table for the +// hook. It returns true if the packet should continue to traverse through the +// network stack or tables, or false when it must be dropped. +// +// Precondition: The packet's network and transport header must be set. +func (it *IPTables) check(table Table, hook Hook, pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) bool { + ruleIdx := table.BuiltinChains[hook] + switch verdict := it.checkChain(hook, pkt, table, ruleIdx, r, addressEP, inNicName, outNicName); verdict { + // If the table returns Accept, move on to the next table. + case chainAccept: + return true + // The Drop verdict is final. + case chainDrop: + return false + case chainReturn: + // Any Return from a built-in chain means we have to + // call the underflow. + underflow := table.Rules[table.Underflows[hook]] + switch v, _ := underflow.Target.Action(pkt, hook, r, addressEP); v { + case RuleAccept: + return true + case RuleDrop: + return false + case RuleJump, RuleReturn: + panic("Underflows should only return RuleAccept or RuleDrop.") + default: + panic(fmt.Sprintf("Unknown verdict: %d", v)) + } + default: + panic(fmt.Sprintf("Unknown verdict %v.", verdict)) + } +} + +// beforeSave is invoked by stateify. +func (it *IPTables) beforeSave() { + // Ensure the reaper exits cleanly. + it.reaper.Stop() + // Prevent others from modifying the connection table. + it.connections.mu.Lock() +} + +// afterLoad is invoked by stateify. +func (it *IPTables) afterLoad(context.Context) { + it.startReaper(reaperDelay) +} + +// startReaper periodically reaps timed out connections. +func (it *IPTables) startReaper(interval time.Duration) { + bucket := 0 + it.reaper = it.connections.clock.AfterFunc(interval, func() { + bucket, interval = it.connections.reapUnused(bucket, interval) + it.reaper.Reset(interval) + }) +} + +// Preconditions: +// - pkt is a IPv4 packet of at least length header.IPv4MinimumSize. +// - pkt.NetworkHeader is not nil. +func (it *IPTables) checkChain(hook Hook, pkt *PacketBuffer, table Table, ruleIdx int, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) chainVerdict { + // Start from ruleIdx and walk the list of rules until a rule gives us + // a verdict. + for ruleIdx < len(table.Rules) { + switch verdict, jumpTo := it.checkRule(hook, pkt, table, ruleIdx, r, addressEP, inNicName, outNicName); verdict { + case RuleAccept: + return chainAccept + + case RuleDrop: + return chainDrop + + case RuleReturn: + return chainReturn + + case RuleJump: + // "Jumping" to the next rule just means we're + // continuing on down the list. + if jumpTo == ruleIdx+1 { + ruleIdx++ + continue + } + switch verdict := it.checkChain(hook, pkt, table, jumpTo, r, addressEP, inNicName, outNicName); verdict { + case chainAccept: + return chainAccept + case chainDrop: + return chainDrop + case chainReturn: + ruleIdx++ + continue + default: + panic(fmt.Sprintf("Unknown verdict: %d", verdict)) + } + + default: + panic(fmt.Sprintf("Unknown verdict: %d", verdict)) + } + } + + // We got through the entire table without a decision. Default to DROP + // for safety. + return chainDrop +} + +// Preconditions: +// - pkt is a IPv4 packet of at least length header.IPv4MinimumSize. +// - pkt.NetworkHeader is not nil. +// +// * pkt is a IPv4 packet of at least length header.IPv4MinimumSize. +// * pkt.NetworkHeader is not nil. +func (it *IPTables) checkRule(hook Hook, pkt *PacketBuffer, table Table, ruleIdx int, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) (RuleVerdict, int) { + rule := table.Rules[ruleIdx] + + // Check whether the packet matches the IP header filter. + if !rule.Filter.match(pkt, hook, inNicName, outNicName) { + // Continue on to the next rule. + return RuleJump, ruleIdx + 1 + } + + // Go through each rule matcher. If they all match, run + // the rule target. + for _, matcher := range rule.Matchers { + matches, hotdrop := matcher.Match(hook, pkt, inNicName, outNicName) + if hotdrop { + return RuleDrop, 0 + } + if !matches { + // Continue on to the next rule. + return RuleJump, ruleIdx + 1 + } + } + + // All the matchers matched, so run the target. + return rule.Target.Action(pkt, hook, r, addressEP) +} + +// OriginalDst returns the original destination of redirected connections. It +// returns an error if the connection doesn't exist or isn't redirected. +func (it *IPTables) OriginalDst(epID TransportEndpointID, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber) (tcpip.Address, uint16, tcpip.Error) { + it.mu.RLock() + defer it.mu.RUnlock() + if !it.modified { + return tcpip.Address{}, 0, &tcpip.ErrNotConnected{} + } + return it.connections.originalDst(epID, netProto, transProto) +} diff --git a/pkg/tcpip/stack/iptables_mutex.go b/pkg/tcpip/stack/iptables_mutex.go new file mode 100644 index 0000000..5e6a78f --- /dev/null +++ b/pkg/tcpip/stack/iptables_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type ipTablesRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var ipTableslockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type ipTableslockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *ipTablesRWMutex) Lock() { + locking.AddGLock(ipTablesprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *ipTablesRWMutex) NestedLock(i ipTableslockNameIndex) { + locking.AddGLock(ipTablesprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *ipTablesRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(ipTablesprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *ipTablesRWMutex) NestedUnlock(i ipTableslockNameIndex) { + m.mu.Unlock() + locking.DelGLock(ipTablesprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *ipTablesRWMutex) RLock() { + locking.AddGLock(ipTablesprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *ipTablesRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(ipTablesprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *ipTablesRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *ipTablesRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *ipTablesRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var ipTablesprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func ipTablesinitLockNames() {} + +func init() { + ipTablesinitLockNames() + ipTablesprefixIndex = locking.NewMutexClass(reflect.TypeOf(ipTablesRWMutex{}), ipTableslockNames) +} diff --git a/pkg/tcpip/stack/iptables_targets.go b/pkg/tcpip/stack/iptables_targets.go new file mode 100644 index 0000000..575c411 --- /dev/null +++ b/pkg/tcpip/stack/iptables_targets.go @@ -0,0 +1,492 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "math" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +// AcceptTarget accepts packets. +// +// +stateify savable +type AcceptTarget struct { + // NetworkProtocol is the network protocol the target is used with. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (*AcceptTarget) Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) { + return RuleAccept, 0 +} + +// DropTarget drops packets. +// +// +stateify savable +type DropTarget struct { + // NetworkProtocol is the network protocol the target is used with. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (*DropTarget) Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) { + return RuleDrop, 0 +} + +// RejectIPv4WithHandler handles rejecting a packet. +type RejectIPv4WithHandler interface { + // SendRejectionError sends an error packet in response to the packet. + SendRejectionError(pkt *PacketBuffer, rejectWith RejectIPv4WithICMPType, inputHook bool) tcpip.Error +} + +// RejectIPv4WithICMPType indicates the type of ICMP error that should be sent. +type RejectIPv4WithICMPType int + +// The types of errors that may be returned when rejecting IPv4 packets. +const ( + _ RejectIPv4WithICMPType = iota + RejectIPv4WithICMPNetUnreachable + RejectIPv4WithICMPHostUnreachable + RejectIPv4WithICMPPortUnreachable + RejectIPv4WithICMPNetProhibited + RejectIPv4WithICMPHostProhibited + RejectIPv4WithICMPAdminProhibited +) + +// RejectIPv4Target drops packets and sends back an error packet in response to the +// matched packet. +// +// +stateify savable +type RejectIPv4Target struct { + Handler RejectIPv4WithHandler + RejectWith RejectIPv4WithICMPType +} + +// Action implements Target.Action. +func (rt *RejectIPv4Target) Action(pkt *PacketBuffer, hook Hook, _ *Route, _ AddressableEndpoint) (RuleVerdict, int) { + switch hook { + case Input, Forward, Output: + // There is nothing reasonable for us to do in response to an error here; + // we already drop the packet. + _ = rt.Handler.SendRejectionError(pkt, rt.RejectWith, hook == Input) + return RuleDrop, 0 + case Prerouting, Postrouting: + panic(fmt.Sprintf("%s hook not supported for REDIRECT", hook)) + default: + panic(fmt.Sprintf("unhandled hook = %s", hook)) + } +} + +// RejectIPv6WithHandler handles rejecting a packet. +type RejectIPv6WithHandler interface { + // SendRejectionError sends an error packet in response to the packet. + SendRejectionError(pkt *PacketBuffer, rejectWith RejectIPv6WithICMPType, forwardingHook bool) tcpip.Error +} + +// RejectIPv6WithICMPType indicates the type of ICMP error that should be sent. +type RejectIPv6WithICMPType int + +// The types of errors that may be returned when rejecting IPv6 packets. +const ( + _ RejectIPv6WithICMPType = iota + RejectIPv6WithICMPNoRoute + RejectIPv6WithICMPAddrUnreachable + RejectIPv6WithICMPPortUnreachable + RejectIPv6WithICMPAdminProhibited +) + +// RejectIPv6Target drops packets and sends back an error packet in response to the +// matched packet. +// +// +stateify savable +type RejectIPv6Target struct { + Handler RejectIPv6WithHandler + RejectWith RejectIPv6WithICMPType +} + +// Action implements Target.Action. +func (rt *RejectIPv6Target) Action(pkt *PacketBuffer, hook Hook, _ *Route, _ AddressableEndpoint) (RuleVerdict, int) { + switch hook { + case Input, Forward, Output: + // There is nothing reasonable for us to do in response to an error here; + // we already drop the packet. + _ = rt.Handler.SendRejectionError(pkt, rt.RejectWith, hook == Input) + return RuleDrop, 0 + case Prerouting, Postrouting: + panic(fmt.Sprintf("%s hook not supported for REDIRECT", hook)) + default: + panic(fmt.Sprintf("unhandled hook = %s", hook)) + } +} + +// ErrorTarget logs an error and drops the packet. It represents a target that +// should be unreachable. +// +// +stateify savable +type ErrorTarget struct { + // NetworkProtocol is the network protocol the target is used with. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (*ErrorTarget) Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) { + log.Debugf("ErrorTarget triggered.") + return RuleDrop, 0 +} + +// UserChainTarget marks a rule as the beginning of a user chain. +// +// +stateify savable +type UserChainTarget struct { + // Name is the chain name. + Name string + + // NetworkProtocol is the network protocol the target is used with. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (*UserChainTarget) Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) { + panic("UserChainTarget should never be called.") +} + +// ReturnTarget returns from the current chain. If the chain is a built-in, the +// hook's underflow should be called. +// +// +stateify savable +type ReturnTarget struct { + // NetworkProtocol is the network protocol the target is used with. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (*ReturnTarget) Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) { + return RuleReturn, 0 +} + +// DNATTarget modifies the destination port/IP of packets. +// +// +stateify savable +type DNATTarget struct { + // The new destination address for packets. + // + // Immutable. + Addr tcpip.Address + + // The new destination port for packets. + // + // Immutable. + Port uint16 + + // NetworkProtocol is the network protocol the target is used with. + // + // Immutable. + NetworkProtocol tcpip.NetworkProtocolNumber + + // ChangeAddress indicates whether we should check addresses. + // + // Immutable. + ChangeAddress bool + + // ChangePort indicates whether we should check ports. + // + // Immutable. + ChangePort bool +} + +// Action implements Target.Action. +func (rt *DNATTarget) Action(pkt *PacketBuffer, hook Hook, r *Route, addressEP AddressableEndpoint) (RuleVerdict, int) { + // Sanity check. + if rt.NetworkProtocol != pkt.NetworkProtocolNumber { + panic(fmt.Sprintf( + "DNATTarget.Action with NetworkProtocol %d called on packet with NetworkProtocolNumber %d", + rt.NetworkProtocol, pkt.NetworkProtocolNumber)) + } + + switch hook { + case Prerouting, Output: + case Input, Forward, Postrouting: + panic(fmt.Sprintf("%s not supported for DNAT", hook)) + default: + panic(fmt.Sprintf("%s unrecognized", hook)) + } + + return dnatAction(pkt, hook, r, rt.Port, rt.Addr, rt.ChangePort, rt.ChangeAddress) +} + +// RedirectTarget redirects the packet to this machine by modifying the +// destination port/IP. Outgoing packets are redirected to the loopback device, +// and incoming packets are redirected to the incoming interface (rather than +// forwarded). +// +// +stateify savable +type RedirectTarget struct { + // Port indicates port used to redirect. It is immutable. + Port uint16 + + // NetworkProtocol is the network protocol the target is used with. It + // is immutable. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (rt *RedirectTarget) Action(pkt *PacketBuffer, hook Hook, r *Route, addressEP AddressableEndpoint) (RuleVerdict, int) { + // Sanity check. + if rt.NetworkProtocol != pkt.NetworkProtocolNumber { + panic(fmt.Sprintf( + "RedirectTarget.Action with NetworkProtocol %d called on packet with NetworkProtocolNumber %d", + rt.NetworkProtocol, pkt.NetworkProtocolNumber)) + } + + // Change the address to loopback (127.0.0.1 or ::1) in Output and to + // the primary address of the incoming interface in Prerouting. + var address tcpip.Address + switch hook { + case Output: + if pkt.NetworkProtocolNumber == header.IPv4ProtocolNumber { + address = tcpip.AddrFrom4([4]byte{127, 0, 0, 1}) + } else { + address = header.IPv6Loopback + } + case Prerouting: + // addressEP is expected to be set for the prerouting hook. + address = addressEP.MainAddress().Address + default: + panic("redirect target is supported only on output and prerouting hooks") + } + + return dnatAction(pkt, hook, r, rt.Port, address, true /* changePort */, true /* changeAddress */) +} + +// SNATTarget modifies the source port/IP in the outgoing packets. +// +// +stateify savable +type SNATTarget struct { + Addr tcpip.Address + Port uint16 + + // NetworkProtocol is the network protocol the target is used with. It + // is immutable. + NetworkProtocol tcpip.NetworkProtocolNumber + + // ChangeAddress indicates whether we should check addresses. + // + // Immutable. + ChangeAddress bool + + // ChangePort indicates whether we should check ports. + // + // Immutable. + ChangePort bool +} + +func dnatAction(pkt *PacketBuffer, hook Hook, r *Route, port uint16, address tcpip.Address, changePort, changeAddress bool) (RuleVerdict, int) { + return natAction(pkt, hook, r, portOrIdentRange{start: port, size: 1}, address, true /* dnat */, changePort, changeAddress) +} + +func targetPortRangeForTCPAndUDP(originalSrcPort uint16) portOrIdentRange { + // As per iptables(8), + // + // If no port range is specified, then source ports below 512 will be + // mapped to other ports below 512: those between 512 and 1023 inclusive + // will be mapped to ports below 1024, and other ports will be mapped to + // 1024 or above. + switch { + case originalSrcPort < 512: + return portOrIdentRange{start: 1, size: 511} + case originalSrcPort < 1024: + return portOrIdentRange{start: 1, size: 1023} + default: + return portOrIdentRange{start: 1024, size: math.MaxUint16 - 1023} + } +} + +func snatAction(pkt *PacketBuffer, hook Hook, r *Route, port uint16, address tcpip.Address, changePort, changeAddress bool) (RuleVerdict, int) { + portsOrIdents := portOrIdentRange{start: port, size: 1} + + switch pkt.TransportProtocolNumber { + case header.UDPProtocolNumber: + if port == 0 { + portsOrIdents = targetPortRangeForTCPAndUDP(header.UDP(pkt.TransportHeader().Slice()).SourcePort()) + } + case header.TCPProtocolNumber: + if port == 0 { + portsOrIdents = targetPortRangeForTCPAndUDP(header.TCP(pkt.TransportHeader().Slice()).SourcePort()) + } + case header.ICMPv4ProtocolNumber, header.ICMPv6ProtocolNumber: + // Allow NAT-ing to any 16-bit value for ICMP's Ident field to match Linux + // behaviour. + // + // https://github.com/torvalds/linux/blob/58e1100fdc5990b0cc0d4beaf2562a92e621ac7d/net/netfilter/nf_nat_core.c#L391 + portsOrIdents = portOrIdentRange{start: 0, size: math.MaxUint16 + 1} + } + + return natAction(pkt, hook, r, portsOrIdents, address, false /* dnat */, changePort, changeAddress) +} + +func natAction(pkt *PacketBuffer, hook Hook, r *Route, portsOrIdents portOrIdentRange, address tcpip.Address, dnat, changePort, changeAddress bool) (RuleVerdict, int) { + // Drop the packet if network and transport header are not set. + if len(pkt.NetworkHeader().Slice()) == 0 || len(pkt.TransportHeader().Slice()) == 0 { + return RuleDrop, 0 + } + + if t := pkt.tuple; t != nil { + t.conn.performNAT(pkt, hook, r, portsOrIdents, address, dnat, changePort, changeAddress) + return RuleAccept, 0 + } + + return RuleDrop, 0 +} + +// Action implements Target.Action. +func (st *SNATTarget) Action(pkt *PacketBuffer, hook Hook, r *Route, _ AddressableEndpoint) (RuleVerdict, int) { + // Sanity check. + if st.NetworkProtocol != pkt.NetworkProtocolNumber { + panic(fmt.Sprintf( + "SNATTarget.Action with NetworkProtocol %d called on packet with NetworkProtocolNumber %d", + st.NetworkProtocol, pkt.NetworkProtocolNumber)) + } + + switch hook { + case Postrouting, Input: + case Prerouting, Output, Forward: + panic(fmt.Sprintf("%s not supported", hook)) + default: + panic(fmt.Sprintf("%s unrecognized", hook)) + } + + return snatAction(pkt, hook, r, st.Port, st.Addr, st.ChangePort, st.ChangeAddress) +} + +// MasqueradeTarget modifies the source port/IP in the outgoing packets. +// +// +stateify savable +type MasqueradeTarget struct { + // NetworkProtocol is the network protocol the target is used with. It + // is immutable. + NetworkProtocol tcpip.NetworkProtocolNumber +} + +// Action implements Target.Action. +func (mt *MasqueradeTarget) Action(pkt *PacketBuffer, hook Hook, r *Route, addressEP AddressableEndpoint) (RuleVerdict, int) { + // Sanity check. + if mt.NetworkProtocol != pkt.NetworkProtocolNumber { + panic(fmt.Sprintf( + "MasqueradeTarget.Action with NetworkProtocol %d called on packet with NetworkProtocolNumber %d", + mt.NetworkProtocol, pkt.NetworkProtocolNumber)) + } + + switch hook { + case Postrouting: + case Prerouting, Input, Forward, Output: + panic(fmt.Sprintf("masquerade target is supported only on postrouting hook; hook = %d", hook)) + default: + panic(fmt.Sprintf("%s unrecognized", hook)) + } + + // addressEP is expected to be set for the postrouting hook. + ep := addressEP.AcquireOutgoingPrimaryAddress(pkt.Network().DestinationAddress(), tcpip.Address{} /* srcHint */, false /* allowExpired */) + if ep == nil { + // No address exists that we can use as a source address. + return RuleDrop, 0 + } + + address := ep.AddressWithPrefix().Address + ep.DecRef() + return snatAction(pkt, hook, r, 0 /* port */, address, true /* changePort */, true /* changeAddress */) +} + +func rewritePacket(n header.Network, t header.Transport, updateSRCFields, fullChecksum, updatePseudoHeader bool, newPortOrIdent uint16, newAddr tcpip.Address) { + switch t := t.(type) { + case header.ChecksummableTransport: + if updateSRCFields { + if fullChecksum { + t.SetSourcePortWithChecksumUpdate(newPortOrIdent) + } else { + t.SetSourcePort(newPortOrIdent) + } + } else { + if fullChecksum { + t.SetDestinationPortWithChecksumUpdate(newPortOrIdent) + } else { + t.SetDestinationPort(newPortOrIdent) + } + } + + if updatePseudoHeader { + var oldAddr tcpip.Address + if updateSRCFields { + oldAddr = n.SourceAddress() + } else { + oldAddr = n.DestinationAddress() + } + + t.UpdateChecksumPseudoHeaderAddress(oldAddr, newAddr, fullChecksum) + } + case header.ICMPv4: + switch icmpType := t.Type(); icmpType { + case header.ICMPv4Echo: + if updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + case header.ICMPv4EchoReply: + if !updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + default: + panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) + } + case header.ICMPv6: + switch icmpType := t.Type(); icmpType { + case header.ICMPv6EchoRequest: + if updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + case header.ICMPv6EchoReply: + if !updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + default: + panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) + } + + var oldAddr tcpip.Address + if updateSRCFields { + oldAddr = n.SourceAddress() + } else { + oldAddr = n.DestinationAddress() + } + + t.UpdateChecksumPseudoHeaderAddress(oldAddr, newAddr) + default: + panic(fmt.Sprintf("unhandled transport = %#v", t)) + } + + if checksummableNetHeader, ok := n.(header.ChecksummableNetwork); ok { + if updateSRCFields { + checksummableNetHeader.SetSourceAddressWithChecksumUpdate(newAddr) + } else { + checksummableNetHeader.SetDestinationAddressWithChecksumUpdate(newAddr) + } + } else if updateSRCFields { + n.SetSourceAddress(newAddr) + } else { + n.SetDestinationAddress(newAddr) + } +} diff --git a/pkg/tcpip/stack/iptables_types.go b/pkg/tcpip/stack/iptables_types.go new file mode 100644 index 0000000..500d218 --- /dev/null +++ b/pkg/tcpip/stack/iptables_types.go @@ -0,0 +1,385 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "strings" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +// A Hook specifies one of the hooks built into the network stack. +// +// Userspace app Userspace app +// ^ | +// | v +// [Input] [Output] +// ^ | +// | v +// | routing +// | | +// | v +// ----->[Prerouting]----->routing----->[Forward]---------[Postrouting]-----> +type Hook uint + +const ( + // Prerouting happens before a packet is routed to applications or to + // be forwarded. + Prerouting Hook = iota + + // Input happens before a packet reaches an application. + Input + + // Forward happens once it's decided that a packet should be forwarded + // to another host. + Forward + + // Output happens after a packet is written by an application to be + // sent out. + Output + + // Postrouting happens just before a packet goes out on the wire. + Postrouting + + // NumHooks is the total number of hooks. + NumHooks +) + +// A RuleVerdict is what a rule decides should be done with a packet. +type RuleVerdict int + +const ( + // RuleAccept indicates the packet should continue through netstack. + RuleAccept RuleVerdict = iota + + // RuleDrop indicates the packet should be dropped. + RuleDrop + + // RuleJump indicates the packet should jump to another chain. + RuleJump + + // RuleReturn indicates the packet should return to the previous chain. + RuleReturn +) + +// IPTables holds all the tables for a netstack. +// +// +stateify savable +type IPTables struct { + connections ConnTrack + + reaper tcpip.Timer + + mu ipTablesRWMutex `state:"nosave"` + // v4Tables and v6tables map tableIDs to tables. They hold builtin + // tables only, not user tables. + // + // mu protects the array of tables, but not the tables themselves. + // +checklocks:mu + v4Tables [NumTables]Table + // + // mu protects the array of tables, but not the tables themselves. + // +checklocks:mu + v6Tables [NumTables]Table + // modified is whether tables have been modified at least once. It is + // used to elide the iptables performance overhead for workloads that + // don't utilize iptables. + // + // +checklocks:mu + modified bool +} + +// Modified returns whether iptables has been modified. It is inherently racy +// and intended for use only in tests. +func (it *IPTables) Modified() bool { + it.mu.Lock() + defer it.mu.Unlock() + return it.modified +} + +// VisitTargets traverses all the targets of all tables and replaces each with +// transform(target). +func (it *IPTables) VisitTargets(transform func(Target) Target) { + it.mu.Lock() + defer it.mu.Unlock() + + for tid := range it.v4Tables { + for i, rule := range it.v4Tables[tid].Rules { + it.v4Tables[tid].Rules[i].Target = transform(rule.Target) + } + } + for tid := range it.v6Tables { + for i, rule := range it.v6Tables[tid].Rules { + it.v6Tables[tid].Rules[i].Target = transform(rule.Target) + } + } +} + +// A Table defines a set of chains and hooks into the network stack. +// +// It is a list of Rules, entry points (BuiltinChains), and error handlers +// (Underflows). As packets traverse netstack, they hit hooks. When a packet +// hits a hook, iptables compares it to Rules starting from that hook's entry +// point. So if a packet hits the Input hook, we look up the corresponding +// entry point in BuiltinChains and jump to that point. +// +// If the Rule doesn't match the packet, iptables continues to the next Rule. +// If a Rule does match, it can issue a verdict on the packet (e.g. RuleAccept +// or RuleDrop) that causes the packet to stop traversing iptables. It can also +// jump to other rules or perform custom actions based on Rule.Target. +// +// Underflow Rules are invoked when a chain returns without reaching a verdict. +// +// +stateify savable +type Table struct { + // Rules holds the rules that make up the table. + Rules []Rule + + // BuiltinChains maps builtin chains to their entrypoint rule in Rules. + BuiltinChains [NumHooks]int + + // Underflows maps builtin chains to their underflow rule in Rules + // (i.e. the rule to execute if the chain returns without a verdict). + Underflows [NumHooks]int +} + +// ValidHooks returns a bitmap of the builtin hooks for the given table. +func (table *Table) ValidHooks() uint32 { + hooks := uint32(0) + for hook, ruleIdx := range table.BuiltinChains { + if ruleIdx != HookUnset { + hooks |= 1 << hook + } + } + return hooks +} + +// A Rule is a packet processing rule. It consists of two pieces. First it +// contains zero or more matchers, each of which is a specification of which +// packets this rule applies to. If there are no matchers in the rule, it +// applies to any packet. +// +// +stateify savable +type Rule struct { + // Filter holds basic IP filtering fields common to every rule. + Filter IPHeaderFilter + + // Matchers is the list of matchers for this rule. + Matchers []Matcher + + // Target is the action to invoke if all the matchers match the packet. + Target Target +} + +// IPHeaderFilter performs basic IP header matching common to every rule. +// +// +stateify savable +type IPHeaderFilter struct { + // Protocol matches the transport protocol. + Protocol tcpip.TransportProtocolNumber + + // CheckProtocol determines whether the Protocol field should be + // checked during matching. + CheckProtocol bool + + // Dst matches the destination IP address. + Dst tcpip.Address + + // DstMask masks bits of the destination IP address when comparing with + // Dst. + DstMask tcpip.Address + + // DstInvert inverts the meaning of the destination IP check, i.e. when + // true the filter will match packets that fail the destination + // comparison. + DstInvert bool + + // Src matches the source IP address. + Src tcpip.Address + + // SrcMask masks bits of the source IP address when comparing with Src. + SrcMask tcpip.Address + + // SrcInvert inverts the meaning of the source IP check, i.e. when true the + // filter will match packets that fail the source comparison. + SrcInvert bool + + // InputInterface matches the name of the incoming interface for the packet. + InputInterface string + + // InputInterfaceMask masks the characters of the interface name when + // comparing with InputInterface. + InputInterfaceMask string + + // InputInterfaceInvert inverts the meaning of incoming interface check, + // i.e. when true the filter will match packets that fail the incoming + // interface comparison. + InputInterfaceInvert bool + + // OutputInterface matches the name of the outgoing interface for the packet. + OutputInterface string + + // OutputInterfaceMask masks the characters of the interface name when + // comparing with OutputInterface. + OutputInterfaceMask string + + // OutputInterfaceInvert inverts the meaning of outgoing interface check, + // i.e. when true the filter will match packets that fail the outgoing + // interface comparison. + OutputInterfaceInvert bool +} + +// EmptyFilter4 returns an initialized IPv4 header filter. +func EmptyFilter4() IPHeaderFilter { + return IPHeaderFilter{ + Dst: tcpip.AddrFrom4([4]byte{}), + DstMask: tcpip.AddrFrom4([4]byte{}), + Src: tcpip.AddrFrom4([4]byte{}), + SrcMask: tcpip.AddrFrom4([4]byte{}), + } +} + +// EmptyFilter6 returns an initialized IPv6 header filter. +func EmptyFilter6() IPHeaderFilter { + return IPHeaderFilter{ + Dst: tcpip.AddrFrom16([16]byte{}), + DstMask: tcpip.AddrFrom16([16]byte{}), + Src: tcpip.AddrFrom16([16]byte{}), + SrcMask: tcpip.AddrFrom16([16]byte{}), + } +} + +// match returns whether pkt matches the filter. +// +// Preconditions: pkt.NetworkHeader is set and is at least of the minimal IPv4 +// or IPv6 header length. +func (fl IPHeaderFilter) match(pkt *PacketBuffer, hook Hook, inNicName, outNicName string) bool { + // Extract header fields. + var ( + transProto tcpip.TransportProtocolNumber + dstAddr tcpip.Address + srcAddr tcpip.Address + ) + switch proto := pkt.NetworkProtocolNumber; proto { + case header.IPv4ProtocolNumber: + hdr := header.IPv4(pkt.NetworkHeader().Slice()) + transProto = hdr.TransportProtocol() + dstAddr = hdr.DestinationAddress() + srcAddr = hdr.SourceAddress() + + case header.IPv6ProtocolNumber: + hdr := header.IPv6(pkt.NetworkHeader().Slice()) + transProto = hdr.TransportProtocol() + dstAddr = hdr.DestinationAddress() + srcAddr = hdr.SourceAddress() + + default: + panic(fmt.Sprintf("unknown network protocol with EtherType: %d", proto)) + } + + // Check the transport protocol. + if fl.CheckProtocol && fl.Protocol != transProto { + return false + } + + // Check the addresses. + if !filterAddress(dstAddr, fl.DstMask, fl.Dst, fl.DstInvert) || + !filterAddress(srcAddr, fl.SrcMask, fl.Src, fl.SrcInvert) { + return false + } + + switch hook { + case Prerouting, Input: + return matchIfName(inNicName, fl.InputInterface, fl.InputInterfaceInvert) + case Postrouting, Output: + return matchIfName(outNicName, fl.OutputInterface, fl.OutputInterfaceInvert) + case Forward: + if !matchIfName(inNicName, fl.InputInterface, fl.InputInterfaceInvert) { + return false + } + + if !matchIfName(outNicName, fl.OutputInterface, fl.OutputInterfaceInvert) { + return false + } + + return true + default: + panic(fmt.Sprintf("unknown hook: %d", hook)) + } +} + +func matchIfName(nicName string, ifName string, invert bool) bool { + n := len(ifName) + if n == 0 { + // If the interface name is omitted in the filter, any interface will match. + return true + } + // If the interface name ends with '+', any interface which begins with the + // name should be matched. + var matches bool + if strings.HasSuffix(ifName, "+") { + matches = strings.HasPrefix(nicName, ifName[:n-1]) + } else { + matches = nicName == ifName + } + return matches != invert +} + +// NetworkProtocol returns the protocol (IPv4 or IPv6) on to which the header +// applies. +func (fl IPHeaderFilter) NetworkProtocol() tcpip.NetworkProtocolNumber { + switch fl.Src.BitLen() { + case header.IPv4AddressSizeBits: + return header.IPv4ProtocolNumber + case header.IPv6AddressSizeBits: + return header.IPv6ProtocolNumber + } + panic(fmt.Sprintf("invalid address in IPHeaderFilter: %s", fl.Src)) +} + +// filterAddress returns whether addr matches the filter. +func filterAddress(addr, mask, filterAddr tcpip.Address, invert bool) bool { + matches := true + addrBytes := addr.AsSlice() + maskBytes := mask.AsSlice() + filterBytes := filterAddr.AsSlice() + for i := range filterAddr.AsSlice() { + if addrBytes[i]&maskBytes[i] != filterBytes[i] { + matches = false + break + } + } + return matches != invert +} + +// A Matcher is the interface for matching packets. +type Matcher interface { + // Match returns whether the packet matches and whether the packet + // should be "hotdropped", i.e. dropped immediately. This is usually + // used for suspicious packets. + // + // Precondition: packet.NetworkHeader is set. + Match(hook Hook, packet *PacketBuffer, inputInterfaceName, outputInterfaceName string) (matches bool, hotdrop bool) +} + +// A Target is the interface for taking an action for a packet. +type Target interface { + // Action takes an action on the packet and returns a verdict on how + // traversal should (or should not) continue. If the return value is + // Jump, it also returns the index of the rule to jump to. + Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) +} diff --git a/pkg/tcpip/stack/multi_port_endpoint_mutex.go b/pkg/tcpip/stack/multi_port_endpoint_mutex.go new file mode 100644 index 0000000..9e73cfb --- /dev/null +++ b/pkg/tcpip/stack/multi_port_endpoint_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type multiPortEndpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var multiPortEndpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type multiPortEndpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *multiPortEndpointRWMutex) Lock() { + locking.AddGLock(multiPortEndpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *multiPortEndpointRWMutex) NestedLock(i multiPortEndpointlockNameIndex) { + locking.AddGLock(multiPortEndpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *multiPortEndpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(multiPortEndpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *multiPortEndpointRWMutex) NestedUnlock(i multiPortEndpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(multiPortEndpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *multiPortEndpointRWMutex) RLock() { + locking.AddGLock(multiPortEndpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *multiPortEndpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(multiPortEndpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *multiPortEndpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *multiPortEndpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *multiPortEndpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var multiPortEndpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func multiPortEndpointinitLockNames() {} + +func init() { + multiPortEndpointinitLockNames() + multiPortEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(multiPortEndpointRWMutex{}), multiPortEndpointlockNames) +} diff --git a/pkg/tcpip/stack/neighbor_cache.go b/pkg/tcpip/stack/neighbor_cache.go new file mode 100644 index 0000000..dfe71f1 --- /dev/null +++ b/pkg/tcpip/stack/neighbor_cache.go @@ -0,0 +1,314 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// NeighborCacheSize is the size of the neighborCache. Exceeding this size will +// result in the least recently used entry being evicted. +const NeighborCacheSize = 512 // max entries per interface + +// NeighborStats holds metrics for the neighbor table. +type NeighborStats struct { + // UnreachableEntryLookups counts the number of lookups performed on an + // entry in Unreachable state. + UnreachableEntryLookups *tcpip.StatCounter +} + +// +stateify savable +type dynamicCacheEntry struct { + lru neighborEntryList + + // count tracks the amount of dynamic entries in the cache. This is + // needed since static entries do not count towards the LRU cache + // eviction strategy. + count uint16 +} + +// +stateify savable +type neighborCacheMu struct { + neighborCacheRWMutex `state:"nosave"` + + cache map[tcpip.Address]*neighborEntry + dynamic dynamicCacheEntry +} + +// neighborCache maps IP addresses to link addresses. It uses the Least +// Recently Used (LRU) eviction strategy to implement a bounded cache for +// dynamically acquired entries. It contains the state machine and configuration +// for running Neighbor Unreachability Detection (NUD). +// +// There are two types of entries in the neighbor cache: +// 1. Dynamic entries are discovered automatically by neighbor discovery +// protocols (e.g. ARP, NDP). These protocols will attempt to reconfirm +// reachability with the device once the entry's state becomes Stale. +// 2. Static entries are explicitly added by a user and have no expiration. +// Their state is always Static. The amount of static entries stored in the +// cache is unbounded. +// +// +stateify savable +type neighborCache struct { + nic *nic + state *NUDState + linkRes LinkAddressResolver + mu neighborCacheMu +} + +// getOrCreateEntry retrieves a cache entry associated with addr. The +// returned entry is always refreshed in the cache (it is reachable via the +// map, and its place is bumped in LRU). +// +// If a matching entry exists in the cache, it is returned. If no matching +// entry exists and the cache is full, an existing entry is evicted via LRU, +// reset to state incomplete, and returned. If no matching entry exists and the +// cache is not full, a new entry with state incomplete is allocated and +// returned. +func (n *neighborCache) getOrCreateEntry(remoteAddr tcpip.Address) *neighborEntry { + n.mu.Lock() + defer n.mu.Unlock() + + if entry, ok := n.mu.cache[remoteAddr]; ok { + entry.mu.RLock() + if entry.mu.neigh.State != Static { + n.mu.dynamic.lru.Remove(entry) + n.mu.dynamic.lru.PushFront(entry) + } + entry.mu.RUnlock() + return entry + } + + // The entry that needs to be created must be dynamic since all static + // entries are directly added to the cache via addStaticEntry. + entry := newNeighborEntry(n, remoteAddr, n.state) + if n.mu.dynamic.count == NeighborCacheSize { + e := n.mu.dynamic.lru.Back() + e.mu.Lock() + + delete(n.mu.cache, e.mu.neigh.Addr) + n.mu.dynamic.lru.Remove(e) + n.mu.dynamic.count-- + + e.removeLocked() + e.mu.Unlock() + } + n.mu.cache[remoteAddr] = entry + n.mu.dynamic.lru.PushFront(entry) + n.mu.dynamic.count++ + return entry +} + +// entry looks up neighbor information matching the remote address, and returns +// it if readily available. +// +// Returns ErrWouldBlock if the link address is not readily available, along +// with a notification channel for the caller to block on. Triggers address +// resolution asynchronously. +// +// If onResolve is provided, it will be called either immediately, if resolution +// is not required, or when address resolution is complete, with the resolved +// link address and whether resolution succeeded. After any callbacks have been +// called, the returned notification channel is closed. +// +// NB: if a callback is provided, it should not call into the neighbor cache. +// +// If specified, the local address must be an address local to the interface the +// neighbor cache belongs to. The local address is the source address of a +// packet prompting NUD/link address resolution. +func (n *neighborCache) entry(remoteAddr, localAddr tcpip.Address, onResolve func(LinkResolutionResult)) (*neighborEntry, <-chan struct{}, tcpip.Error) { + entry := n.getOrCreateEntry(remoteAddr) + entry.mu.Lock() + defer entry.mu.Unlock() + + switch s := entry.mu.neigh.State; s { + case Stale: + entry.handlePacketQueuedLocked(localAddr) + fallthrough + case Reachable, Static, Delay, Probe: + // As per RFC 4861 section 7.3.3: + // "Neighbor Unreachability Detection operates in parallel with the sending + // of packets to a neighbor. While reasserting a neighbor's reachability, + // a node continues sending packets to that neighbor using the cached + // link-layer address." + if onResolve != nil { + onResolve(LinkResolutionResult{LinkAddress: entry.mu.neigh.LinkAddr, Err: nil}) + } + return entry, nil, nil + case Unknown, Incomplete, Unreachable: + if onResolve != nil { + entry.mu.onResolve = append(entry.mu.onResolve, onResolve) + } + if entry.mu.done == nil { + // Address resolution needs to be initiated. + entry.mu.done = make(chan struct{}) + } + entry.handlePacketQueuedLocked(localAddr) + return entry, entry.mu.done, &tcpip.ErrWouldBlock{} + default: + panic(fmt.Sprintf("Invalid cache entry state: %s", s)) + } +} + +// entries returns all entries in the neighbor cache. +func (n *neighborCache) entries() []NeighborEntry { + n.mu.RLock() + defer n.mu.RUnlock() + + entries := make([]NeighborEntry, 0, len(n.mu.cache)) + for _, entry := range n.mu.cache { + entry.mu.RLock() + entries = append(entries, entry.mu.neigh) + entry.mu.RUnlock() + } + return entries +} + +// addStaticEntry adds a static entry to the neighbor cache, mapping an IP +// address to a link address. If a dynamic entry exists in the neighbor cache +// with the same address, it will be replaced with this static entry. If a +// static entry exists with the same address but different link address, it +// will be updated with the new link address. If a static entry exists with the +// same address and link address, nothing will happen. +func (n *neighborCache) addStaticEntry(addr tcpip.Address, linkAddr tcpip.LinkAddress) { + n.mu.Lock() + defer n.mu.Unlock() + + if entry, ok := n.mu.cache[addr]; ok { + entry.mu.Lock() + if entry.mu.neigh.State != Static { + // Dynamic entry found with the same address. + n.mu.dynamic.lru.Remove(entry) + n.mu.dynamic.count-- + } else if entry.mu.neigh.LinkAddr == linkAddr { + // Static entry found with the same address and link address. + entry.mu.Unlock() + return + } else { + // Static entry found with the same address but different link address. + entry.mu.neigh.LinkAddr = linkAddr + entry.dispatchChangeEventLocked() + entry.mu.Unlock() + return + } + + entry.removeLocked() + entry.mu.Unlock() + } + + entry := newStaticNeighborEntry(n, addr, linkAddr, n.state) + n.mu.cache[addr] = entry + + entry.mu.Lock() + defer entry.mu.Unlock() + entry.dispatchAddEventLocked() +} + +// removeEntry removes a dynamic or static entry by address from the neighbor +// cache. Returns true if the entry was found and deleted. +func (n *neighborCache) removeEntry(addr tcpip.Address) bool { + n.mu.Lock() + defer n.mu.Unlock() + + entry, ok := n.mu.cache[addr] + if !ok { + return false + } + + entry.mu.Lock() + defer entry.mu.Unlock() + + if entry.mu.neigh.State != Static { + n.mu.dynamic.lru.Remove(entry) + n.mu.dynamic.count-- + } + + entry.removeLocked() + delete(n.mu.cache, entry.mu.neigh.Addr) + return true +} + +// clear removes all dynamic and static entries from the neighbor cache. +func (n *neighborCache) clear() { + n.mu.Lock() + defer n.mu.Unlock() + + for _, entry := range n.mu.cache { + entry.mu.Lock() + entry.removeLocked() + entry.mu.Unlock() + } + + n.mu.dynamic.lru = neighborEntryList{} + clear(n.mu.cache) + n.mu.dynamic.count = 0 +} + +// config returns the NUD configuration. +func (n *neighborCache) config() NUDConfigurations { + return n.state.Config() +} + +// setConfig changes the NUD configuration. +// +// If config contains invalid NUD configuration values, it will be fixed to +// use default values for the erroneous values. +func (n *neighborCache) setConfig(config NUDConfigurations) { + config.resetInvalidFields() + n.state.SetConfig(config) +} + +// handleProbe handles a neighbor probe as defined by RFC 4861 section 7.2.3. +// +// Validation of the probe is expected to be handled by the caller. +func (n *neighborCache) handleProbe(remoteAddr tcpip.Address, remoteLinkAddr tcpip.LinkAddress) { + entry := n.getOrCreateEntry(remoteAddr) + entry.mu.Lock() + entry.handleProbeLocked(remoteLinkAddr) + entry.mu.Unlock() +} + +// handleConfirmation handles a neighbor confirmation as defined by +// RFC 4861 section 7.2.5. +// +// Validation of the confirmation is expected to be handled by the caller. +func (n *neighborCache) handleConfirmation(addr tcpip.Address, linkAddr tcpip.LinkAddress, flags ReachabilityConfirmationFlags) { + n.mu.RLock() + entry, ok := n.mu.cache[addr] + n.mu.RUnlock() + if ok { + entry.mu.Lock() + entry.handleConfirmationLocked(linkAddr, flags) + entry.mu.Unlock() + } else { + // The confirmation SHOULD be silently discarded if the recipient did not + // initiate any communication with the target. This is indicated if there is + // no matching entry for the remote address. + n.nic.stats.neighbor.droppedConfirmationForNoninitiatedNeighbor.Increment() + } +} + +func (n *neighborCache) init(nic *nic, r LinkAddressResolver) { + *n = neighborCache{ + nic: nic, + state: NewNUDState(nic.stack.nudConfigs, nic.stack.clock, nic.stack.insecureRNG), + linkRes: r, + } + n.mu.Lock() + n.mu.cache = make(map[tcpip.Address]*neighborEntry, NeighborCacheSize) + n.mu.Unlock() +} diff --git a/pkg/tcpip/stack/neighbor_cache_mutex.go b/pkg/tcpip/stack/neighbor_cache_mutex.go new file mode 100644 index 0000000..8d5a9e4 --- /dev/null +++ b/pkg/tcpip/stack/neighbor_cache_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type neighborCacheRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var neighborCachelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type neighborCachelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *neighborCacheRWMutex) Lock() { + locking.AddGLock(neighborCacheprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *neighborCacheRWMutex) NestedLock(i neighborCachelockNameIndex) { + locking.AddGLock(neighborCacheprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *neighborCacheRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(neighborCacheprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *neighborCacheRWMutex) NestedUnlock(i neighborCachelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(neighborCacheprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *neighborCacheRWMutex) RLock() { + locking.AddGLock(neighborCacheprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *neighborCacheRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(neighborCacheprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *neighborCacheRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *neighborCacheRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *neighborCacheRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var neighborCacheprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func neighborCacheinitLockNames() {} + +func init() { + neighborCacheinitLockNames() + neighborCacheprefixIndex = locking.NewMutexClass(reflect.TypeOf(neighborCacheRWMutex{}), neighborCachelockNames) +} diff --git a/pkg/tcpip/stack/neighbor_entry.go b/pkg/tcpip/stack/neighbor_entry.go new file mode 100644 index 0000000..9a34bf6 --- /dev/null +++ b/pkg/tcpip/stack/neighbor_entry.go @@ -0,0 +1,653 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +const ( + // immediateDuration is a duration of zero for scheduling work that needs to + // be done immediately but asynchronously to avoid deadlock. + immediateDuration time.Duration = 0 +) + +// NeighborEntry describes a neighboring device in the local network. +// +// +stateify savable +type NeighborEntry struct { + Addr tcpip.Address + LinkAddr tcpip.LinkAddress + State NeighborState + UpdatedAt tcpip.MonotonicTime +} + +// NeighborState defines the state of a NeighborEntry within the Neighbor +// Unreachability Detection state machine, as per RFC 4861 section 7.3.2 and +// RFC 7048. +type NeighborState uint8 + +const ( + // Unknown means reachability has not been verified yet. This is the initial + // state of entries that have been created automatically by the Neighbor + // Unreachability Detection state machine. + Unknown NeighborState = iota + // Incomplete means that there is an outstanding request to resolve the + // address. + Incomplete + // Reachable means the path to the neighbor is functioning properly for both + // receive and transmit paths. + Reachable + // Stale means reachability to the neighbor is unknown, but packets are still + // able to be transmitted to the possibly stale link address. + Stale + // Delay means reachability to the neighbor is unknown and pending + // confirmation from an upper-level protocol like TCP, but packets are still + // able to be transmitted to the possibly stale link address. + Delay + // Probe means a reachability confirmation is actively being sought by + // periodically retransmitting reachability probes until a reachability + // confirmation is received, or until the maximum number of probes has been + // sent. + Probe + // Static describes entries that have been explicitly added by the user. They + // do not expire and are not deleted until explicitly removed. + Static + // Unreachable means reachability confirmation failed; the maximum number of + // reachability probes has been sent and no replies have been received. + // + // TODO(gvisor.dev/issue/5472): Add the following sentence when we implement + // RFC 7048: "Packets continue to be sent to the neighbor while + // re-attempting to resolve the address." + Unreachable +) + +// +stateify savable +type timer struct { + // done indicates to the timer that the timer was stopped. + done *bool + + timer tcpip.Timer `state:"nosave"` +} + +// +stateify savable +type neighborEntryMu struct { + neighborEntryRWMutex `state:"nosave"` + + neigh NeighborEntry + + // done is closed when address resolution is complete. It is nil iff s is + // incomplete and resolution is not yet in progress. + done chan struct{} `state:"nosave"` + + // onResolve is called with the result of address resolution. + onResolve []func(LinkResolutionResult) `state:"nosave"` + + isRouter bool + + timer timer +} + +// neighborEntry implements a neighbor entry's individual node behavior, as per +// RFC 4861 section 7.3.3. Neighbor Unreachability Detection operates in +// parallel with the sending of packets to a neighbor, necessitating the +// entry's lock to be acquired for all operations. +// +// +stateify savable +type neighborEntry struct { + neighborEntryEntry + + cache *neighborCache + + // nudState points to the Neighbor Unreachability Detection configuration. + nudState *NUDState + + mu neighborEntryMu +} + +// newNeighborEntry creates a neighbor cache entry starting at the default +// state, Unknown. Transition out of Unknown by calling either +// `handlePacketQueuedLocked` or `handleProbeLocked` on the newly created +// neighborEntry. +func newNeighborEntry(cache *neighborCache, remoteAddr tcpip.Address, nudState *NUDState) *neighborEntry { + n := &neighborEntry{ + cache: cache, + nudState: nudState, + } + n.mu.Lock() + n.mu.neigh = NeighborEntry{ + Addr: remoteAddr, + State: Unknown, + } + n.mu.Unlock() + return n +} + +// newStaticNeighborEntry creates a neighbor cache entry starting at the +// Static state. The entry can only transition out of Static by directly +// calling `setStateLocked`. +func newStaticNeighborEntry(cache *neighborCache, addr tcpip.Address, linkAddr tcpip.LinkAddress, state *NUDState) *neighborEntry { + entry := NeighborEntry{ + Addr: addr, + LinkAddr: linkAddr, + State: Static, + UpdatedAt: cache.nic.stack.clock.NowMonotonic(), + } + n := &neighborEntry{ + cache: cache, + nudState: state, + } + n.mu.Lock() + n.mu.neigh = entry + n.mu.Unlock() + return n +} + +// notifyCompletionLocked notifies those waiting for address resolution, with +// the link address if resolution completed successfully. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) notifyCompletionLocked(err tcpip.Error) { + res := LinkResolutionResult{LinkAddress: e.mu.neigh.LinkAddr, Err: err} + for _, callback := range e.mu.onResolve { + callback(res) + } + e.mu.onResolve = nil + if ch := e.mu.done; ch != nil { + close(ch) + e.mu.done = nil + // Dequeue the pending packets asynchronously to not hold up the current + // goroutine as writing packets may be a costly operation. + // + // At the time of writing, when writing packets, a neighbor's link address + // is resolved (which ends up obtaining the entry's lock) while holding the + // link resolution queue's lock. Dequeuing packets asynchronously avoids a + // lock ordering violation. + // + // NB: this is equivalent to spawning a goroutine directly using the go + // keyword but allows tests that use manual clocks to deterministically + // wait for this work to complete. + e.cache.nic.stack.clock.AfterFunc(0, func() { + e.cache.nic.linkResQueue.dequeue(ch, e.mu.neigh.LinkAddr, err) + }) + } +} + +// dispatchAddEventLocked signals to stack's NUD Dispatcher that the entry has +// been added. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) dispatchAddEventLocked() { + if nudDisp := e.cache.nic.stack.nudDisp; nudDisp != nil { + nudDisp.OnNeighborAdded(e.cache.nic.id, e.mu.neigh) + } +} + +// dispatchChangeEventLocked signals to stack's NUD Dispatcher that the entry +// has changed state or link-layer address. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) dispatchChangeEventLocked() { + if nudDisp := e.cache.nic.stack.nudDisp; nudDisp != nil { + nudDisp.OnNeighborChanged(e.cache.nic.id, e.mu.neigh) + } +} + +// dispatchRemoveEventLocked signals to stack's NUD Dispatcher that the entry +// has been removed. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) dispatchRemoveEventLocked() { + if nudDisp := e.cache.nic.stack.nudDisp; nudDisp != nil { + nudDisp.OnNeighborRemoved(e.cache.nic.id, e.mu.neigh) + } +} + +// cancelTimerLocked cancels the currently scheduled action, if there is one. +// Entries in Unknown, Stale, or Static state do not have a scheduled action. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) cancelTimerLocked() { + if e.mu.timer.timer != nil { + e.mu.timer.timer.Stop() + *e.mu.timer.done = true + + e.mu.timer = timer{} + } +} + +// removeLocked prepares the entry for removal. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) removeLocked() { + e.mu.neigh.UpdatedAt = e.cache.nic.stack.clock.NowMonotonic() + e.dispatchRemoveEventLocked() + // Set state to unknown to invalidate this entry if it's cached in a Route. + e.setStateLocked(Unknown) + e.cancelTimerLocked() + // TODO(https://gvisor.dev/issues/5583): test the case where this function is + // called during resolution; that can happen in at least these scenarios: + // + // - manual address removal during resolution + // + // - neighbor cache eviction during resolution + e.notifyCompletionLocked(&tcpip.ErrAborted{}) +} + +// setStateLocked transitions the entry to the specified state immediately. +// +// Follows the logic defined in RFC 4861 section 7.3.3. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) setStateLocked(next NeighborState) { + e.cancelTimerLocked() + + prev := e.mu.neigh.State + e.mu.neigh.State = next + e.mu.neigh.UpdatedAt = e.cache.nic.stack.clock.NowMonotonic() + config := e.nudState.Config() + + switch next { + case Incomplete: + panic(fmt.Sprintf("should never transition to Incomplete with setStateLocked; neigh = %#v, prev state = %s", e.mu.neigh, prev)) + + case Reachable: + // Protected by e.mu. + done := false + + e.mu.timer = timer{ + done: &done, + timer: e.cache.nic.stack.Clock().AfterFunc(e.nudState.ReachableTime(), func() { + e.mu.Lock() + defer e.mu.Unlock() + + if done { + // The timer was stopped because the entry changed state. + return + } + + e.setStateLocked(Stale) + e.dispatchChangeEventLocked() + }), + } + + case Delay: + // Protected by e.mu. + done := false + + e.mu.timer = timer{ + done: &done, + timer: e.cache.nic.stack.Clock().AfterFunc(config.DelayFirstProbeTime, func() { + e.mu.Lock() + defer e.mu.Unlock() + + if done { + // The timer was stopped because the entry changed state. + return + } + + e.setStateLocked(Probe) + e.dispatchChangeEventLocked() + }), + } + + case Probe: + // Protected by e.mu. + done := false + + remaining := config.MaxUnicastProbes + addr := e.mu.neigh.Addr + linkAddr := e.mu.neigh.LinkAddr + + // Send a probe in another gorountine to free this thread of execution + // for finishing the state transition. This is necessary to escape the + // currently held lock so we can send the probe message without holding + // a shared lock. + e.mu.timer = timer{ + done: &done, + timer: e.cache.nic.stack.Clock().AfterFunc(immediateDuration, func() { + var err tcpip.Error = &tcpip.ErrTimeout{} + if remaining != 0 { + err = e.cache.linkRes.LinkAddressRequest(addr, tcpip.Address{} /* localAddr */, linkAddr) + } + + e.mu.Lock() + defer e.mu.Unlock() + + if done { + // The timer was stopped because the entry changed state. + return + } + + if err != nil { + e.setStateLocked(Unreachable) + e.notifyCompletionLocked(err) + e.dispatchChangeEventLocked() + return + } + + remaining-- + e.mu.timer.timer.Reset(config.RetransmitTimer) + }), + } + + case Unreachable: + + case Unknown, Stale, Static: + // Do nothing + + default: + panic(fmt.Sprintf("Invalid state transition from %q to %q", prev, next)) + } +} + +// handlePacketQueuedLocked advances the state machine according to a packet +// being queued for outgoing transmission. +// +// Follows the logic defined in RFC 4861 section 7.3.3. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) handlePacketQueuedLocked(localAddr tcpip.Address) { + switch e.mu.neigh.State { + case Unknown, Unreachable: + prev := e.mu.neigh.State + e.mu.neigh.State = Incomplete + e.mu.neigh.UpdatedAt = e.cache.nic.stack.clock.NowMonotonic() + + switch prev { + case Unknown: + e.dispatchAddEventLocked() + case Unreachable: + e.dispatchChangeEventLocked() + e.cache.nic.stats.neighbor.unreachableEntryLookups.Increment() + } + + config := e.nudState.Config() + + // Protected by e.mu. + done := false + + remaining := config.MaxMulticastProbes + addr := e.mu.neigh.Addr + + // Send a probe in another gorountine to free this thread of execution + // for finishing the state transition. This is necessary to escape the + // currently held lock so we can send the probe message without holding + // a shared lock. + e.mu.timer = timer{ + done: &done, + timer: e.cache.nic.stack.Clock().AfterFunc(immediateDuration, func() { + var err tcpip.Error = &tcpip.ErrTimeout{} + if remaining != 0 { + // As per RFC 4861 section 7.2.2: + // + // If the source address of the packet prompting the solicitation is + // the same as one of the addresses assigned to the outgoing interface, + // that address SHOULD be placed in the IP Source Address of the + // outgoing solicitation. + // + err = e.cache.linkRes.LinkAddressRequest(addr, localAddr, "" /* linkAddr */) + } + + e.mu.Lock() + defer e.mu.Unlock() + + if done { + // The timer was stopped because the entry changed state. + return + } + + if err != nil { + e.setStateLocked(Unreachable) + e.notifyCompletionLocked(err) + e.dispatchChangeEventLocked() + return + } + + remaining-- + e.mu.timer.timer.Reset(config.RetransmitTimer) + }), + } + + case Stale: + e.setStateLocked(Delay) + e.dispatchChangeEventLocked() + + case Incomplete, Reachable, Delay, Probe, Static: + // Do nothing + default: + panic(fmt.Sprintf("Invalid cache entry state: %s", e.mu.neigh.State)) + } +} + +// handleProbeLocked processes an incoming neighbor probe (e.g. ARP request or +// Neighbor Solicitation for ARP or NDP, respectively). +// +// Follows the logic defined in RFC 4861 section 7.2.3. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) handleProbeLocked(remoteLinkAddr tcpip.LinkAddress) { + // Probes MUST be silently discarded if the target address is tentative, does + // not exist, or not bound to the NIC as per RFC 4861 section 7.2.3. These + // checks MUST be done by the NetworkEndpoint. + + switch e.mu.neigh.State { + case Unknown: + e.mu.neigh.LinkAddr = remoteLinkAddr + e.setStateLocked(Stale) + e.dispatchAddEventLocked() + + case Incomplete: + // "If an entry already exists, and the cached link-layer address + // differs from the one in the received Source Link-Layer option, the + // cached address should be replaced by the received address, and the + // entry's reachability state MUST be set to STALE." + // - RFC 4861 section 7.2.3 + e.mu.neigh.LinkAddr = remoteLinkAddr + e.setStateLocked(Stale) + e.notifyCompletionLocked(nil) + e.dispatchChangeEventLocked() + + case Reachable, Delay, Probe: + if e.mu.neigh.LinkAddr != remoteLinkAddr { + e.mu.neigh.LinkAddr = remoteLinkAddr + e.setStateLocked(Stale) + e.dispatchChangeEventLocked() + } + + case Stale: + if e.mu.neigh.LinkAddr != remoteLinkAddr { + e.mu.neigh.LinkAddr = remoteLinkAddr + e.dispatchChangeEventLocked() + } + + case Unreachable: + // TODO(gvisor.dev/issue/5472): Do not change the entry if the link + // address is the same, as per RFC 7048. + e.mu.neigh.LinkAddr = remoteLinkAddr + e.setStateLocked(Stale) + e.dispatchChangeEventLocked() + + case Static: + // Do nothing + + default: + panic(fmt.Sprintf("Invalid cache entry state: %s", e.mu.neigh.State)) + } +} + +// handleConfirmationLocked processes an incoming neighbor confirmation +// (e.g. ARP reply or Neighbor Advertisement for ARP or NDP, respectively). +// +// Follows the state machine defined by RFC 4861 section 7.2.5. +// +// TODO(gvisor.dev/issue/2277): To protect against ARP poisoning and other +// attacks against NDP functions, Secure Neighbor Discovery (SEND) Protocol +// should be deployed where preventing access to the broadcast segment might +// not be possible. SEND uses RSA key pairs to produce Cryptographically +// Generated Addresses (CGA), as defined in RFC 3972. This ensures that the +// claimed source of an NDP message is the owner of the claimed address. +// +// Precondition: e.mu MUST be locked. +func (e *neighborEntry) handleConfirmationLocked(linkAddr tcpip.LinkAddress, flags ReachabilityConfirmationFlags) { + switch e.mu.neigh.State { + case Incomplete: + if len(linkAddr) == 0 { + // "If the link layer has addresses and no Target Link-Layer Address + // option is included, the receiving node SHOULD silently discard the + // received advertisement." - RFC 4861 section 7.2.5 + e.cache.nic.stats.neighbor.droppedInvalidLinkAddressConfirmations.Increment() + break + } + + e.mu.neigh.LinkAddr = linkAddr + if flags.Solicited { + e.setStateLocked(Reachable) + } else { + e.setStateLocked(Stale) + } + e.dispatchChangeEventLocked() + e.mu.isRouter = flags.IsRouter + e.notifyCompletionLocked(nil) + + // "Note that the Override flag is ignored if the entry is in the + // INCOMPLETE state." - RFC 4861 section 7.2.5 + + case Reachable, Stale, Delay, Probe: + isLinkAddrDifferent := len(linkAddr) != 0 && e.mu.neigh.LinkAddr != linkAddr + + if isLinkAddrDifferent { + if !flags.Override { + if e.mu.neigh.State == Reachable { + e.setStateLocked(Stale) + e.dispatchChangeEventLocked() + } + break + } + + e.mu.neigh.LinkAddr = linkAddr + + if !flags.Solicited { + if e.mu.neigh.State != Stale { + e.setStateLocked(Stale) + e.dispatchChangeEventLocked() + } else { + // Notify the LinkAddr change, even though NUD state hasn't changed. + e.dispatchChangeEventLocked() + } + break + } + } + + if flags.Solicited && (flags.Override || !isLinkAddrDifferent) { + wasReachable := e.mu.neigh.State == Reachable + // Set state to Reachable again to refresh timers. + e.setStateLocked(Reachable) + e.notifyCompletionLocked(nil) + if !wasReachable { + e.dispatchChangeEventLocked() + } + } + + if e.mu.isRouter && !flags.IsRouter && header.IsV6UnicastAddress(e.mu.neigh.Addr) { + // "In those cases where the IsRouter flag changes from TRUE to FALSE as + // a result of this update, the node MUST remove that router from the + // Default Router List and update the Destination Cache entries for all + // destinations using that neighbor as a router as specified in Section + // 7.3.3. This is needed to detect when a node that is used as a router + // stops forwarding packets due to being configured as a host." + // - RFC 4861 section 7.2.5 + // + // TODO(gvisor.dev/issue/4085): Remove the special casing we do for IPv6 + // here. + ep := e.cache.nic.getNetworkEndpoint(header.IPv6ProtocolNumber) + if ep == nil { + panic(fmt.Sprintf("have a neighbor entry for an IPv6 router but no IPv6 network endpoint")) + } + + if ndpEP, ok := ep.(NDPEndpoint); ok { + ndpEP.InvalidateDefaultRouter(e.mu.neigh.Addr) + } + } + e.mu.isRouter = flags.IsRouter + + case Unknown, Unreachable, Static: + // Do nothing + + default: + panic(fmt.Sprintf("Invalid cache entry state: %s", e.mu.neigh.State)) + } +} + +// handleUpperLevelConfirmation processes an incoming upper-level protocol +// (e.g. TCP acknowledgements) reachability confirmation. +func (e *neighborEntry) handleUpperLevelConfirmation() { + tryHandleConfirmation := func() bool { + switch e.mu.neigh.State { + case Stale, Delay, Probe: + return true + case Reachable: + // Avoid setStateLocked; Timer.Reset is cheaper. + // + // Note that setting the timer does not need to be protected by the + // entry's write lock since we do not modify the timer pointer, but the + // time the timer should fire. The timer should have internal locks to + // synchronize timer resets changes with the clock. + e.mu.timer.timer.Reset(e.nudState.ReachableTime()) + return false + case Unknown, Incomplete, Unreachable, Static: + // Do nothing + return false + default: + panic(fmt.Sprintf("Invalid cache entry state: %s", e.mu.neigh.State)) + } + } + + e.mu.RLock() + needsTransition := tryHandleConfirmation() + e.mu.RUnlock() + if !needsTransition { + return + } + + // We need to transition the neighbor to Reachable so take the write lock and + // perform the transition, but only if we still need the transition since the + // state could have changed since we dropped the read lock above. + e.mu.Lock() + defer e.mu.Unlock() + if needsTransition := tryHandleConfirmation(); needsTransition { + e.setStateLocked(Reachable) + e.dispatchChangeEventLocked() + } +} + +// getRemoteLinkAddress returns the entry's link address and whether that link +// address is valid. +func (e *neighborEntry) getRemoteLinkAddress() (tcpip.LinkAddress, bool) { + e.mu.RLock() + defer e.mu.RUnlock() + switch e.mu.neigh.State { + case Reachable, Static, Delay, Probe: + return e.mu.neigh.LinkAddr, true + case Unknown, Incomplete, Unreachable, Stale: + return "", false + default: + panic(fmt.Sprintf("invalid state for neighbor entry %v: %v", e.mu.neigh, e.mu.neigh.State)) + } +} diff --git a/pkg/tcpip/stack/neighbor_entry_list.go b/pkg/tcpip/stack/neighbor_entry_list.go new file mode 100644 index 0000000..3973d7c --- /dev/null +++ b/pkg/tcpip/stack/neighbor_entry_list.go @@ -0,0 +1,239 @@ +package stack + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type neighborEntryElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (neighborEntryElementMapper) linkerFor(elem *neighborEntry) *neighborEntry { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type neighborEntryList struct { + head *neighborEntry + tail *neighborEntry +} + +// Reset resets list l to the empty state. +func (l *neighborEntryList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *neighborEntryList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *neighborEntryList) Front() *neighborEntry { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *neighborEntryList) Back() *neighborEntry { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *neighborEntryList) Len() (count int) { + for e := l.Front(); e != nil; e = (neighborEntryElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *neighborEntryList) PushFront(e *neighborEntry) { + linker := neighborEntryElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + neighborEntryElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *neighborEntryList) PushFrontList(m *neighborEntryList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + neighborEntryElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + neighborEntryElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *neighborEntryList) PushBack(e *neighborEntry) { + linker := neighborEntryElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + neighborEntryElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *neighborEntryList) PushBackList(m *neighborEntryList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + neighborEntryElementMapper{}.linkerFor(l.tail).SetNext(m.head) + neighborEntryElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *neighborEntryList) InsertAfter(b, e *neighborEntry) { + bLinker := neighborEntryElementMapper{}.linkerFor(b) + eLinker := neighborEntryElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + neighborEntryElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *neighborEntryList) InsertBefore(a, e *neighborEntry) { + aLinker := neighborEntryElementMapper{}.linkerFor(a) + eLinker := neighborEntryElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + neighborEntryElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *neighborEntryList) Remove(e *neighborEntry) { + linker := neighborEntryElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + neighborEntryElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + neighborEntryElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type neighborEntryEntry struct { + next *neighborEntry + prev *neighborEntry +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *neighborEntryEntry) Next() *neighborEntry { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *neighborEntryEntry) Prev() *neighborEntry { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *neighborEntryEntry) SetNext(elem *neighborEntry) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *neighborEntryEntry) SetPrev(elem *neighborEntry) { + e.prev = elem +} diff --git a/pkg/tcpip/stack/neighbor_entry_mutex.go b/pkg/tcpip/stack/neighbor_entry_mutex.go new file mode 100644 index 0000000..3f28901 --- /dev/null +++ b/pkg/tcpip/stack/neighbor_entry_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type neighborEntryRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var neighborEntrylockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type neighborEntrylockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *neighborEntryRWMutex) Lock() { + locking.AddGLock(neighborEntryprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *neighborEntryRWMutex) NestedLock(i neighborEntrylockNameIndex) { + locking.AddGLock(neighborEntryprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *neighborEntryRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(neighborEntryprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *neighborEntryRWMutex) NestedUnlock(i neighborEntrylockNameIndex) { + m.mu.Unlock() + locking.DelGLock(neighborEntryprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *neighborEntryRWMutex) RLock() { + locking.AddGLock(neighborEntryprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *neighborEntryRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(neighborEntryprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *neighborEntryRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *neighborEntryRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *neighborEntryRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var neighborEntryprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func neighborEntryinitLockNames() {} + +func init() { + neighborEntryinitLockNames() + neighborEntryprefixIndex = locking.NewMutexClass(reflect.TypeOf(neighborEntryRWMutex{}), neighborEntrylockNames) +} diff --git a/pkg/tcpip/stack/neighborstate_string.go b/pkg/tcpip/stack/neighborstate_string.go new file mode 100644 index 0000000..cc5060e --- /dev/null +++ b/pkg/tcpip/stack/neighborstate_string.go @@ -0,0 +1,44 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by "stringer -type NeighborState"; DO NOT EDIT. + +package stack + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Unknown-0] + _ = x[Incomplete-1] + _ = x[Reachable-2] + _ = x[Stale-3] + _ = x[Delay-4] + _ = x[Probe-5] + _ = x[Static-6] + _ = x[Unreachable-7] +} + +const _NeighborState_name = "UnknownIncompleteReachableStaleDelayProbeStaticUnreachable" + +var _NeighborState_index = [...]uint8{0, 7, 17, 26, 31, 36, 41, 47, 58} + +func (i NeighborState) String() string { + if i >= NeighborState(len(_NeighborState_index)-1) { + return "NeighborState(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _NeighborState_name[_NeighborState_index[i]:_NeighborState_index[i+1]] +} diff --git a/pkg/tcpip/stack/nftables_types.go b/pkg/tcpip/stack/nftables_types.go new file mode 100644 index 0000000..113d8f1 --- /dev/null +++ b/pkg/tcpip/stack/nftables_types.go @@ -0,0 +1,170 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" +) + +// NFTablesInterface is an interface for evaluating chains. +type NFTablesInterface interface { + CheckPrerouting(pkt *PacketBuffer, af AddressFamily) bool + CheckInput(pkt *PacketBuffer, af AddressFamily) bool + CheckForward(pkt *PacketBuffer, af AddressFamily) bool + CheckOutput(pkt *PacketBuffer, af AddressFamily) bool + CheckPostrouting(pkt *PacketBuffer, af AddressFamily) bool + CheckIngress(pkt *PacketBuffer, af AddressFamily) bool + CheckEgress(pkt *PacketBuffer, af AddressFamily) bool +} + +// NFHook describes specific points in the pipeline where chains can be attached. +// Each address family has its own set of hooks (defined in supportedHooks). +// For IPv4/IPv6/Inet and Bridge, there are two possible pipelines: +// 1. Prerouting -> Input -> ~Local Process~ -> Output -> Postrouting +// 2. Prerouting -> Forward -> Postrouting +type NFHook uint16 + +const ( + // NFPrerouting Hook is supported by IPv4/IPv6/Inet, Bridge Families. + // Prerouting is evaluated before a packet is routed to applications or forwarded. + NFPrerouting NFHook = iota + + // NFInput Hook is supported by IPv4/IPv6/Inet, Bridge, ARP Families. + // Input is evaluated before a packet reaches an application. + NFInput + + // NFForward Hook is supported by IPv4/IPv6/Inet, Bridge Families. + // Forward is evaluated once it's decided that a packet should be forwarded to another host. + NFForward + + // NFOutput Hook is supported by IPv4/IPv6/Inet, Bridge, ARP Families. + // Output is evaluated after a packet is written by an application to be sent out. + NFOutput + + // NFPostrouting Hook is supported by IPv4/IPv6/Inet, Bridge Families. + // Postrouting is evaluated just before a packet goes out on the wire. + NFPostrouting + + // NFIngress Hook is supported by IPv4/IPv6/Inet, Bridge, Netdev Families. + // Ingress is the first hook evaluated, even before prerouting. + NFIngress + + // NFEgress Hook is supported by Netdev Family only. + // Egress is the last hook evaluated, after the packet has been processed by the + // application and is being prepared for transmission out of the network interface. + NFEgress + + // NFNumHooks is the number of hooks supported by nftables. + NFNumHooks +) + +// hookStrings maps hooks to their string representation. +var hookStrings = map[NFHook]string{ + NFPrerouting: "Prerouting", + NFInput: "Input", + NFForward: "Forward", + NFOutput: "Output", + NFPostrouting: "Postrouting", + NFIngress: "Ingress", + NFEgress: "Egress", +} + +// String for Hook returns the name of the hook. +func (h NFHook) String() string { + if hook, ok := hookStrings[h]; ok { + return hook + } + panic(fmt.Sprintf("invalid NFHook: %d", int(h))) +} + +// AddressFamily describes the 6 address families supported by nftables. +// The address family determines the type of packets processed, and each family +// contains hooks at specific stages of the packet processing pipeline. +type AddressFamily int + +const ( + // Unspec represents an unspecified address family. + Unspec AddressFamily = iota + + // IP represents IPv4 Family. + IP + + // IP6 represents IPv6 Family. + IP6 + + // Inet represents Internet Family for hybrid IPv4/IPv6 rules. + Inet + + // Arp represents ARP Family for IPv4 ARP packets. + Arp + + // Bridge represents Bridge Family for Ethernet packets across bridge devices. + Bridge + + // Netdev represents Netdev Family for packets on ingress and egress. + Netdev + + // NumAFs is the number of address families supported by nftables. + NumAFs +) + +// AddressFamilyStrings maps address families to their string representation. +var AddressFamilyStrings = map[AddressFamily]string{ + Unspec: "UNSPEC", + IP: "IPv4", + IP6: "IPv6", + Inet: "Internet (Both IPv4/IPv6)", + Arp: "ARP", + Bridge: "Bridge", + Netdev: "Netdev", +} + +// ValidateAddressFamily ensures the family address is valid (within bounds). +// Unspecified address family is not valid. It is only used to reference all address families. +func ValidateAddressFamily(family AddressFamily) error { + if family < 1 || family >= NumAFs { + return fmt.Errorf("invalid address family: %d", int(family)) + } + return nil +} + +// String for AddressFamily returns the name of the address family. +func (f AddressFamily) String() string { + if af, ok := AddressFamilyStrings[f]; ok { + return af + } + panic(fmt.Sprintf("invalid address family: %d", int(f))) +} + +// +// Verdict Implementation. +// There are two types of verdicts: +// 1. Netfilter (External) Verdicts: Drop, Accept, Stolen, Queue, Repeat, Stop +// These are terminal verdicts that are returned to the kernel. +// 2. Nftable (Internal) Verdicts:, Continue, Break, Jump, Goto, Return +// These are internal verdicts that only exist within the nftables library. +// Both share the same numeric space (uint32 Verdict Code). +// + +// NFVerdict represents the result of evaluating a packet against a rule or chain. +type NFVerdict struct { + // Code is the numeric code that represents the verdict issued. + Code uint32 + + // ChainName is the name of the chain to continue evaluation if the verdict is + // Jump or Goto. + // Note: the chain must be in the same table as the current chain. + ChainName string +} diff --git a/pkg/tcpip/stack/nic.go b/pkg/tcpip/stack/nic.go new file mode 100644 index 0000000..63e3c51 --- /dev/null +++ b/pkg/tcpip/stack/nic.go @@ -0,0 +1,1124 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "reflect" + "sort" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +// +stateify savable +type linkResolver struct { + resolver LinkAddressResolver + + neigh neighborCache +} + +var ( + _ NetworkInterface = (*nic)(nil) + _ NetworkDispatcher = (*nic)(nil) +) + +// nic represents a "network interface card" to which the networking stack is +// attached. +// +// +stateify savable +type nic struct { + NetworkLinkEndpoint + + stack *Stack + id tcpip.NICID + name string + context NICContext + + stats sharedStats + + // enableDisableMu is used to synchronize attempts to enable/disable the NIC. + // Without this mutex, calls to enable/disable the NIC may interleave and + // leave the NIC in an inconsistent state. + enableDisableMu nicRWMutex `state:"nosave"` + + // The network endpoints themselves may be modified by calling the interface's + // methods, but the map reference and entries must be constant. + networkEndpoints map[tcpip.NetworkProtocolNumber]NetworkEndpoint + linkAddrResolvers map[tcpip.NetworkProtocolNumber]*linkResolver + duplicateAddressDetectors map[tcpip.NetworkProtocolNumber]DuplicateAddressDetector + + // enabled indicates whether the NIC is enabled. + enabled atomicbitops.Bool + + // spoofing indicates whether the NIC is spoofing. + spoofing atomicbitops.Bool + + // promiscuous indicates whether the NIC is promiscuous. + promiscuous atomicbitops.Bool + + // linkResQueue holds packets that are waiting for link resolution to + // complete. + linkResQueue packetsPendingLinkResolution + + // packetEPsMu protects packetEPs. + packetEPsMu packetEPsRWMutex `state:"nosave"` + + // eps is protected by the mutex, but the values contained in it are not. + // + // +checklocks:packetEPsMu + packetEPs map[tcpip.NetworkProtocolNumber]*packetEndpointList + + qDisc QueueingDiscipline + + // deliverLinkPackets specifies whether this NIC delivers packets to + // packet sockets. It is immutable. + // + // deliverLinkPackets is off by default because some users already + // deliver link packets by explicitly calling nic.DeliverLinkPackets. + deliverLinkPackets bool + + // Primary is the main controlling interface in a bonded setup. + Primary *nic + + // experimentIPOptionEnabled indicates whether the NIC supports the + // experiment IP option. + experimentIPOptionEnabled bool +} + +// makeNICStats initializes the NIC statistics and associates them to the global +// NIC statistics. +func makeNICStats(global tcpip.NICStats) sharedStats { + var stats sharedStats + tcpip.InitStatCounters(reflect.ValueOf(&stats.local).Elem()) + stats.init(&stats.local, &global) + return stats +} + +// +stateify savable +type packetEndpointList struct { + mu packetEndpointListRWMutex `state:"nosave"` + + // eps is protected by mu, but the contained PacketEndpoint values are not. + // + // +checklocks:mu + eps []PacketEndpoint +} + +func (p *packetEndpointList) add(ep PacketEndpoint) { + p.mu.Lock() + defer p.mu.Unlock() + p.eps = append(p.eps, ep) +} + +func (p *packetEndpointList) remove(ep PacketEndpoint) { + p.mu.Lock() + defer p.mu.Unlock() + for i, epOther := range p.eps { + if epOther == ep { + p.eps = append(p.eps[:i], p.eps[i+1:]...) + break + } + } +} + +func (p *packetEndpointList) len() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.eps) +} + +// forEach calls fn with each endpoints in p while holding the read lock on p. +func (p *packetEndpointList) forEach(fn func(PacketEndpoint)) { + p.mu.RLock() + defer p.mu.RUnlock() + for _, ep := range p.eps { + fn(ep) + } +} + +var _ QueueingDiscipline = (*delegatingQueueingDiscipline)(nil) + +// +stateify savable +type delegatingQueueingDiscipline struct { + LinkWriter +} + +func (*delegatingQueueingDiscipline) Close() {} + +// WritePacket passes the packet through to the underlying LinkWriter's WritePackets. +func (qDisc *delegatingQueueingDiscipline) WritePacket(pkt *PacketBuffer) tcpip.Error { + var pkts PacketBufferList + pkts.PushBack(pkt) + _, err := qDisc.LinkWriter.WritePackets(pkts) + return err +} + +// newNIC returns a new NIC using the default NDP configurations from stack. +func newNIC(stack *Stack, id tcpip.NICID, ep LinkEndpoint, opts NICOptions) *nic { + // TODO(b/141011931): Validate a LinkEndpoint (ep) is valid. For + // example, make sure that the link address it provides is a valid + // unicast ethernet address. + + // If no queueing discipline was specified provide a stub implementation that + // just delegates to the lower link endpoint. + qDisc := opts.QDisc + if qDisc == nil { + qDisc = &delegatingQueueingDiscipline{LinkWriter: ep} + } + + // TODO(b/143357959): RFC 8200 section 5 requires that IPv6 endpoints + // observe an MTU of at least 1280 bytes. Ensure that this requirement + // of IPv6 is supported on this endpoint's LinkEndpoint. + nic := &nic{ + NetworkLinkEndpoint: ep, + stack: stack, + id: id, + name: opts.Name, + context: opts.Context, + stats: makeNICStats(stack.Stats().NICs), + networkEndpoints: make(map[tcpip.NetworkProtocolNumber]NetworkEndpoint), + linkAddrResolvers: make(map[tcpip.NetworkProtocolNumber]*linkResolver), + duplicateAddressDetectors: make(map[tcpip.NetworkProtocolNumber]DuplicateAddressDetector), + packetEPs: make(map[tcpip.NetworkProtocolNumber]*packetEndpointList), + qDisc: qDisc, + deliverLinkPackets: opts.DeliverLinkPackets, + experimentIPOptionEnabled: opts.EnableExperimentIPOption, + } + nic.linkResQueue.init(nic) + + resolutionRequired := ep.Capabilities()&CapabilityResolutionRequired != 0 + + for _, netProto := range stack.networkProtocols { + netNum := netProto.Number() + netEP := netProto.NewEndpoint(nic, nic) + nic.networkEndpoints[netNum] = netEP + + if resolutionRequired { + if r, ok := netEP.(LinkAddressResolver); ok { + l := &linkResolver{resolver: r} + l.neigh.init(nic, r) + nic.linkAddrResolvers[r.LinkAddressProtocol()] = l + } + } + + if d, ok := netEP.(DuplicateAddressDetector); ok { + nic.duplicateAddressDetectors[d.DuplicateAddressProtocol()] = d + } + } + + nic.NetworkLinkEndpoint.Attach(nic) + + return nic +} + +func (n *nic) getNetworkEndpoint(proto tcpip.NetworkProtocolNumber) NetworkEndpoint { + return n.networkEndpoints[proto] +} + +// Enabled implements NetworkInterface. +func (n *nic) Enabled() bool { + return n.enabled.Load() +} + +// setEnabled sets the enabled status for the NIC. +// +// Returns true if the enabled status was updated. +// +// +checklocks:n.enableDisableMu +func (n *nic) setEnabled(v bool) bool { + return n.enabled.Swap(v) != v +} + +// disable disables n. +// +// It undoes the work done by enable. +func (n *nic) disable() { + n.enableDisableMu.Lock() + defer n.enableDisableMu.Unlock() + n.disableLocked() +} + +// disableLocked disables n. +// +// It undoes the work done by enable. +// +// +checklocks:n.enableDisableMu +func (n *nic) disableLocked() { + if !n.Enabled() { + return + } + + // TODO(gvisor.dev/issue/1491): Should Routes that are currently bound to n be + // invalidated? Currently, Routes will continue to work when a NIC is enabled + // again, and applications may not know that the underlying NIC was ever + // disabled. + + for _, ep := range n.networkEndpoints { + ep.Disable() + + // Clear the neighbour table (including static entries) as we cannot + // guarantee that the current neighbour table will be valid when the NIC is + // enabled again. + // + // This matches linux's behaviour at the time of writing: + // https://github.com/torvalds/linux/blob/71c061d2443814de15e177489d5cc00a4a253ef3/net/core/neighbour.c#L371 + netProto := ep.NetworkProtocolNumber() + switch err := n.clearNeighbors(netProto); err.(type) { + case nil, *tcpip.ErrNotSupported: + default: + panic(fmt.Sprintf("n.clearNeighbors(%d): %s", netProto, err)) + } + } + + if !n.setEnabled(false) { + panic("should have only done work to disable the NIC if it was enabled") + } +} + +// enable enables n. +// +// If the stack has IPv6 enabled, enable will join the IPv6 All-Nodes Multicast +// address (ff02::1), start DAD for permanent addresses, and start soliciting +// routers if the stack is not operating as a router. If the stack is also +// configured to auto-generate a link-local address, one will be generated. +func (n *nic) enable() tcpip.Error { + n.enableDisableMu.Lock() + defer n.enableDisableMu.Unlock() + + if !n.setEnabled(true) { + return nil + } + + for _, ep := range n.networkEndpoints { + if err := ep.Enable(); err != nil { + return err + } + } + + return nil +} + +// remove detaches NIC from the link endpoint and releases network endpoint +// resources. This guarantees no packets between this NIC and the network +// stack. +// +// It returns an action that has to be excuted after releasing the Stack lock +// and any error encountered. +func (n *nic) remove(closeLinkEndpoint bool) (func(), tcpip.Error) { + n.enableDisableMu.Lock() + + n.disableLocked() + + for _, ep := range n.networkEndpoints { + ep.Close() + } + + n.enableDisableMu.Unlock() + + // Drain and drop any packets pending link resolution. + // We must not hold n.enableDisableMu here. + n.linkResQueue.cancel() + + var deferAct func() + // Prevent packets from going down to the link before shutting the link down. + n.qDisc.Close() + n.NetworkLinkEndpoint.Attach(nil) + if closeLinkEndpoint { + ep := n.NetworkLinkEndpoint + ep.SetOnCloseAction(nil) + // The link endpoint has to be closed without holding a + // netstack lock, because it can trigger other netstack + // operations. + deferAct = ep.Close + } + + return deferAct, nil +} + +// setPromiscuousMode enables or disables promiscuous mode. +func (n *nic) setPromiscuousMode(enable bool) { + n.promiscuous.Store(enable) +} + +// Promiscuous implements NetworkInterface. +func (n *nic) Promiscuous() bool { + return n.promiscuous.Load() +} + +// IsLoopback implements NetworkInterface. +func (n *nic) IsLoopback() bool { + return n.NetworkLinkEndpoint.Capabilities()&CapabilityLoopback != 0 +} + +// WritePacket implements NetworkEndpoint. +func (n *nic) WritePacket(r *Route, pkt *PacketBuffer) tcpip.Error { + routeInfo, _, err := r.resolvedFields(nil) + switch err.(type) { + case nil: + pkt.EgressRoute = routeInfo + return n.writePacket(pkt) + case *tcpip.ErrWouldBlock: + // As per relevant RFCs, we should queue packets while we wait for link + // resolution to complete. + // + // RFC 1122 section 2.3.2.2 (for IPv4): + // The link layer SHOULD save (rather than discard) at least + // one (the latest) packet of each set of packets destined to + // the same unresolved IP address, and transmit the saved + // packet when the address has been resolved. + // + // RFC 4861 section 7.2.2 (for IPv6): + // While waiting for address resolution to complete, the sender MUST, for + // each neighbor, retain a small queue of packets waiting for address + // resolution to complete. The queue MUST hold at least one packet, and + // MAY contain more. However, the number of queued packets per neighbor + // SHOULD be limited to some small value. When a queue overflows, the new + // arrival SHOULD replace the oldest entry. Once address resolution + // completes, the node transmits any queued packets. + return n.linkResQueue.enqueue(r, pkt) + default: + return err + } +} + +// WritePacketToRemote implements NetworkInterface. +func (n *nic) WritePacketToRemote(remoteLinkAddr tcpip.LinkAddress, pkt *PacketBuffer) tcpip.Error { + pkt.EgressRoute = RouteInfo{ + routeInfo: routeInfo{ + NetProto: pkt.NetworkProtocolNumber, + LocalLinkAddress: n.LinkAddress(), + }, + RemoteLinkAddress: remoteLinkAddr, + } + return n.writePacket(pkt) +} + +func (n *nic) writePacket(pkt *PacketBuffer) tcpip.Error { + n.NetworkLinkEndpoint.AddHeader(pkt) + return n.writeRawPacket(pkt) +} + +func (n *nic) writeRawPacketWithLinkHeaderInPayload(pkt *PacketBuffer) tcpip.Error { + if !n.NetworkLinkEndpoint.ParseHeader(pkt) { + return &tcpip.ErrMalformedHeader{} + } + return n.writeRawPacket(pkt) +} + +func (n *nic) writeRawPacket(pkt *PacketBuffer) tcpip.Error { + // Always an outgoing packet. + pkt.PktType = tcpip.PacketOutgoing + + if n.deliverLinkPackets { + n.DeliverLinkPacket(pkt.NetworkProtocolNumber, pkt) + } + + if err := n.qDisc.WritePacket(pkt); err != nil { + if _, ok := err.(*tcpip.ErrNoBufferSpace); ok { + n.stats.txPacketsDroppedNoBufferSpace.Increment() + } + return err + } + + n.stats.tx.packets.Increment() + n.stats.tx.bytes.IncrementBy(uint64(pkt.Size())) + return nil +} + +// setSpoofing enables or disables address spoofing. +func (n *nic) setSpoofing(enable bool) { + n.spoofing.Store(enable) +} + +// Spoofing implements NetworkInterface. +func (n *nic) Spoofing() bool { + return n.spoofing.Load() +} + +// primaryAddress returns an address that can be used to communicate with +// remoteAddr. +func (n *nic) primaryEndpoint(protocol tcpip.NetworkProtocolNumber, remoteAddr, srcHint tcpip.Address) AssignableAddressEndpoint { + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return nil + } + + addressableEndpoint, ok := ep.(AddressableEndpoint) + if !ok { + return nil + } + + return addressableEndpoint.AcquireOutgoingPrimaryAddress(remoteAddr, srcHint, n.Spoofing()) +} + +type getAddressBehaviour int + +const ( + // spoofing indicates that the NIC's spoofing flag should be observed when + // getting a NIC's address endpoint. + spoofing getAddressBehaviour = iota + + // promiscuous indicates that the NIC's promiscuous flag should be observed + // when getting a NIC's address endpoint. + promiscuous +) + +func (n *nic) getAddress(protocol tcpip.NetworkProtocolNumber, dst tcpip.Address) AssignableAddressEndpoint { + return n.getAddressOrCreateTemp(protocol, dst, CanBePrimaryEndpoint, promiscuous) +} + +func (n *nic) hasAddress(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) bool { + ep := n.getAddressOrCreateTempInner(protocol, addr, false, NeverPrimaryEndpoint) + if ep != nil { + ep.DecRef() + return true + } + + return false +} + +// findEndpoint finds the endpoint, if any, with the given address. +func (n *nic) findEndpoint(protocol tcpip.NetworkProtocolNumber, address tcpip.Address, peb PrimaryEndpointBehavior) AssignableAddressEndpoint { + return n.getAddressOrCreateTemp(protocol, address, peb, spoofing) +} + +// getAddressEpOrCreateTemp returns the address endpoint for the given protocol +// and address. +// +// If none exists a temporary one may be created if we are in promiscuous mode +// or spoofing. Promiscuous mode will only be checked if promiscuous is true. +// Similarly, spoofing will only be checked if spoofing is true. +// +// If the address is the IPv4 broadcast address for an endpoint's network, that +// endpoint will be returned. +func (n *nic) getAddressOrCreateTemp(protocol tcpip.NetworkProtocolNumber, address tcpip.Address, peb PrimaryEndpointBehavior, tempRef getAddressBehaviour) AssignableAddressEndpoint { + var spoofingOrPromiscuous bool + switch tempRef { + case spoofing: + spoofingOrPromiscuous = n.Spoofing() + case promiscuous: + spoofingOrPromiscuous = n.Promiscuous() + } + return n.getAddressOrCreateTempInner(protocol, address, spoofingOrPromiscuous, peb) +} + +// getAddressOrCreateTempInner is like getAddressEpOrCreateTemp except a boolean +// is passed to indicate whether or not we should generate temporary endpoints. +func (n *nic) getAddressOrCreateTempInner(protocol tcpip.NetworkProtocolNumber, address tcpip.Address, createTemp bool, peb PrimaryEndpointBehavior) AssignableAddressEndpoint { + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return nil + } + + addressableEndpoint, ok := ep.(AddressableEndpoint) + if !ok { + return nil + } + + return addressableEndpoint.AcquireAssignedAddress(address, createTemp, peb, false) +} + +// addAddress adds a new address to n, so that it starts accepting packets +// targeted at the given address (and network protocol). +func (n *nic) addAddress(protocolAddress tcpip.ProtocolAddress, properties AddressProperties) tcpip.Error { + ep := n.getNetworkEndpoint(protocolAddress.Protocol) + if ep == nil { + return &tcpip.ErrUnknownProtocol{} + } + + addressableEndpoint, ok := ep.(AddressableEndpoint) + if !ok { + return &tcpip.ErrNotSupported{} + } + + addressEndpoint, err := addressableEndpoint.AddAndAcquirePermanentAddress(protocolAddress.AddressWithPrefix, properties) + if err == nil { + // We have no need for the address endpoint. + addressEndpoint.DecRef() + } + return err +} + +// allPermanentAddresses returns all permanent addresses associated with +// this NIC. +func (n *nic) allPermanentAddresses() []tcpip.ProtocolAddress { + var addrs []tcpip.ProtocolAddress + for p, ep := range n.networkEndpoints { + addressableEndpoint, ok := ep.(AddressableEndpoint) + if !ok { + continue + } + + for _, a := range addressableEndpoint.PermanentAddresses() { + addrs = append(addrs, tcpip.ProtocolAddress{Protocol: p, AddressWithPrefix: a}) + } + } + return addrs +} + +// primaryAddresses returns the primary addresses associated with this NIC. +func (n *nic) primaryAddresses() []tcpip.ProtocolAddress { + var addrs []tcpip.ProtocolAddress + + protocolNumbers := make([]tcpip.NetworkProtocolNumber, 0, len(n.networkEndpoints)) + for p := range n.networkEndpoints { + protocolNumbers = append(protocolNumbers, p) + } + // Sort the network protocol numbers so that IPv4 address is always + // added to the list before IPv6 address. + sort.Slice(protocolNumbers, func(i, j int) bool { + return protocolNumbers[i] < protocolNumbers[j] + }) + + for _, p := range protocolNumbers { + addressableEndpoint, ok := n.networkEndpoints[p].(AddressableEndpoint) + if !ok { + continue + } + for _, a := range addressableEndpoint.PrimaryAddresses() { + addrs = append(addrs, tcpip.ProtocolAddress{Protocol: p, AddressWithPrefix: a}) + } + } + return addrs +} + +// PrimaryAddress implements NetworkInterface. +func (n *nic) PrimaryAddress(proto tcpip.NetworkProtocolNumber) (tcpip.AddressWithPrefix, tcpip.Error) { + ep := n.getNetworkEndpoint(proto) + if ep == nil { + return tcpip.AddressWithPrefix{}, &tcpip.ErrUnknownProtocol{} + } + + addressableEndpoint, ok := ep.(AddressableEndpoint) + if !ok { + return tcpip.AddressWithPrefix{}, &tcpip.ErrNotSupported{} + } + + return addressableEndpoint.MainAddress(), nil +} + +// removeAddress removes an address from n. +func (n *nic) removeAddress(addr tcpip.Address) tcpip.Error { + for _, ep := range n.networkEndpoints { + addressableEndpoint, ok := ep.(AddressableEndpoint) + if !ok { + continue + } + + switch err := addressableEndpoint.RemovePermanentAddress(addr); err.(type) { + case *tcpip.ErrBadLocalAddress: + continue + default: + return err + } + } + + return &tcpip.ErrBadLocalAddress{} +} + +func (n *nic) setAddressLifetimes(addr tcpip.Address, lifetimes AddressLifetimes) tcpip.Error { + for _, ep := range n.networkEndpoints { + ep, ok := ep.(AddressableEndpoint) + if !ok { + continue + } + + switch err := ep.SetLifetimes(addr, lifetimes); err.(type) { + case *tcpip.ErrBadLocalAddress: + continue + default: + return err + } + } + + return &tcpip.ErrBadLocalAddress{} +} + +func (n *nic) getLinkAddress(addr, localAddr tcpip.Address, protocol tcpip.NetworkProtocolNumber, onResolve func(LinkResolutionResult)) tcpip.Error { + linkRes, ok := n.linkAddrResolvers[protocol] + if !ok { + return &tcpip.ErrNotSupported{} + } + + if linkAddr, ok := linkRes.resolver.ResolveStaticAddress(addr); ok { + onResolve(LinkResolutionResult{LinkAddress: linkAddr, Err: nil}) + return nil + } + + _, _, err := linkRes.neigh.entry(addr, localAddr, onResolve) + return err +} + +func (n *nic) neighbors(protocol tcpip.NetworkProtocolNumber) ([]NeighborEntry, tcpip.Error) { + if linkRes, ok := n.linkAddrResolvers[protocol]; ok { + return linkRes.neigh.entries(), nil + } + + return nil, &tcpip.ErrNotSupported{} +} + +func (n *nic) addStaticNeighbor(addr tcpip.Address, protocol tcpip.NetworkProtocolNumber, linkAddress tcpip.LinkAddress) tcpip.Error { + if linkRes, ok := n.linkAddrResolvers[protocol]; ok { + linkRes.neigh.addStaticEntry(addr, linkAddress) + return nil + } + + return &tcpip.ErrNotSupported{} +} + +func (n *nic) removeNeighbor(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) tcpip.Error { + if linkRes, ok := n.linkAddrResolvers[protocol]; ok { + if !linkRes.neigh.removeEntry(addr) { + return &tcpip.ErrBadAddress{} + } + return nil + } + + return &tcpip.ErrNotSupported{} +} + +func (n *nic) clearNeighbors(protocol tcpip.NetworkProtocolNumber) tcpip.Error { + if linkRes, ok := n.linkAddrResolvers[protocol]; ok { + linkRes.neigh.clear() + return nil + } + + return &tcpip.ErrNotSupported{} +} + +// joinGroup adds a new endpoint for the given multicast address, if none +// exists yet. Otherwise it just increments its count. +func (n *nic) joinGroup(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) tcpip.Error { + // TODO(b/143102137): When implementing MLD, make sure MLD packets are + // not sent unless a valid link-local address is available for use on n + // as an MLD packet's source address must be a link-local address as + // outlined in RFC 3810 section 5. + + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return &tcpip.ErrNotSupported{} + } + + gep, ok := ep.(GroupAddressableEndpoint) + if !ok { + return &tcpip.ErrNotSupported{} + } + + return gep.JoinGroup(addr) +} + +// leaveGroup decrements the count for the given multicast address, and when it +// reaches zero removes the endpoint for this address. +func (n *nic) leaveGroup(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) tcpip.Error { + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return &tcpip.ErrNotSupported{} + } + + gep, ok := ep.(GroupAddressableEndpoint) + if !ok { + return &tcpip.ErrNotSupported{} + } + + return gep.LeaveGroup(addr) +} + +// isInGroup returns true if n has joined the multicast group addr. +func (n *nic) isInGroup(addr tcpip.Address) bool { + for _, ep := range n.networkEndpoints { + gep, ok := ep.(GroupAddressableEndpoint) + if !ok { + continue + } + + if gep.IsInGroup(addr) { + return true + } + } + + return false +} + +// DeliverNetworkPacket finds the appropriate network protocol endpoint and +// hands the packet over for further processing. This function is called when +// the NIC receives a packet from the link endpoint. +func (n *nic) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) { + enabled := n.Enabled() + // If the NIC is not yet enabled, don't receive any packets. + if !enabled { + n.stats.disabledRx.packets.Increment() + n.stats.disabledRx.bytes.IncrementBy(uint64(pkt.Data().Size())) + return + } + + n.stats.rx.packets.Increment() + n.stats.rx.bytes.IncrementBy(uint64(pkt.Data().Size())) + + networkEndpoint := n.getNetworkEndpoint(protocol) + if networkEndpoint == nil { + n.stats.unknownL3ProtocolRcvdPacketCounts.Increment(uint64(protocol)) + return + } + + pkt.RXChecksumValidated = n.NetworkLinkEndpoint.Capabilities()&CapabilityRXChecksumOffload != 0 + + if n.deliverLinkPackets { + n.DeliverLinkPacket(protocol, pkt) + } + + networkEndpoint.HandlePacket(pkt) +} + +func (n *nic) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) { + // Deliver to interested packet endpoints without holding NIC lock. + var packetEPPkt *PacketBuffer + defer func() { + if packetEPPkt != nil { + packetEPPkt.DecRef() + } + }() + deliverPacketEPs := func(ep PacketEndpoint) { + if packetEPPkt == nil { + // Packet endpoints hold the full packet. + // + // We perform a deep copy because higher-level endpoints may point to + // the middle of a view that is held by a packet endpoint. Save/Restore + // does not support overlapping slices and will panic in this case. + // + // TODO(https://gvisor.dev/issue/6517): Avoid this copy once S/R supports + // overlapping slices (e.g. by passing a shallow copy of pkt to the packet + // endpoint). + packetEPPkt = NewPacketBuffer(PacketBufferOptions{ + Payload: BufferSince(pkt.LinkHeader()), + }) + // If a link header was populated in the original packet buffer, then + // populate it in the packet buffer we provide to packet endpoints as + // packet endpoints inspect link headers. + packetEPPkt.LinkHeader().Consume(len(pkt.LinkHeader().Slice())) + packetEPPkt.PktType = pkt.PktType + // Assume the packet is for us if the packet type is unset. + // The packet type is set to PacketOutgoing when sending packets so + // this may only be unset for incoming packets where link endpoints + // have not set it. + if packetEPPkt.PktType == 0 { + packetEPPkt.PktType = tcpip.PacketHost + } + } + + clone := packetEPPkt.Clone() + defer clone.DecRef() + ep.HandlePacket(n.id, protocol, clone) + } + + n.packetEPsMu.Lock() + // Are any packet type sockets listening for this network protocol? + protoEPs, protoEPsOK := n.packetEPs[protocol] + // Other packet type sockets that are listening for all protocols. + anyEPs, anyEPsOK := n.packetEPs[header.EthernetProtocolAll] + n.packetEPsMu.Unlock() + + // On Linux, only ETH_P_ALL endpoints get outbound packets. + if pkt.PktType != tcpip.PacketOutgoing && protoEPsOK { + protoEPs.forEach(deliverPacketEPs) + } + if anyEPsOK { + anyEPs.forEach(deliverPacketEPs) + } +} + +// DeliverTransportPacket delivers the packets to the appropriate transport +// protocol endpoint. +func (n *nic) DeliverTransportPacket(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) TransportPacketDisposition { + state, ok := n.stack.transportProtocols[protocol] + if !ok { + n.stats.unknownL4ProtocolRcvdPacketCounts.Increment(uint64(protocol)) + return TransportPacketProtocolUnreachable + } + + transProto := state.proto + + if len(pkt.TransportHeader().Slice()) == 0 { + n.stats.malformedL4RcvdPackets.Increment() + return TransportPacketHandled + } + + srcPort, dstPort, err := transProto.ParsePorts(pkt.TransportHeader().Slice()) + if err != nil { + n.stats.malformedL4RcvdPackets.Increment() + return TransportPacketHandled + } + + netProto, ok := n.stack.networkProtocols[pkt.NetworkProtocolNumber] + if !ok { + panic(fmt.Sprintf("expected network protocol = %d, have = %#v", pkt.NetworkProtocolNumber, n.stack.networkProtocolNumbers())) + } + + src, dst := netProto.ParseAddresses(pkt.NetworkHeader().Slice()) + id := TransportEndpointID{ + LocalPort: dstPort, + LocalAddress: dst, + RemotePort: srcPort, + RemoteAddress: src, + } + if n.stack.demux.deliverPacket(protocol, pkt, id) { + return TransportPacketHandled + } + + // Try to deliver to per-stack default handler. + if state.defaultHandler != nil { + if state.defaultHandler(id, pkt) { + return TransportPacketHandled + } + } + + // We could not find an appropriate destination for this packet so + // give the protocol specific error handler a chance to handle it. + // If it doesn't handle it then we should do so. + switch res := transProto.HandleUnknownDestinationPacket(id, pkt); res { + case UnknownDestinationPacketMalformed: + n.stats.malformedL4RcvdPackets.Increment() + return TransportPacketHandled + case UnknownDestinationPacketUnhandled: + return TransportPacketDestinationPortUnreachable + case UnknownDestinationPacketHandled: + return TransportPacketHandled + default: + panic(fmt.Sprintf("unrecognized result from HandleUnknownDestinationPacket = %d", res)) + } +} + +// DeliverTransportError implements TransportDispatcher. +func (n *nic) DeliverTransportError(local, remote tcpip.Address, net tcpip.NetworkProtocolNumber, trans tcpip.TransportProtocolNumber, transErr TransportError, pkt *PacketBuffer) { + state, ok := n.stack.transportProtocols[trans] + if !ok { + return + } + + transProto := state.proto + + // ICMPv4 only guarantees that 8 bytes of the transport protocol will + // be present in the payload. We know that the ports are within the + // first 8 bytes for all known transport protocols. + transHeader, ok := pkt.Data().PullUp(8) + if !ok { + return + } + + srcPort, dstPort, err := transProto.ParsePorts(transHeader) + if err != nil { + return + } + + id := TransportEndpointID{srcPort, local, dstPort, remote} + if n.stack.demux.deliverError(n, net, trans, transErr, pkt, id) { + return + } +} + +// DeliverRawPacket implements TransportDispatcher. +func (n *nic) DeliverRawPacket(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) { + // For ICMPv4 only we validate the header length for compatibility with + // raw(7) ICMP_FILTER. The same check is made in Linux here: + // https://github.com/torvalds/linux/blob/70585216/net/ipv4/raw.c#L189. + if protocol == header.ICMPv4ProtocolNumber && len(pkt.TransportHeader().Slice())+pkt.Data().Size() < header.ICMPv4MinimumSize { + return + } + n.stack.demux.deliverRawPacket(protocol, pkt) +} + +// ID implements NetworkInterface. +func (n *nic) ID() tcpip.NICID { + return n.id +} + +// Name implements NetworkInterface. +func (n *nic) Name() string { + return n.name +} + +// nudConfigs gets the NUD configurations for n. +func (n *nic) nudConfigs(protocol tcpip.NetworkProtocolNumber) (NUDConfigurations, tcpip.Error) { + if linkRes, ok := n.linkAddrResolvers[protocol]; ok { + return linkRes.neigh.config(), nil + } + + return NUDConfigurations{}, &tcpip.ErrNotSupported{} +} + +// setNUDConfigs sets the NUD configurations for n. +// +// Note, if c contains invalid NUD configuration values, it will be fixed to +// use default values for the erroneous values. +func (n *nic) setNUDConfigs(protocol tcpip.NetworkProtocolNumber, c NUDConfigurations) tcpip.Error { + if linkRes, ok := n.linkAddrResolvers[protocol]; ok { + c.resetInvalidFields() + linkRes.neigh.setConfig(c) + return nil + } + + return &tcpip.ErrNotSupported{} +} + +func (n *nic) registerPacketEndpoint(netProto tcpip.NetworkProtocolNumber, ep PacketEndpoint) { + n.packetEPsMu.Lock() + defer n.packetEPsMu.Unlock() + + eps, ok := n.packetEPs[netProto] + if !ok { + eps = new(packetEndpointList) + n.packetEPs[netProto] = eps + } + eps.add(ep) +} + +func (n *nic) unregisterPacketEndpoint(netProto tcpip.NetworkProtocolNumber, ep PacketEndpoint) { + n.packetEPsMu.Lock() + defer n.packetEPsMu.Unlock() + + eps, ok := n.packetEPs[netProto] + if !ok { + return + } + eps.remove(ep) + if eps.len() == 0 { + delete(n.packetEPs, netProto) + } +} + +// isValidForOutgoing returns true if the endpoint can be used to send out a +// packet. It requires the endpoint to not be marked expired (i.e., its address +// has been removed) unless the NIC is in spoofing mode, or temporary. +func (n *nic) isValidForOutgoing(ep AssignableAddressEndpoint) bool { + return n.Enabled() && ep.IsAssigned(n.Spoofing()) +} + +// HandleNeighborProbe implements NetworkInterface. +func (n *nic) HandleNeighborProbe(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address, linkAddr tcpip.LinkAddress) tcpip.Error { + if l, ok := n.linkAddrResolvers[protocol]; ok { + l.neigh.handleProbe(addr, linkAddr) + return nil + } + + return &tcpip.ErrNotSupported{} +} + +// HandleNeighborConfirmation implements NetworkInterface. +func (n *nic) HandleNeighborConfirmation(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address, linkAddr tcpip.LinkAddress, flags ReachabilityConfirmationFlags) tcpip.Error { + if l, ok := n.linkAddrResolvers[protocol]; ok { + l.neigh.handleConfirmation(addr, linkAddr, flags) + return nil + } + + return &tcpip.ErrNotSupported{} +} + +// CheckLocalAddress implements NetworkInterface. +func (n *nic) CheckLocalAddress(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) bool { + if n.Spoofing() { + return true + } + + if addressEndpoint := n.getAddressOrCreateTempInner(protocol, addr, false /* createTemp */, NeverPrimaryEndpoint); addressEndpoint != nil { + addressEndpoint.DecRef() + return true + } + + return false +} + +func (n *nic) checkDuplicateAddress(protocol tcpip.NetworkProtocolNumber, addr tcpip.Address, h DADCompletionHandler) (DADCheckAddressDisposition, tcpip.Error) { + d, ok := n.duplicateAddressDetectors[protocol] + if !ok { + return 0, &tcpip.ErrNotSupported{} + } + + return d.CheckDuplicateAddress(addr, h), nil +} + +func (n *nic) setForwarding(protocol tcpip.NetworkProtocolNumber, enable bool) (bool, tcpip.Error) { + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return false, &tcpip.ErrUnknownProtocol{} + } + + forwardingEP, ok := ep.(ForwardingNetworkEndpoint) + if !ok { + return false, &tcpip.ErrNotSupported{} + } + + return forwardingEP.SetForwarding(enable), nil +} + +func (n *nic) forwarding(protocol tcpip.NetworkProtocolNumber) (bool, tcpip.Error) { + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return false, &tcpip.ErrUnknownProtocol{} + } + + forwardingEP, ok := ep.(ForwardingNetworkEndpoint) + if !ok { + return false, &tcpip.ErrNotSupported{} + } + + return forwardingEP.Forwarding(), nil +} + +func (n *nic) multicastForwardingEndpoint(protocol tcpip.NetworkProtocolNumber) (MulticastForwardingNetworkEndpoint, tcpip.Error) { + ep := n.getNetworkEndpoint(protocol) + if ep == nil { + return nil, &tcpip.ErrUnknownProtocol{} + } + + forwardingEP, ok := ep.(MulticastForwardingNetworkEndpoint) + if !ok { + return nil, &tcpip.ErrNotSupported{} + } + + return forwardingEP, nil +} + +func (n *nic) setMulticastForwarding(protocol tcpip.NetworkProtocolNumber, enable bool) (bool, tcpip.Error) { + ep, err := n.multicastForwardingEndpoint(protocol) + if err != nil { + return false, err + } + + return ep.SetMulticastForwarding(enable), nil +} + +func (n *nic) multicastForwarding(protocol tcpip.NetworkProtocolNumber) (bool, tcpip.Error) { + ep, err := n.multicastForwardingEndpoint(protocol) + if err != nil { + return false, err + } + + return ep.MulticastForwarding(), nil +} + +// GetExperimentIPOptionEnabled returns whether the NIC is responsible for +// passing the experiment IP option. +func (n *nic) GetExperimentIPOptionEnabled() bool { + return n.experimentIPOptionEnabled +} + +// CoordinatorNIC represents NetworkLinkEndpoint that can join multiple network devices. +type CoordinatorNIC interface { + // AddNIC adds the specified NIC device. + AddNIC(n *nic) tcpip.Error + // DelNIC deletes the specified NIC device. + DelNIC(n *nic) tcpip.Error +} diff --git a/pkg/tcpip/stack/nic_mutex.go b/pkg/tcpip/stack/nic_mutex.go new file mode 100644 index 0000000..43d8325 --- /dev/null +++ b/pkg/tcpip/stack/nic_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type nicRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var niclockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type niclockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *nicRWMutex) Lock() { + locking.AddGLock(nicprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *nicRWMutex) NestedLock(i niclockNameIndex) { + locking.AddGLock(nicprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *nicRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(nicprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *nicRWMutex) NestedUnlock(i niclockNameIndex) { + m.mu.Unlock() + locking.DelGLock(nicprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *nicRWMutex) RLock() { + locking.AddGLock(nicprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *nicRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(nicprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *nicRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *nicRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *nicRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var nicprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func nicinitLockNames() {} + +func init() { + nicinitLockNames() + nicprefixIndex = locking.NewMutexClass(reflect.TypeOf(nicRWMutex{}), niclockNames) +} diff --git a/pkg/tcpip/stack/nic_stats.go b/pkg/tcpip/stack/nic_stats.go new file mode 100644 index 0000000..819c9ee --- /dev/null +++ b/pkg/tcpip/stack/nic_stats.go @@ -0,0 +1,84 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// +stateify savable +type sharedStats struct { + local tcpip.NICStats + multiCounterNICStats +} + +// LINT.IfChange(multiCounterNICPacketStats) + +// +stateify savable +type multiCounterNICPacketStats struct { + packets tcpip.MultiCounterStat + bytes tcpip.MultiCounterStat +} + +func (m *multiCounterNICPacketStats) init(a, b *tcpip.NICPacketStats) { + m.packets.Init(a.Packets, b.Packets) + m.bytes.Init(a.Bytes, b.Bytes) +} + +// LINT.ThenChange(../tcpip.go:NICPacketStats) + +// LINT.IfChange(multiCounterNICNeighborStats) + +// +stateify savable +type multiCounterNICNeighborStats struct { + unreachableEntryLookups tcpip.MultiCounterStat + droppedConfirmationForNoninitiatedNeighbor tcpip.MultiCounterStat + droppedInvalidLinkAddressConfirmations tcpip.MultiCounterStat +} + +func (m *multiCounterNICNeighborStats) init(a, b *tcpip.NICNeighborStats) { + m.unreachableEntryLookups.Init(a.UnreachableEntryLookups, b.UnreachableEntryLookups) + m.droppedConfirmationForNoninitiatedNeighbor.Init(a.DroppedConfirmationForNoninitiatedNeighbor, b.DroppedConfirmationForNoninitiatedNeighbor) + m.droppedInvalidLinkAddressConfirmations.Init(a.DroppedInvalidLinkAddressConfirmations, b.DroppedInvalidLinkAddressConfirmations) +} + +// LINT.ThenChange(../tcpip.go:NICNeighborStats) + +// LINT.IfChange(multiCounterNICStats) + +// +stateify savable +type multiCounterNICStats struct { + unknownL3ProtocolRcvdPacketCounts tcpip.MultiIntegralStatCounterMap + unknownL4ProtocolRcvdPacketCounts tcpip.MultiIntegralStatCounterMap + malformedL4RcvdPackets tcpip.MultiCounterStat + tx multiCounterNICPacketStats + txPacketsDroppedNoBufferSpace tcpip.MultiCounterStat + rx multiCounterNICPacketStats + disabledRx multiCounterNICPacketStats + neighbor multiCounterNICNeighborStats +} + +func (m *multiCounterNICStats) init(a, b *tcpip.NICStats) { + m.unknownL3ProtocolRcvdPacketCounts.Init(a.UnknownL3ProtocolRcvdPacketCounts, b.UnknownL3ProtocolRcvdPacketCounts) + m.unknownL4ProtocolRcvdPacketCounts.Init(a.UnknownL4ProtocolRcvdPacketCounts, b.UnknownL4ProtocolRcvdPacketCounts) + m.malformedL4RcvdPackets.Init(a.MalformedL4RcvdPackets, b.MalformedL4RcvdPackets) + m.tx.init(&a.Tx, &b.Tx) + m.txPacketsDroppedNoBufferSpace.Init(a.TxPacketsDroppedNoBufferSpace, b.TxPacketsDroppedNoBufferSpace) + m.rx.init(&a.Rx, &b.Rx) + m.disabledRx.init(&a.DisabledRx, &b.DisabledRx) + m.neighbor.init(&a.Neighbor, &b.Neighbor) +} + +// LINT.ThenChange(../tcpip.go:NICStats) diff --git a/pkg/tcpip/stack/nud.go b/pkg/tcpip/stack/nud.go new file mode 100644 index 0000000..8685968 --- /dev/null +++ b/pkg/tcpip/stack/nud.go @@ -0,0 +1,429 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "math" + "math/rand" + "sync" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // defaultBaseReachableTime is the default base duration for computing the + // random reachable time. + // + // Reachable time is the duration for which a neighbor is considered + // reachable after a positive reachability confirmation is received. It is a + // function of a uniformly distributed random value between the minimum and + // maximum random factors, multiplied by the base reachable time. Using a + // random component eliminates the possibility that Neighbor Unreachability + // Detection messages will synchronize with each other. + // + // Default taken from REACHABLE_TIME of RFC 4861 section 10. + defaultBaseReachableTime = 30 * time.Second + + // minimumBaseReachableTime is the minimum base duration for computing the + // random reachable time. + // + // Minimum = 1ms + minimumBaseReachableTime = time.Millisecond + + // defaultMinRandomFactor is the default minimum value of the random factor + // used for computing reachable time. + // + // Default taken from MIN_RANDOM_FACTOR of RFC 4861 section 10. + defaultMinRandomFactor = 0.5 + + // defaultMaxRandomFactor is the default maximum value of the random factor + // used for computing reachable time. + // + // The default value depends on the value of MinRandomFactor. + // If MinRandomFactor is less than MAX_RANDOM_FACTOR of RFC 4861 section 10, + // the value from the RFC will be used; otherwise, the default is + // MinRandomFactor multiplied by three. + defaultMaxRandomFactor = 1.5 + + // defaultRetransmitTimer is the default amount of time to wait between + // sending reachability probes. + // + // Default taken from RETRANS_TIMER of RFC 4861 section 10. + defaultRetransmitTimer = time.Second + + // minimumRetransmitTimer is the minimum amount of time to wait between + // sending reachability probes. + // + // Note, RFC 4861 does not impose a minimum Retransmit Timer, but we do here + // to make sure the messages are not sent all at once. We also come to this + // value because in the RetransmitTimer field of a Router Advertisement, a + // value of 0 means unspecified, so the smallest valid value is 1. Note, the + // unit of the RetransmitTimer field in the Router Advertisement is + // milliseconds. + minimumRetransmitTimer = time.Millisecond + + // defaultDelayFirstProbeTime is the default duration to wait for a + // non-Neighbor-Discovery related protocol to reconfirm reachability after + // entering the DELAY state. After this time, a reachability probe will be + // sent and the entry will transition to the PROBE state. + // + // Default taken from DELAY_FIRST_PROBE_TIME of RFC 4861 section 10. + defaultDelayFirstProbeTime = 5 * time.Second + + // defaultMaxMulticastProbes is the default number of reachabililty probes + // to send before concluding negative reachability and deleting the neighbor + // entry from the INCOMPLETE state. + // + // Default taken from MAX_MULTICAST_SOLICIT of RFC 4861 section 10. + defaultMaxMulticastProbes = 3 + + // defaultMaxUnicastProbes is the default number of reachability probes to + // send before concluding retransmission from within the PROBE state should + // cease and the entry SHOULD be deleted. + // + // Default taken from MAX_UNICASE_SOLICIT of RFC 4861 section 10. + defaultMaxUnicastProbes = 3 + + // defaultMaxAnycastDelayTime is the default time in which the stack SHOULD + // delay sending a response for a random time between 0 and this time, if the + // target address is an anycast address. + // + // Default taken from MAX_ANYCAST_DELAY_TIME of RFC 4861 section 10. + defaultMaxAnycastDelayTime = time.Second + + // defaultMaxReachbilityConfirmations is the default amount of unsolicited + // reachability confirmation messages a node MAY send to all-node multicast + // address when it determines its link-layer address has changed. + // + // Default taken from MAX_NEIGHBOR_ADVERTISEMENT of RFC 4861 section 10. + defaultMaxReachbilityConfirmations = 3 +) + +// NUDDispatcher is the interface integrators of netstack must implement to +// receive and handle NUD related events. +type NUDDispatcher interface { + // OnNeighborAdded will be called when a new entry is added to a NIC's (with + // ID nicID) neighbor table. + // + // This function is permitted to block indefinitely without interfering with + // the stack's operation. + // + // May be called concurrently. + OnNeighborAdded(tcpip.NICID, NeighborEntry) + + // OnNeighborChanged will be called when an entry in a NIC's (with ID nicID) + // neighbor table changes state and/or link address. + // + // This function is permitted to block indefinitely without interfering with + // the stack's operation. + // + // May be called concurrently. + OnNeighborChanged(tcpip.NICID, NeighborEntry) + + // OnNeighborRemoved will be called when an entry is removed from a NIC's + // (with ID nicID) neighbor table. + // + // This function is permitted to block indefinitely without interfering with + // the stack's operation. + // + // May be called concurrently. + OnNeighborRemoved(tcpip.NICID, NeighborEntry) +} + +// ReachabilityConfirmationFlags describes the flags used within a reachability +// confirmation (e.g. ARP reply or Neighbor Advertisement for ARP or NDP, +// respectively). +type ReachabilityConfirmationFlags struct { + // Solicited indicates that the advertisement was sent in response to a + // reachability probe. + Solicited bool + + // Override indicates that the reachability confirmation should override an + // existing neighbor cache entry and update the cached link-layer address. + // When Override is not set the confirmation will not update a cached + // link-layer address, but will update an existing neighbor cache entry for + // which no link-layer address is known. + Override bool + + // IsRouter indicates that the sender is a router. + IsRouter bool +} + +// NUDConfigurations is the NUD configurations for the netstack. This is used +// by the neighbor cache to operate the NUD state machine on each device in the +// local network. +// +// +stateify savable +type NUDConfigurations struct { + // BaseReachableTime is the base duration for computing the random reachable + // time. + // + // Reachable time is the duration for which a neighbor is considered + // reachable after a positive reachability confirmation is received. It is a + // function of uniformly distributed random value between minRandomFactor and + // maxRandomFactor multiplied by baseReachableTime. Using a random component + // eliminates the possibility that Neighbor Unreachability Detection messages + // will synchronize with each other. + // + // After this time, a neighbor entry will transition from REACHABLE to STALE + // state. + // + // Must be greater than 0. + BaseReachableTime time.Duration + + // LearnBaseReachableTime enables learning BaseReachableTime during runtime + // from the neighbor discovery protocol, if supported. + // + // TODO(gvisor.dev/issue/2240): Implement this NUD configuration option. + LearnBaseReachableTime bool + + // MinRandomFactor is the minimum value of the random factor used for + // computing reachable time. + // + // See BaseReachbleTime for more information on computing the reachable time. + // + // Must be greater than 0. + MinRandomFactor float32 + + // MaxRandomFactor is the maximum value of the random factor used for + // computing reachabile time. + // + // See BaseReachbleTime for more information on computing the reachable time. + // + // Must be great than or equal to MinRandomFactor. + MaxRandomFactor float32 + + // RetransmitTimer is the duration between retransmission of reachability + // probes in the PROBE state. + RetransmitTimer time.Duration + + // LearnRetransmitTimer enables learning RetransmitTimer during runtime from + // the neighbor discovery protocol, if supported. + // + // TODO(gvisor.dev/issue/2241): Implement this NUD configuration option. + LearnRetransmitTimer bool + + // DelayFirstProbeTime is the duration to wait for a non-Neighbor-Discovery + // related protocol to reconfirm reachability after entering the DELAY state. + // After this time, a reachability probe will be sent and the entry will + // transition to the PROBE state. + // + // Must be greater than 0. + DelayFirstProbeTime time.Duration + + // MaxMulticastProbes is the number of reachability probes to send before + // concluding negative reachability and deleting the neighbor entry from the + // INCOMPLETE state. + // + // Must be greater than 0. + MaxMulticastProbes uint32 + + // MaxUnicastProbes is the number of reachability probes to send before + // concluding retransmission from within the PROBE state should cease and + // entry SHOULD be deleted. + // + // Must be greater than 0. + MaxUnicastProbes uint32 + + // MaxAnycastDelayTime is the time in which the stack SHOULD delay sending a + // response for a random time between 0 and this time, if the target address + // is an anycast address. + // + // TODO(gvisor.dev/issue/2242): Use this option when sending solicited + // neighbor confirmations to anycast addresses and proxying neighbor + // confirmations. + MaxAnycastDelayTime time.Duration + + // MaxReachabilityConfirmations is the number of unsolicited reachability + // confirmation messages a node MAY send to all-node multicast address when + // it determines its link-layer address has changed. + // + // TODO(gvisor.dev/issue/2246): Discuss if implementation of this NUD + // configuration option is necessary. + MaxReachabilityConfirmations uint32 +} + +// DefaultNUDConfigurations returns a NUDConfigurations populated with default +// values defined by RFC 4861 section 10. +func DefaultNUDConfigurations() NUDConfigurations { + return NUDConfigurations{ + BaseReachableTime: defaultBaseReachableTime, + LearnBaseReachableTime: true, + MinRandomFactor: defaultMinRandomFactor, + MaxRandomFactor: defaultMaxRandomFactor, + RetransmitTimer: defaultRetransmitTimer, + LearnRetransmitTimer: true, + DelayFirstProbeTime: defaultDelayFirstProbeTime, + MaxMulticastProbes: defaultMaxMulticastProbes, + MaxUnicastProbes: defaultMaxUnicastProbes, + MaxAnycastDelayTime: defaultMaxAnycastDelayTime, + MaxReachabilityConfirmations: defaultMaxReachbilityConfirmations, + } +} + +// resetInvalidFields modifies an invalid NDPConfigurations with valid values. +// If invalid values are present in c, the corresponding default values will be +// used instead. This is needed to check, and conditionally fix, user-specified +// NUDConfigurations. +func (c *NUDConfigurations) resetInvalidFields() { + if c.BaseReachableTime < minimumBaseReachableTime { + c.BaseReachableTime = defaultBaseReachableTime + } + if c.MinRandomFactor <= 0 { + c.MinRandomFactor = defaultMinRandomFactor + } + if c.MaxRandomFactor < c.MinRandomFactor { + c.MaxRandomFactor = calcMaxRandomFactor(c.MinRandomFactor) + } + if c.RetransmitTimer < minimumRetransmitTimer { + c.RetransmitTimer = defaultRetransmitTimer + } + if c.DelayFirstProbeTime == 0 { + c.DelayFirstProbeTime = defaultDelayFirstProbeTime + } + if c.MaxMulticastProbes == 0 { + c.MaxMulticastProbes = defaultMaxMulticastProbes + } + if c.MaxUnicastProbes == 0 { + c.MaxUnicastProbes = defaultMaxUnicastProbes + } +} + +// calcMaxRandomFactor calculates the maximum value of the random factor used +// for computing reachable time. This function is necessary for when the +// default specified in RFC 4861 section 10 is less than the current +// MinRandomFactor. +// +// Assumes minRandomFactor is positive since validation of the minimum value +// should come before the validation of the maximum. +func calcMaxRandomFactor(minRandomFactor float32) float32 { + if minRandomFactor > defaultMaxRandomFactor { + return minRandomFactor * 3 + } + return defaultMaxRandomFactor +} + +// +stateify savable +type nudStateMu struct { + sync.RWMutex `state:"nosave"` + + config NUDConfigurations + + // reachableTime is the duration to wait for a REACHABLE entry to + // transition into STALE after inactivity. This value is calculated with + // the algorithm defined in RFC 4861 section 6.3.2. + reachableTime time.Duration + + expiration tcpip.MonotonicTime + prevBaseReachableTime time.Duration + prevMinRandomFactor float32 + prevMaxRandomFactor float32 +} + +// NUDState stores states needed for calculating reachable time. +// +// +stateify savable +type NUDState struct { + clock tcpip.Clock + // TODO(b/341946753): Restore when netstack is savable. + rng *rand.Rand `state:"nosave"` + mu nudStateMu +} + +// NewNUDState returns new NUDState using c as configuration and the specified +// random number generator for use in recomputing ReachableTime. +func NewNUDState(c NUDConfigurations, clock tcpip.Clock, rng *rand.Rand) *NUDState { + s := &NUDState{ + clock: clock, + rng: rng, + } + s.mu.config = c + return s +} + +// Config returns the NUD configuration. +func (s *NUDState) Config() NUDConfigurations { + s.mu.RLock() + defer s.mu.RUnlock() + return s.mu.config +} + +// SetConfig replaces the existing NUD configurations with c. +func (s *NUDState) SetConfig(c NUDConfigurations) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.config = c +} + +// ReachableTime returns the duration to wait for a REACHABLE entry to +// transition into STALE after inactivity. This value is recalculated for new +// values of BaseReachableTime, MinRandomFactor, and MaxRandomFactor using the +// algorithm defined in RFC 4861 section 6.3.2. +func (s *NUDState) ReachableTime() time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + + if s.clock.NowMonotonic().After(s.mu.expiration) || + s.mu.config.BaseReachableTime != s.mu.prevBaseReachableTime || + s.mu.config.MinRandomFactor != s.mu.prevMinRandomFactor || + s.mu.config.MaxRandomFactor != s.mu.prevMaxRandomFactor { + s.recomputeReachableTimeLocked() + } + return s.mu.reachableTime +} + +// recomputeReachableTimeLocked forces a recalculation of ReachableTime using +// the algorithm defined in RFC 4861 section 6.3.2. +// +// This SHOULD automatically be invoked during certain situations, as per +// RFC 4861 section 6.3.4: +// +// If the received Reachable Time value is non-zero, the host SHOULD set its +// BaseReachableTime variable to the received value. If the new value +// differs from the previous value, the host SHOULD re-compute a new random +// ReachableTime value. ReachableTime is computed as a uniformly +// distributed random value between MIN_RANDOM_FACTOR and MAX_RANDOM_FACTOR +// times the BaseReachableTime. Using a random component eliminates the +// possibility that Neighbor Unreachability Detection messages will +// synchronize with each other. +// +// In most cases, the advertised Reachable Time value will be the same in +// consecutive Router Advertisements, and a host's BaseReachableTime rarely +// changes. In such cases, an implementation SHOULD ensure that a new +// random value gets re-computed at least once every few hours. +// +// s.mu MUST be locked for writing. +func (s *NUDState) recomputeReachableTimeLocked() { + s.mu.prevBaseReachableTime = s.mu.config.BaseReachableTime + s.mu.prevMinRandomFactor = s.mu.config.MinRandomFactor + s.mu.prevMaxRandomFactor = s.mu.config.MaxRandomFactor + + randomFactor := s.mu.config.MinRandomFactor + s.rng.Float32()*(s.mu.config.MaxRandomFactor-s.mu.config.MinRandomFactor) + + // Check for overflow, given that minRandomFactor and maxRandomFactor are + // guaranteed to be positive numbers. + if math.MaxInt64/randomFactor < float32(s.mu.config.BaseReachableTime) { + s.mu.reachableTime = time.Duration(math.MaxInt64) + } else if randomFactor == 1 { + // Avoid loss of precision when a large base reachable time is used. + s.mu.reachableTime = s.mu.config.BaseReachableTime + } else { + reachableTime := int64(float32(s.mu.config.BaseReachableTime) * randomFactor) + s.mu.reachableTime = time.Duration(reachableTime) + } + + s.mu.expiration = s.clock.NowMonotonic().Add(2 * time.Hour) +} diff --git a/pkg/tcpip/stack/packet_buffer.go b/pkg/tcpip/stack/packet_buffer.go new file mode 100644 index 0000000..53d8de5 --- /dev/null +++ b/pkg/tcpip/stack/packet_buffer.go @@ -0,0 +1,770 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +type headerType int + +const ( + virtioNetHeader headerType = iota + linkHeader + networkHeader + transportHeader + numHeaderType +) + +var pkPool = sync.Pool{ + New: func() any { + return &PacketBuffer{} + }, +} + +// PacketBufferOptions specifies options for PacketBuffer creation. +type PacketBufferOptions struct { + // ReserveHeaderBytes is the number of bytes to reserve for headers. Total + // number of bytes pushed onto the headers must not exceed this value. + ReserveHeaderBytes int + + // Payload is the initial unparsed data for the new packet. If set, it will + // be owned by the new packet. + Payload buffer.Buffer + + // IsForwardedPacket identifies that the PacketBuffer being created is for a + // forwarded packet. + IsForwardedPacket bool + + // OnRelease is a function to be run when the packet buffer is no longer + // referenced (released back to the pool). + OnRelease func() +} + +// A PacketBuffer contains all the data of a network packet. +// +// As a PacketBuffer traverses up the stack, it may be necessary to pass it to +// multiple endpoints. +// +// The whole packet is expected to be a series of bytes in the following order: +// LinkHeader, NetworkHeader, TransportHeader, and Data. Any of them can be +// empty. Use of PacketBuffer in any other order is unsupported. +// +// PacketBuffer must be created with NewPacketBuffer, which sets the initial +// reference count to 1. Owners should call `DecRef()` when they are finished +// with the buffer to return it to the pool. +// +// Internal structure: A PacketBuffer holds a pointer to buffer.Buffer, which +// exposes a logically-contiguous byte storage. The underlying storage structure +// is abstracted out, and should not be a concern here for most of the time. +// +// |- reserved ->| +// |--->| consumed (incoming) +// 0 V V +// +--------+----+----+--------------------+ +// | | | | current data ... | (buf) +// +--------+----+----+--------------------+ +// ^ | +// |<---| pushed (outgoing) +// +// When a PacketBuffer is created, a `reserved` header region can be specified, +// which stack pushes headers in this region for an outgoing packet. There could +// be no such region for an incoming packet, and `reserved` is 0. The value of +// `reserved` never changes in the entire lifetime of the packet. +// +// Outgoing Packet: When a header is pushed, `pushed` gets incremented by the +// pushed length, and the current value is stored for each header. PacketBuffer +// subtracts this value from `reserved` to compute the starting offset of each +// header in `buf`. +// +// Incoming Packet: When a header is consumed (a.k.a. parsed), the current +// `consumed` value is stored for each header, and it gets incremented by the +// consumed length. PacketBuffer adds this value to `reserved` to compute the +// starting offset of each header in `buf`. +// +// +stateify savable +type PacketBuffer struct { + _ sync.NoCopy + + packetBufferRefs + + // buf is the underlying buffer for the packet. See struct level docs for + // details. + buf buffer.Buffer + reserved int + pushed int + consumed int + + // headers stores metadata about each header. + headers [numHeaderType]headerInfo + + // NetworkProtocolNumber is only valid when NetworkHeader().View().IsEmpty() + // returns false. + // TODO(gvisor.dev/issue/3574): Remove the separately passed protocol + // numbers in registration APIs that take a PacketBuffer. + NetworkProtocolNumber tcpip.NetworkProtocolNumber + + // TransportProtocol is only valid if it is non zero. + // TODO(gvisor.dev/issue/3810): This and the network protocol number should + // be moved into the headerinfo. This should resolve the validity issue. + TransportProtocolNumber tcpip.TransportProtocolNumber + + // Hash is the transport layer hash of this packet. A value of zero + // indicates no valid hash has been set. + Hash uint32 + + // Owner is implemented by task to get the uid and gid. + // Only set for locally generated packets. + Owner tcpip.PacketOwner + + // The following fields are only set by the qdisc layer when the packet + // is added to a queue. + EgressRoute RouteInfo + GSOOptions GSO + + // snatDone indicates if the packet's source has been manipulated as per + // iptables NAT table. + snatDone bool + + // dnatDone indicates if the packet's destination has been manipulated as per + // iptables NAT table. + dnatDone bool + + // PktType indicates the SockAddrLink.PacketType of the packet as defined in + // https://www.man7.org/linux/man-pages/man7/packet.7.html. + PktType tcpip.PacketType + + // NICID is the ID of the last interface the network packet was handled at. + NICID tcpip.NICID + + // RXChecksumValidated indicates that checksum verification may be + // safely skipped. + RXChecksumValidated bool + + // NetworkPacketInfo holds an incoming packet's network-layer information. + NetworkPacketInfo NetworkPacketInfo + + tuple *tuple + + // onRelease is a function to be run when the packet buffer is no longer + // referenced (released back to the pool). + onRelease func() `state:"nosave"` +} + +// NewPacketBuffer creates a new PacketBuffer with opts. +func NewPacketBuffer(opts PacketBufferOptions) *PacketBuffer { + pk := pkPool.Get().(*PacketBuffer) + pk.reset() + if opts.ReserveHeaderBytes != 0 { + v := buffer.NewViewSize(opts.ReserveHeaderBytes) + pk.buf.Append(v) + pk.reserved = opts.ReserveHeaderBytes + } + if opts.Payload.Size() > 0 { + pk.buf.Merge(&opts.Payload) + } + pk.NetworkPacketInfo.IsForwardedPacket = opts.IsForwardedPacket + pk.onRelease = opts.OnRelease + pk.InitRefs() + return pk +} + +// IncRef increments the PacketBuffer's refcount. +func (pk *PacketBuffer) IncRef() *PacketBuffer { + pk.packetBufferRefs.IncRef() + return pk +} + +// DecRef decrements the PacketBuffer's refcount. If the refcount is +// decremented to zero, the PacketBuffer is returned to the PacketBuffer +// pool. +func (pk *PacketBuffer) DecRef() { + pk.packetBufferRefs.DecRef(func() { + if pk.onRelease != nil { + pk.onRelease() + } + + pk.buf.Release() + pkPool.Put(pk) + }) +} + +func (pk *PacketBuffer) reset() { + *pk = PacketBuffer{} +} + +// ReservedHeaderBytes returns the number of bytes initially reserved for +// headers. +func (pk *PacketBuffer) ReservedHeaderBytes() int { + return pk.reserved +} + +// AvailableHeaderBytes returns the number of bytes currently available for +// headers. This is relevant to PacketHeader.Push method only. +func (pk *PacketBuffer) AvailableHeaderBytes() int { + return pk.reserved - pk.pushed +} + +// VirtioNetHeader returns the handle to virtio-layer header. +func (pk *PacketBuffer) VirtioNetHeader() PacketHeader { + return PacketHeader{ + pk: pk, + typ: virtioNetHeader, + } +} + +// LinkHeader returns the handle to link-layer header. +func (pk *PacketBuffer) LinkHeader() PacketHeader { + return PacketHeader{ + pk: pk, + typ: linkHeader, + } +} + +// NetworkHeader returns the handle to network-layer header. +func (pk *PacketBuffer) NetworkHeader() PacketHeader { + return PacketHeader{ + pk: pk, + typ: networkHeader, + } +} + +// TransportHeader returns the handle to transport-layer header. +func (pk *PacketBuffer) TransportHeader() PacketHeader { + return PacketHeader{ + pk: pk, + typ: transportHeader, + } +} + +// HeaderSize returns the total size of all headers in bytes. +func (pk *PacketBuffer) HeaderSize() int { + return pk.pushed + pk.consumed +} + +// Size returns the size of packet in bytes. +func (pk *PacketBuffer) Size() int { + return int(pk.buf.Size()) - pk.headerOffset() +} + +// MemSize returns the estimation size of the pk in memory, including backing +// buffer data. +func (pk *PacketBuffer) MemSize() int { + return int(pk.buf.Size()) + PacketBufferStructSize +} + +// Data returns the handle to data portion of pk. +func (pk *PacketBuffer) Data() PacketData { + return PacketData{pk: pk} +} + +// AsSlices returns the underlying storage of the whole packet. +// +// Note that AsSlices can allocate a lot. In hot paths it may be preferable to +// iterate over a PacketBuffer's data via AsViewList. +func (pk *PacketBuffer) AsSlices() [][]byte { + vl := pk.buf.AsViewList() + views := make([][]byte, 0, vl.Len()) + offset := pk.headerOffset() + pk.buf.SubApply(offset, int(pk.buf.Size())-offset, func(v *buffer.View) { + views = append(views, v.AsSlice()) + }) + return views +} + +// AsViewList returns the list of Views backing the PacketBuffer along with the +// header offset into them. Users may not save or modify the ViewList returned. +func (pk *PacketBuffer) AsViewList() (buffer.ViewList, int) { + return pk.buf.AsViewList(), pk.headerOffset() +} + +// ToBuffer returns a caller-owned copy of the underlying storage of the whole +// packet. +func (pk *PacketBuffer) ToBuffer() buffer.Buffer { + b := pk.buf.Clone() + b.TrimFront(int64(pk.headerOffset())) + return b +} + +// ToView returns a caller-owned copy of the underlying storage of the whole +// packet as a view. +func (pk *PacketBuffer) ToView() *buffer.View { + p := buffer.NewView(int(pk.buf.Size())) + offset := pk.headerOffset() + pk.buf.SubApply(offset, int(pk.buf.Size())-offset, func(v *buffer.View) { + p.Write(v.AsSlice()) + }) + return p +} + +func (pk *PacketBuffer) headerOffset() int { + return pk.reserved - pk.pushed +} + +func (pk *PacketBuffer) headerOffsetOf(typ headerType) int { + return pk.reserved + pk.headers[typ].offset +} + +func (pk *PacketBuffer) dataOffset() int { + return pk.reserved + pk.consumed +} + +func (pk *PacketBuffer) push(typ headerType, size int) []byte { + h := &pk.headers[typ] + if h.length > 0 { + panic(fmt.Sprintf("push(%s, %d) called after previous push", typ, size)) + } + if pk.pushed+size > pk.reserved { + panic(fmt.Sprintf("push(%s, %d) overflows; pushed=%d reserved=%d", typ, size, pk.pushed, pk.reserved)) + } + pk.pushed += size + h.offset = -pk.pushed + h.length = size + view := pk.headerView(typ) + return view.AsSlice() +} + +func (pk *PacketBuffer) consume(typ headerType, size int) (v []byte, consumed bool) { + h := &pk.headers[typ] + if h.length > 0 { + panic(fmt.Sprintf("consume must not be called twice: type %s", typ)) + } + if pk.reserved+pk.consumed+size > int(pk.buf.Size()) { + return nil, false + } + h.offset = pk.consumed + h.length = size + pk.consumed += size + view := pk.headerView(typ) + return view.AsSlice(), true +} + +func (pk *PacketBuffer) headerView(typ headerType) buffer.View { + h := &pk.headers[typ] + if h.length == 0 { + return buffer.View{} + } + v, ok := pk.buf.PullUp(pk.headerOffsetOf(typ), h.length) + if !ok { + panic("PullUp failed") + } + return v +} + +// Clone makes a semi-deep copy of pk. The underlying packet payload is +// shared. Hence, no modifications is done to underlying packet payload. +func (pk *PacketBuffer) Clone() *PacketBuffer { + newPk := pkPool.Get().(*PacketBuffer) + newPk.reset() + newPk.buf = pk.buf.Clone() + newPk.reserved = pk.reserved + newPk.pushed = pk.pushed + newPk.consumed = pk.consumed + newPk.headers = pk.headers + newPk.Hash = pk.Hash + newPk.Owner = pk.Owner + newPk.GSOOptions = pk.GSOOptions + newPk.EgressRoute = pk.EgressRoute + newPk.NetworkProtocolNumber = pk.NetworkProtocolNumber + newPk.dnatDone = pk.dnatDone + newPk.snatDone = pk.snatDone + newPk.TransportProtocolNumber = pk.TransportProtocolNumber + newPk.PktType = pk.PktType + newPk.NICID = pk.NICID + newPk.RXChecksumValidated = pk.RXChecksumValidated + newPk.NetworkPacketInfo = pk.NetworkPacketInfo + newPk.tuple = pk.tuple + newPk.InitRefs() + return newPk +} + +// ReserveHeaderBytes prepends reserved space for headers at the front +// of the underlying buf. Can only be called once per packet. +func (pk *PacketBuffer) ReserveHeaderBytes(reserved int) { + if pk.reserved != 0 { + panic(fmt.Sprintf("ReserveHeaderBytes(...) called on packet with reserved=%d, want reserved=0", pk.reserved)) + } + pk.reserved = reserved + pk.buf.Prepend(buffer.NewViewSize(reserved)) +} + +// Network returns the network header as a header.Network. +// +// Network should only be called when NetworkHeader has been set. +func (pk *PacketBuffer) Network() header.Network { + switch netProto := pk.NetworkProtocolNumber; netProto { + case header.IPv4ProtocolNumber: + return header.IPv4(pk.NetworkHeader().Slice()) + case header.IPv6ProtocolNumber: + return header.IPv6(pk.NetworkHeader().Slice()) + default: + panic(fmt.Sprintf("unknown network protocol number %d", netProto)) + } +} + +// CloneToInbound makes a semi-deep copy of the packet buffer (similar to +// Clone) to be used as an inbound packet. +// +// See PacketBuffer.Data for details about how a packet buffer holds an inbound +// packet. +func (pk *PacketBuffer) CloneToInbound() *PacketBuffer { + newPk := pkPool.Get().(*PacketBuffer) + newPk.reset() + newPk.buf = pk.buf.Clone() + newPk.InitRefs() + // Treat unfilled header portion as reserved. + newPk.reserved = pk.AvailableHeaderBytes() + newPk.tuple = pk.tuple + return newPk +} + +// DeepCopyForForwarding creates a deep copy of the packet buffer for +// forwarding. +// +// The returned packet buffer will have the network and transport headers +// set if the original packet buffer did. +func (pk *PacketBuffer) DeepCopyForForwarding(reservedHeaderBytes int) *PacketBuffer { + payload := BufferSince(pk.NetworkHeader()) + defer payload.Release() + newPk := NewPacketBuffer(PacketBufferOptions{ + ReserveHeaderBytes: reservedHeaderBytes, + Payload: payload.DeepClone(), + IsForwardedPacket: true, + }) + + { + consumeBytes := len(pk.NetworkHeader().Slice()) + if _, consumed := newPk.NetworkHeader().Consume(consumeBytes); !consumed { + panic(fmt.Sprintf("expected to consume network header %d bytes from new packet", consumeBytes)) + } + newPk.NetworkProtocolNumber = pk.NetworkProtocolNumber + } + + { + consumeBytes := len(pk.TransportHeader().Slice()) + if _, consumed := newPk.TransportHeader().Consume(consumeBytes); !consumed { + panic(fmt.Sprintf("expected to consume transport header %d bytes from new packet", consumeBytes)) + } + newPk.TransportProtocolNumber = pk.TransportProtocolNumber + } + + newPk.tuple = pk.tuple + + return newPk +} + +// headerInfo stores metadata about a header in a packet. +// +// +stateify savable +type headerInfo struct { + // offset is the offset of the header in pk.buf relative to + // pk.buf[pk.reserved]. See the PacketBuffer struct for details. + offset int + + // length is the length of this header. + length int +} + +// PacketHeader is a handle object to a header in the underlying packet. +type PacketHeader struct { + pk *PacketBuffer + typ headerType +} + +// View returns an caller-owned copy of the underlying storage of h as a +// *buffer.View. +func (h PacketHeader) View() *buffer.View { + view := h.pk.headerView(h.typ) + if view.Size() == 0 { + return nil + } + return view.Clone() +} + +// Slice returns the underlying storage of h as a []byte. The returned slice +// should not be modified if the underlying packet could be shared, cloned, or +// borrowed. +func (h PacketHeader) Slice() []byte { + view := h.pk.headerView(h.typ) + return view.AsSlice() +} + +// Push pushes size bytes in the front of its residing packet, and returns the +// backing storage. Callers may only call one of Push or Consume once on each +// header in the lifetime of the underlying packet. +func (h PacketHeader) Push(size int) []byte { + return h.pk.push(h.typ, size) +} + +// Consume moves the first size bytes of the unparsed data portion in the packet +// to h, and returns the backing storage. In the case of data is shorter than +// size, consumed will be false, and the state of h will not be affected. +// Callers may only call one of Push or Consume once on each header in the +// lifetime of the underlying packet. +func (h PacketHeader) Consume(size int) (v []byte, consumed bool) { + return h.pk.consume(h.typ, size) +} + +// PacketData represents the data portion of a PacketBuffer. +// +// +stateify savable +type PacketData struct { + pk *PacketBuffer +} + +// PullUp returns a contiguous slice of size bytes from the beginning of d. +// Callers should not keep the view for later use. Callers can write to the +// returned slice if they have singular ownership over the underlying +// Buffer. +func (d PacketData) PullUp(size int) (b []byte, ok bool) { + view, ok := d.pk.buf.PullUp(d.pk.dataOffset(), size) + return view.AsSlice(), ok +} + +// Consume is the same as PullUp except that is additionally consumes the +// returned bytes. Subsequent PullUp or Consume will not return these bytes. +func (d PacketData) Consume(size int) ([]byte, bool) { + v, ok := d.PullUp(size) + if ok { + d.pk.consumed += size + } + return v, ok +} + +// ReadTo reads bytes from d to dst. It also removes these bytes from d +// unless peek is true. +func (d PacketData) ReadTo(dst io.Writer, peek bool) (int, error) { + var ( + err error + done int + ) + offset := d.pk.dataOffset() + d.pk.buf.SubApply(offset, int(d.pk.buf.Size())-offset, func(v *buffer.View) { + if err != nil { + return + } + var n int + n, err = dst.Write(v.AsSlice()) + done += n + if err != nil { + return + } + if n != v.Size() { + panic(fmt.Sprintf("io.Writer.Write succeeded with incomplete write: %d != %d", n, v.Size())) + } + }) + if !peek { + d.pk.buf.TrimFront(int64(done)) + } + return done, err +} + +// CapLength reduces d to at most length bytes. +func (d PacketData) CapLength(length int) { + if length < 0 { + panic("length < 0") + } + d.pk.buf.Truncate(int64(length + d.pk.dataOffset())) +} + +// ToBuffer returns the underlying storage of d in a buffer.Buffer. +func (d PacketData) ToBuffer() buffer.Buffer { + buf := d.pk.buf.Clone() + offset := d.pk.dataOffset() + buf.TrimFront(int64(offset)) + return buf +} + +// AppendView appends v into d, taking the ownership of v. +func (d PacketData) AppendView(v *buffer.View) { + d.pk.buf.Append(v) +} + +// MergeBuffer merges b into d and clears b. +func (d PacketData) MergeBuffer(b *buffer.Buffer) { + d.pk.buf.Merge(b) +} + +// MergeFragment appends the data portion of frag to dst. It modifies +// frag and frag should not be used again. +func MergeFragment(dst, frag *PacketBuffer) { + frag.buf.TrimFront(int64(frag.dataOffset())) + dst.buf.Merge(&frag.buf) +} + +// ReadFrom moves at most count bytes from the beginning of src to the end +// of d and returns the number of bytes moved. +func (d PacketData) ReadFrom(src *buffer.Buffer, count int) int { + toRead := int64(count) + if toRead > src.Size() { + toRead = src.Size() + } + clone := src.Clone() + clone.Truncate(toRead) + d.pk.buf.Merge(&clone) + src.TrimFront(toRead) + return int(toRead) +} + +// ReadFromPacketData moves count bytes from the beginning of oth to the end of +// d. +func (d PacketData) ReadFromPacketData(oth PacketData, count int) { + buf := oth.ToBuffer() + buf.Truncate(int64(count)) + d.MergeBuffer(&buf) + oth.TrimFront(count) + buf.Release() +} + +// Merge clears headers in oth and merges its data with d. +func (d PacketData) Merge(oth PacketData) { + oth.pk.buf.TrimFront(int64(oth.pk.dataOffset())) + d.pk.buf.Merge(&oth.pk.buf) +} + +// TrimFront removes up to count bytes from the front of d's payload. +func (d PacketData) TrimFront(count int) { + if count > d.Size() { + count = d.Size() + } + buf := d.pk.Data().ToBuffer() + buf.TrimFront(int64(count)) + d.pk.buf.Truncate(int64(d.pk.dataOffset())) + d.pk.buf.Merge(&buf) +} + +// Size returns the number of bytes in the data payload of the packet. +func (d PacketData) Size() int { + return int(d.pk.buf.Size()) - d.pk.dataOffset() +} + +// AsRange returns a Range representing the current data payload of the packet. +func (d PacketData) AsRange() Range { + return Range{ + pk: d.pk, + offset: d.pk.dataOffset(), + length: d.Size(), + } +} + +// Checksum returns a checksum over the data payload of the packet. +func (d PacketData) Checksum() uint16 { + return d.pk.buf.Checksum(d.pk.dataOffset()) +} + +// ChecksumAtOffset returns a checksum over the data payload of the packet +// starting from offset. +func (d PacketData) ChecksumAtOffset(offset int) uint16 { + return d.pk.buf.Checksum(offset) +} + +// Range represents a contiguous subportion of a PacketBuffer. +type Range struct { + pk *PacketBuffer + offset int + length int +} + +// Size returns the number of bytes in r. +func (r Range) Size() int { + return r.length +} + +// SubRange returns a new Range starting at off bytes of r. It returns an empty +// range if off is out-of-bounds. +func (r Range) SubRange(off int) Range { + if off > r.length { + return Range{pk: r.pk} + } + return Range{ + pk: r.pk, + offset: r.offset + off, + length: r.length - off, + } +} + +// Capped returns a new Range with the same starting point of r and length +// capped at max. +func (r Range) Capped(max int) Range { + if r.length <= max { + return r + } + return Range{ + pk: r.pk, + offset: r.offset, + length: max, + } +} + +// ToSlice returns a caller-owned copy of data in r. +func (r Range) ToSlice() []byte { + if r.length == 0 { + return nil + } + all := make([]byte, 0, r.length) + r.iterate(func(v *buffer.View) { + all = append(all, v.AsSlice()...) + }) + return all +} + +// ToView returns a caller-owned copy of data in r. +func (r Range) ToView() *buffer.View { + if r.length == 0 { + return nil + } + newV := buffer.NewView(r.length) + r.iterate(func(v *buffer.View) { + newV.Write(v.AsSlice()) + }) + return newV +} + +// iterate calls fn for each piece in r. fn is always called with a non-empty +// slice. +func (r Range) iterate(fn func(*buffer.View)) { + r.pk.buf.SubApply(r.offset, r.length, fn) +} + +// PayloadSince returns a caller-owned view containing the payload starting from +// and including a particular header. +func PayloadSince(h PacketHeader) *buffer.View { + offset := h.pk.headerOffset() + for i := headerType(0); i < h.typ; i++ { + offset += h.pk.headers[i].length + } + return Range{ + pk: h.pk, + offset: offset, + length: int(h.pk.buf.Size()) - offset, + }.ToView() +} + +// BufferSince returns a caller-owned view containing the packet payload +// starting from and including a particular header. +func BufferSince(h PacketHeader) buffer.Buffer { + offset := h.pk.headerOffset() + for i := headerType(0); i < h.typ; i++ { + offset += h.pk.headers[i].length + } + clone := h.pk.buf.Clone() + clone.TrimFront(int64(offset)) + return clone +} diff --git a/pkg/tcpip/stack/packet_buffer_list.go b/pkg/tcpip/stack/packet_buffer_list.go new file mode 100644 index 0000000..363059a --- /dev/null +++ b/pkg/tcpip/stack/packet_buffer_list.go @@ -0,0 +1,87 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +// PacketBufferList is a slice-backed list. All operations are O(1) unless +// otherwise noted. +// +// Note: this is intentionally backed by a slice, not an intrusive list. We've +// switched PacketBufferList back-and-forth between intrusive list and +// slice-backed implementations, and the latter has proven to be preferable: +// +// - Intrusive lists are a refcounting nightmare, as modifying the list +// sometimes-but-not-always modifies the list for others. +// - The slice-backed implementation has been benchmarked and is slightly more +// performant. +// +// +stateify savable +type PacketBufferList struct { + pbs []*PacketBuffer +} + +// AsSlice returns a slice containing the packets in the list. +// +//go:nosplit +func (pl *PacketBufferList) AsSlice() []*PacketBuffer { + return pl.pbs +} + +// Reset decrements all elements and resets the list to the empty state. +// +//go:nosplit +func (pl *PacketBufferList) Reset() { + for i, pb := range pl.pbs { + pb.DecRef() + pl.pbs[i] = nil + } + pl.pbs = pl.pbs[:0] +} + +// Len returns the number of elements in the list. +// +//go:nosplit +func (pl *PacketBufferList) Len() int { + return len(pl.pbs) +} + +// PushBack inserts the PacketBuffer at the back of the list. +// +//go:nosplit +func (pl *PacketBufferList) PushBack(pb *PacketBuffer) { + pl.pbs = append(pl.pbs, pb) +} + +// PopFront removes the first element in the list if it exists and returns it. +// +//go:nosplit +func (pl *PacketBufferList) PopFront() *PacketBuffer { + if len(pl.pbs) == 0 { + return nil + } + pkt := pl.pbs[0] + pl.pbs = pl.pbs[1:] + return pkt +} + +// DecRef decreases the reference count on each PacketBuffer +// stored in the list. +// +// NOTE: runs in O(n) time. +// +//go:nosplit +func (pl PacketBufferList) DecRef() { + for _, pb := range pl.pbs { + pb.DecRef() + } +} diff --git a/pkg/tcpip/stack/packet_buffer_refs.go b/pkg/tcpip/stack/packet_buffer_refs.go new file mode 100644 index 0000000..5149309 --- /dev/null +++ b/pkg/tcpip/stack/packet_buffer_refs.go @@ -0,0 +1,141 @@ +package stack + +import ( + "context" + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/refs" +) + +// enableLogging indicates whether reference-related events should be logged (with +// stack traces). This is false by default and should only be set to true for +// debugging purposes, as it can generate an extremely large amount of output +// and drastically degrade performance. +const packetBufferenableLogging = false + +// obj is used to customize logging. Note that we use a pointer to T so that +// we do not copy the entire object when passed as a format parameter. +var packetBufferobj *PacketBuffer + +// Refs implements refs.RefCounter. It keeps a reference count using atomic +// operations and calls the destructor when the count reaches zero. +// +// NOTE: Do not introduce additional fields to the Refs struct. It is used by +// many filesystem objects, and we want to keep it as small as possible (i.e., +// the same size as using an int64 directly) to avoid taking up extra cache +// space. In general, this template should not be extended at the cost of +// performance. If it does not offer enough flexibility for a particular object +// (example: b/187877947), we should implement the RefCounter/CheckedObject +// interfaces manually. +// +// +stateify savable +type packetBufferRefs struct { + // refCount is composed of two fields: + // + // [32-bit speculative references]:[32-bit real references] + // + // Speculative references are used for TryIncRef, to avoid a CompareAndSwap + // loop. See IncRef, DecRef and TryIncRef for details of how these fields are + // used. + refCount atomicbitops.Int64 +} + +// InitRefs initializes r with one reference and, if enabled, activates leak +// checking. +func (r *packetBufferRefs) InitRefs() { + r.refCount.RacyStore(1) + refs.Register(r) +} + +// RefType implements refs.CheckedObject.RefType. +func (r *packetBufferRefs) RefType() string { + return fmt.Sprintf("%T", packetBufferobj)[1:] +} + +// LeakMessage implements refs.CheckedObject.LeakMessage. +func (r *packetBufferRefs) LeakMessage() string { + return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs()) +} + +// LogRefs implements refs.CheckedObject.LogRefs. +func (r *packetBufferRefs) LogRefs() bool { + return packetBufferenableLogging +} + +// ReadRefs returns the current number of references. The returned count is +// inherently racy and is unsafe to use without external synchronization. +func (r *packetBufferRefs) ReadRefs() int64 { + return r.refCount.Load() +} + +// IncRef implements refs.RefCounter.IncRef. +// +//go:nosplit +func (r *packetBufferRefs) IncRef() { + v := r.refCount.Add(1) + if packetBufferenableLogging { + refs.LogIncRef(r, v) + } + if v <= 1 { + panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType())) + } +} + +// TryIncRef implements refs.TryRefCounter.TryIncRef. +// +// To do this safely without a loop, a speculative reference is first acquired +// on the object. This allows multiple concurrent TryIncRef calls to distinguish +// other TryIncRef calls from genuine references held. +// +//go:nosplit +func (r *packetBufferRefs) TryIncRef() bool { + const speculativeRef = 1 << 32 + if v := r.refCount.Add(speculativeRef); int32(v) == 0 { + + r.refCount.Add(-speculativeRef) + return false + } + + v := r.refCount.Add(-speculativeRef + 1) + if packetBufferenableLogging { + refs.LogTryIncRef(r, v) + } + return true +} + +// DecRef implements refs.RefCounter.DecRef. +// +// Note that speculative references are counted here. Since they were added +// prior to real references reaching zero, they will successfully convert to +// real references. In other words, we see speculative references only in the +// following case: +// +// A: TryIncRef [speculative increase => sees non-negative references] +// B: DecRef [real decrease] +// A: TryIncRef [transform speculative to real] +// +//go:nosplit +func (r *packetBufferRefs) DecRef(destroy func()) { + v := r.refCount.Add(-1) + if packetBufferenableLogging { + refs.LogDecRef(r, v) + } + switch { + case v < 0: + panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType())) + + case v == 0: + refs.Unregister(r) + + if destroy != nil { + destroy() + } + } +} + +func (r *packetBufferRefs) afterLoad(context.Context) { + if r.ReadRefs() > 0 { + refs.Register(r) + } +} diff --git a/pkg/tcpip/stack/packet_buffer_unsafe.go b/pkg/tcpip/stack/packet_buffer_unsafe.go new file mode 100644 index 0000000..9d1105b --- /dev/null +++ b/pkg/tcpip/stack/packet_buffer_unsafe.go @@ -0,0 +1,28 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import "unsafe" + +// PacketBufferStructSize is the minimal size of the packet buffer overhead. +const PacketBufferStructSize = int(unsafe.Sizeof(PacketBuffer{})) + +// ID returns a unique ID for the underlying storage of the packet. +// +// Two *PacketBuffers have the same IDs if and only if they point to the same +// location in memory. +func (pk *PacketBuffer) ID() uintptr { + return uintptr(unsafe.Pointer(pk)) +} diff --git a/pkg/tcpip/stack/packet_endpoint_list_mutex.go b/pkg/tcpip/stack/packet_endpoint_list_mutex.go new file mode 100644 index 0000000..0844e79 --- /dev/null +++ b/pkg/tcpip/stack/packet_endpoint_list_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type packetEndpointListRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var packetEndpointListlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type packetEndpointListlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *packetEndpointListRWMutex) Lock() { + locking.AddGLock(packetEndpointListprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetEndpointListRWMutex) NestedLock(i packetEndpointListlockNameIndex) { + locking.AddGLock(packetEndpointListprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *packetEndpointListRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(packetEndpointListprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetEndpointListRWMutex) NestedUnlock(i packetEndpointListlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(packetEndpointListprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *packetEndpointListRWMutex) RLock() { + locking.AddGLock(packetEndpointListprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *packetEndpointListRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(packetEndpointListprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *packetEndpointListRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *packetEndpointListRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *packetEndpointListRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var packetEndpointListprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func packetEndpointListinitLockNames() {} + +func init() { + packetEndpointListinitLockNames() + packetEndpointListprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetEndpointListRWMutex{}), packetEndpointListlockNames) +} diff --git a/pkg/tcpip/stack/packet_eps_mutex.go b/pkg/tcpip/stack/packet_eps_mutex.go new file mode 100644 index 0000000..679ba48 --- /dev/null +++ b/pkg/tcpip/stack/packet_eps_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type packetEPsRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var packetEPslockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type packetEPslockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *packetEPsRWMutex) Lock() { + locking.AddGLock(packetEPsprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetEPsRWMutex) NestedLock(i packetEPslockNameIndex) { + locking.AddGLock(packetEPsprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *packetEPsRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(packetEPsprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetEPsRWMutex) NestedUnlock(i packetEPslockNameIndex) { + m.mu.Unlock() + locking.DelGLock(packetEPsprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *packetEPsRWMutex) RLock() { + locking.AddGLock(packetEPsprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *packetEPsRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(packetEPsprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *packetEPsRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *packetEPsRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *packetEPsRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var packetEPsprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func packetEPsinitLockNames() {} + +func init() { + packetEPsinitLockNames() + packetEPsprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetEPsRWMutex{}), packetEPslockNames) +} diff --git a/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go b/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go new file mode 100644 index 0000000..1a71bfb --- /dev/null +++ b/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go @@ -0,0 +1,64 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type packetsPendingLinkResolutionMutex struct { + mu sync.Mutex +} + +var packetsPendingLinkResolutionprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var packetsPendingLinkResolutionlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type packetsPendingLinkResolutionlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *packetsPendingLinkResolutionMutex) Lock() { + locking.AddGLock(packetsPendingLinkResolutionprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetsPendingLinkResolutionMutex) NestedLock(i packetsPendingLinkResolutionlockNameIndex) { + locking.AddGLock(packetsPendingLinkResolutionprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *packetsPendingLinkResolutionMutex) Unlock() { + locking.DelGLock(packetsPendingLinkResolutionprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetsPendingLinkResolutionMutex) NestedUnlock(i packetsPendingLinkResolutionlockNameIndex) { + locking.DelGLock(packetsPendingLinkResolutionprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func packetsPendingLinkResolutioninitLockNames() {} + +func init() { + packetsPendingLinkResolutioninitLockNames() + packetsPendingLinkResolutionprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetsPendingLinkResolutionMutex{}), packetsPendingLinkResolutionlockNames) +} diff --git a/pkg/tcpip/stack/pending_packets.go b/pkg/tcpip/stack/pending_packets.go new file mode 100644 index 0000000..8d61e60 --- /dev/null +++ b/pkg/tcpip/stack/pending_packets.go @@ -0,0 +1,223 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // maxPendingResolutions is the maximum number of pending link-address + // resolutions. + maxPendingResolutions = 64 + maxPendingPacketsPerResolution = 256 +) + +// +stateify savable +type pendingPacket struct { + routeInfo RouteInfo + pkt *PacketBuffer +} + +type packetsPendingLinkResolutionMu struct { + packetsPendingLinkResolutionMutex + + // The packets to send once the resolver completes. + // + // The link resolution channel is used as the key for this map. + packets map[<-chan struct{}][]pendingPacket + + // FIFO of channels used to cancel the oldest goroutine waiting for + // link-address resolution. + // + // cancelChans holds the same channels that are used as keys to packets. + cancelChans []<-chan struct{} +} + +// packetsPendingLinkResolution is a queue of packets pending link resolution. +// +// Once link resolution completes successfully, the packets will be written. +// +// +stateify savable +type packetsPendingLinkResolution struct { + nic *nic + mu packetsPendingLinkResolutionMu `state:"nosave"` +} + +func (f *packetsPendingLinkResolution) incrementOutgoingPacketErrors(pkt *PacketBuffer) { + f.nic.stack.stats.IP.OutgoingPacketErrors.Increment() + + if ipEndpointStats, ok := f.nic.getNetworkEndpoint(pkt.NetworkProtocolNumber).Stats().(IPNetworkEndpointStats); ok { + ipEndpointStats.IPStats().OutgoingPacketErrors.Increment() + } +} + +func (f *packetsPendingLinkResolution) init(nic *nic) { + f.mu.Lock() + defer f.mu.Unlock() + f.nic = nic + f.mu.packets = make(map[<-chan struct{}][]pendingPacket) +} + +// cancel drains all pending packet queues and release all packet +// references. +func (f *packetsPendingLinkResolution) cancel() { + f.mu.Lock() + defer f.mu.Unlock() + for ch, pendingPackets := range f.mu.packets { + for _, p := range pendingPackets { + p.pkt.DecRef() + } + delete(f.mu.packets, ch) + } + f.mu.cancelChans = nil +} + +// dequeue any pending packets associated with ch. +// +// If err is nil, packets will be written and sent to the given remote link +// address. +func (f *packetsPendingLinkResolution) dequeue(ch <-chan struct{}, linkAddr tcpip.LinkAddress, err tcpip.Error) { + f.mu.Lock() + packets, ok := f.mu.packets[ch] + delete(f.mu.packets, ch) + + if ok { + for i, cancelChan := range f.mu.cancelChans { + if cancelChan == ch { + f.mu.cancelChans = append(f.mu.cancelChans[:i], f.mu.cancelChans[i+1:]...) + break + } + } + } + + f.mu.Unlock() + + if ok { + f.dequeuePackets(packets, linkAddr, err) + } +} + +// enqueue a packet to be sent once link resolution completes. +// +// If the maximum number of pending resolutions is reached, the packets +// associated with the oldest link resolution will be dequeued as if they failed +// link resolution. +func (f *packetsPendingLinkResolution) enqueue(r *Route, pkt *PacketBuffer) tcpip.Error { + f.mu.Lock() + // Make sure we attempt resolution while holding f's lock so that we avoid + // a race where link resolution completes before we enqueue the packets. + // + // A @ T1: Call ResolvedFields (get link resolution channel) + // B @ T2: Complete link resolution, dequeue pending packets + // C @ T1: Enqueue packet that already completed link resolution (which will + // never dequeue) + // + // To make sure B does not interleave with A and C, we make sure A and C are + // done while holding the lock. + routeInfo, ch, err := r.resolvedFields(nil) + switch err.(type) { + case nil: + // The route resolved immediately, so we don't need to wait for link + // resolution to send the packet. + f.mu.Unlock() + pkt.EgressRoute = routeInfo + return f.nic.writePacket(pkt) + case *tcpip.ErrWouldBlock: + // We need to wait for link resolution to complete. + default: + f.mu.Unlock() + return err + } + + defer f.mu.Unlock() + + packets, ok := f.mu.packets[ch] + packets = append(packets, pendingPacket{ + routeInfo: routeInfo, + pkt: pkt.Clone(), + }) + + if len(packets) > maxPendingPacketsPerResolution { + f.incrementOutgoingPacketErrors(packets[0].pkt) + packets[0].pkt.DecRef() + packets[0] = pendingPacket{} + packets = packets[1:] + + if numPackets := len(packets); numPackets != maxPendingPacketsPerResolution { + panic(fmt.Sprintf("holding more queued packets than expected; got = %d, want <= %d", numPackets, maxPendingPacketsPerResolution)) + } + } + + f.mu.packets[ch] = packets + + if ok { + return nil + } + + cancelledPackets := f.newCancelChannelLocked(ch) + + if len(cancelledPackets) != 0 { + // Dequeue the pending packets in a new goroutine to not hold up the current + // goroutine as handing link resolution failures may be a costly operation. + go f.dequeuePackets(cancelledPackets, "" /* linkAddr */, &tcpip.ErrAborted{}) + } + + return nil +} + +// newCancelChannelLocked appends the link resolution channel to a FIFO. If the +// maximum number of pending resolutions is reached, the oldest channel will be +// removed and its associated pending packets will be returned. +func (f *packetsPendingLinkResolution) newCancelChannelLocked(newCH <-chan struct{}) []pendingPacket { + f.mu.cancelChans = append(f.mu.cancelChans, newCH) + if len(f.mu.cancelChans) <= maxPendingResolutions { + return nil + } + + ch := f.mu.cancelChans[0] + f.mu.cancelChans[0] = nil + f.mu.cancelChans = f.mu.cancelChans[1:] + if l := len(f.mu.cancelChans); l > maxPendingResolutions { + panic(fmt.Sprintf("max pending resolutions reached; got %d active resolutions, max = %d", l, maxPendingResolutions)) + } + + packets, ok := f.mu.packets[ch] + if !ok { + panic("must have a packet queue for an uncancelled channel") + } + delete(f.mu.packets, ch) + + return packets +} + +func (f *packetsPendingLinkResolution) dequeuePackets(packets []pendingPacket, linkAddr tcpip.LinkAddress, err tcpip.Error) { + for _, p := range packets { + if err == nil { + p.routeInfo.RemoteLinkAddress = linkAddr + p.pkt.EgressRoute = p.routeInfo + _ = f.nic.writePacket(p.pkt) + } else { + f.incrementOutgoingPacketErrors(p.pkt) + + if linkResolvableEP, ok := f.nic.getNetworkEndpoint(p.pkt.NetworkProtocolNumber).(LinkResolvableNetworkEndpoint); ok { + linkResolvableEP.HandleLinkResolutionFailure(p.pkt) + } + } + p.pkt.DecRef() + } +} diff --git a/pkg/tcpip/stack/rand.go b/pkg/tcpip/stack/rand.go new file mode 100644 index 0000000..65c6ab6 --- /dev/null +++ b/pkg/tcpip/stack/rand.go @@ -0,0 +1,40 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "math/rand" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// lockedRandomSource provides a threadsafe rand.Source. +type lockedRandomSource struct { + mu sync.Mutex + src rand.Source +} + +func (r *lockedRandomSource) Int63() (n int64) { + r.mu.Lock() + n = r.src.Int63() + r.mu.Unlock() + return n +} + +func (r *lockedRandomSource) Seed(seed int64) { + r.mu.Lock() + r.src.Seed(seed) + r.mu.Unlock() +} diff --git a/pkg/tcpip/stack/registration.go b/pkg/tcpip/stack/registration.go new file mode 100644 index 0000000..1c1ef64 --- /dev/null +++ b/pkg/tcpip/stack/registration.go @@ -0,0 +1,1487 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// NetworkEndpointID is the identifier of a network layer protocol endpoint. +// Currently the local address is sufficient because all supported protocols +// (i.e., IPv4 and IPv6) have different sizes for their addresses. +type NetworkEndpointID struct { + LocalAddress tcpip.Address +} + +// TransportEndpointID is the identifier of a transport layer protocol endpoint. +// +// +stateify savable +type TransportEndpointID struct { + // LocalPort is the local port associated with the endpoint. + LocalPort uint16 + + // LocalAddress is the local [network layer] address associated with + // the endpoint. + LocalAddress tcpip.Address + + // RemotePort is the remote port associated with the endpoint. + RemotePort uint16 + + // RemoteAddress it the remote [network layer] address associated with + // the endpoint. + RemoteAddress tcpip.Address +} + +// NetworkPacketInfo holds information about a network layer packet. +// +// +stateify savable +type NetworkPacketInfo struct { + // LocalAddressBroadcast is true if the packet's local address is a broadcast + // address. + LocalAddressBroadcast bool + + // LocalAddressTemporary is true if the packet's local address is a temporary + // address. + LocalAddressTemporary bool + + // IsForwardedPacket is true if the packet is being forwarded. + IsForwardedPacket bool +} + +// TransportErrorKind enumerates error types that are handled by the transport +// layer. +type TransportErrorKind int + +const ( + // PacketTooBigTransportError indicates that a packet did not reach its + // destination because a link on the path to the destination had an MTU that + // was too small to carry the packet. + PacketTooBigTransportError TransportErrorKind = iota + + // DestinationHostUnreachableTransportError indicates that the destination + // host was unreachable. + DestinationHostUnreachableTransportError + + // DestinationPortUnreachableTransportError indicates that a packet reached + // the destination host, but the transport protocol was not active on the + // destination port. + DestinationPortUnreachableTransportError + + // DestinationNetworkUnreachableTransportError indicates that the destination + // network was unreachable. + DestinationNetworkUnreachableTransportError + + // DestinationProtoUnreachableTransportError indicates that the destination + // protocol was unreachable. + DestinationProtoUnreachableTransportError + + // SourceRouteFailedTransportError indicates that the source route failed. + SourceRouteFailedTransportError + + // SourceHostIsolatedTransportError indicates that the source machine is not + // on the network. + SourceHostIsolatedTransportError + + // DestinationHostDownTransportError indicates that the destination host is + // down. + DestinationHostDownTransportError +) + +// TransportError is a marker interface for errors that may be handled by the +// transport layer. +type TransportError interface { + tcpip.SockErrorCause + + // Kind returns the type of the transport error. + Kind() TransportErrorKind +} + +// TransportEndpoint is the interface that needs to be implemented by transport +// protocol (e.g., tcp, udp) endpoints that can handle packets. +type TransportEndpoint interface { + // HandlePacket is called by the stack when new packets arrive to this + // transport endpoint. It sets the packet buffer's transport header. + // + // HandlePacket may modify the packet. + HandlePacket(TransportEndpointID, *PacketBuffer) + + // HandleError is called when the transport endpoint receives an error. + // + // HandleError takes may modify the packet buffer. + HandleError(TransportError, *PacketBuffer) + + // Abort initiates an expedited endpoint teardown. It puts the endpoint + // in a closed state and frees all resources associated with it. This + // cleanup may happen asynchronously. Wait can be used to block on this + // asynchronous cleanup. + Abort() + + // Wait waits for any worker goroutines owned by the endpoint to stop. + // + // An endpoint can be requested to stop its worker goroutines by calling + // its Close method. + // + // Wait will not block if the endpoint hasn't started any goroutines + // yet, even if it might later. + Wait() +} + +// RawTransportEndpoint is the interface that needs to be implemented by raw +// transport protocol endpoints. RawTransportEndpoints receive the entire +// packet - including the network and transport headers - as delivered to +// netstack. +type RawTransportEndpoint interface { + // HandlePacket is called by the stack when new packets arrive to + // this transport endpoint. The packet contains all data from the link + // layer up. + // + // HandlePacket may modify the packet. + HandlePacket(*PacketBuffer) +} + +// PacketEndpoint is the interface that needs to be implemented by packet +// transport protocol endpoints. These endpoints receive link layer headers in +// addition to whatever they contain (usually network and transport layer +// headers and a payload). +type PacketEndpoint interface { + // HandlePacket is called by the stack when new packets arrive that + // match the endpoint. + // + // Implementers should treat packet as immutable and should copy it + // before modification. + // + // linkHeader may have a length of 0, in which case the PacketEndpoint + // should construct its own ethernet header for applications. + // + // HandlePacket may modify pkt. + HandlePacket(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, pkt *PacketBuffer) +} + +// MappablePacketEndpoint is a packet endpoint that supports forwarding its +// packets to a PacketMMapEndpoint. +type MappablePacketEndpoint interface { + PacketEndpoint + + // GetPacketMMapOpts returns the options for initializing a PacketMMapEndpoint + // for this endpoint. + GetPacketMMapOpts(req *tcpip.TpacketReq, isRx bool) PacketMMapOpts + + // SetPacketMMapEndpoint sets the PacketMMapEndpoint for this endpoint. All + // packets received by this endpoint will be forwarded to the provided + // PacketMMapEndpoint. + SetPacketMMapEndpoint(ep PacketMMapEndpoint) + + // GetPacketMMapEndpoint returns the PacketMMapEndpoint for this endpoint or + // nil if there is none. + GetPacketMMapEndpoint() PacketMMapEndpoint + + // HandlePacketMMapCopy is a function that is called when a packet received is + // too large for the buffer size specified for the memory mapped endpoint. In + // this case, the packet is copied and passed to the original packet endpoint. + HandlePacketMMapCopy(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, pkt *PacketBuffer) +} + +// PacketMMapOpts are the options for initializing a PacketMMapEndpoint. +// +// +stateify savable +type PacketMMapOpts struct { + Req *tcpip.TpacketReq + IsRx bool + Cooked bool + Stack *Stack + Wq *waiter.Queue + PacketEndpoint MappablePacketEndpoint + Version int + Reserve uint32 +} + +// PacketMMapEndpoint is the interface implemented by endpoints to handle memory +// mapped packets over the packet transport protocol (PACKET_MMAP). +type PacketMMapEndpoint interface { + // HandlePacket is called by the stack when new packets arrive that + // match the endpoint. It returns true if the packet was handled by the + // endpoint and false otherwise. + // + // Implementers should treat packet as immutable and should copy it + // before modification. + // + // linkHeader may have a length of 0, in which case the PacketEndpoint + // should construct its own ethernet header for applications. + // + // HandlePacket may modify pkt. + HandlePacket(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, pkt *PacketBuffer) bool + + // Close releases any resources associated with the endpoint. + Close() + + // Readiness returns the events that the endpoint is ready for. + Readiness(mask waiter.EventMask) waiter.EventMask + + // Stats returns the statistics for the endpoint that can be used for + // getsockopt(PACKET_STATISTICS). + Stats() tcpip.TpacketStats +} + +// UnknownDestinationPacketDisposition enumerates the possible return values from +// HandleUnknownDestinationPacket(). +type UnknownDestinationPacketDisposition int + +const ( + // UnknownDestinationPacketMalformed denotes that the packet was malformed + // and no further processing should be attempted other than updating + // statistics. + UnknownDestinationPacketMalformed UnknownDestinationPacketDisposition = iota + + // UnknownDestinationPacketUnhandled tells the caller that the packet was + // well formed but that the issue was not handled and the stack should take + // the default action. + UnknownDestinationPacketUnhandled + + // UnknownDestinationPacketHandled tells the caller that it should do + // no further processing. + UnknownDestinationPacketHandled +) + +// TransportProtocol is the interface that needs to be implemented by transport +// protocols (e.g., tcp, udp) that want to be part of the networking stack. +type TransportProtocol interface { + // Number returns the transport protocol number. + Number() tcpip.TransportProtocolNumber + + // NewEndpoint creates a new endpoint of the transport protocol. + NewEndpoint(netProto tcpip.NetworkProtocolNumber, waitQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) + + // NewRawEndpoint creates a new raw endpoint of the transport protocol. + NewRawEndpoint(netProto tcpip.NetworkProtocolNumber, waitQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) + + // MinimumPacketSize returns the minimum valid packet size of this + // transport protocol. The stack automatically drops any packets smaller + // than this targeted at this protocol. + MinimumPacketSize() int + + // ParsePorts returns the source and destination ports stored in a + // packet of this protocol. + ParsePorts(b []byte) (src, dst uint16, err tcpip.Error) + + // HandleUnknownDestinationPacket handles packets targeted at this + // protocol that don't match any existing endpoint. For example, + // it is targeted at a port that has no listeners. + // + // HandleUnknownDestinationPacket may modify the packet if it handles + // the issue. + HandleUnknownDestinationPacket(TransportEndpointID, *PacketBuffer) UnknownDestinationPacketDisposition + + // SetOption allows enabling/disabling protocol specific features. + // SetOption returns an error if the option is not supported or the + // provided option value is invalid. + SetOption(option tcpip.SettableTransportProtocolOption) tcpip.Error + + // Option allows retrieving protocol specific option values. + // Option returns an error if the option is not supported or the + // provided option value is invalid. + Option(option tcpip.GettableTransportProtocolOption) tcpip.Error + + // Close requests that any worker goroutines owned by the protocol + // stop. + Close() + + // Wait waits for any worker goroutines owned by the protocol to stop. + Wait() + + // Pause requests that any protocol level background workers pause. + Pause() + + // Resume resumes any protocol level background workers that were + // previously paused by Pause. + Resume() + + // Restore starts any protocol level background workers during restore. + Restore() + + // Parse sets pkt.TransportHeader and trims pkt.Data appropriately. It does + // neither and returns false if pkt.Data is too small, i.e. pkt.Data.Size() < + // MinimumPacketSize() + Parse(pkt *PacketBuffer) (ok bool) +} + +// TransportPacketDisposition is the result from attempting to deliver a packet +// to the transport layer. +type TransportPacketDisposition int + +const ( + // TransportPacketHandled indicates that a transport packet was handled by the + // transport layer and callers need not take any further action. + TransportPacketHandled TransportPacketDisposition = iota + + // TransportPacketProtocolUnreachable indicates that the transport + // protocol requested in the packet is not supported. + TransportPacketProtocolUnreachable + + // TransportPacketDestinationPortUnreachable indicates that there weren't any + // listeners interested in the packet and the transport protocol has no means + // to notify the sender. + TransportPacketDestinationPortUnreachable +) + +// TransportDispatcher contains the methods used by the network stack to deliver +// packets to the appropriate transport endpoint after it has been handled by +// the network layer. +type TransportDispatcher interface { + // DeliverTransportPacket delivers packets to the appropriate + // transport protocol endpoint. + // + // pkt.NetworkHeader must be set before calling DeliverTransportPacket. + // + // DeliverTransportPacket may modify the packet. + DeliverTransportPacket(tcpip.TransportProtocolNumber, *PacketBuffer) TransportPacketDisposition + + // DeliverTransportError delivers an error to the appropriate transport + // endpoint. + // + // DeliverTransportError may modify the packet buffer. + DeliverTransportError(local, remote tcpip.Address, _ tcpip.NetworkProtocolNumber, _ tcpip.TransportProtocolNumber, _ TransportError, _ *PacketBuffer) + + // DeliverRawPacket delivers a packet to any subscribed raw sockets. + // + // DeliverRawPacket does NOT take ownership of the packet buffer. + DeliverRawPacket(tcpip.TransportProtocolNumber, *PacketBuffer) +} + +// PacketLooping specifies where an outbound packet should be sent. +type PacketLooping byte + +const ( + // PacketOut indicates that the packet should be passed to the link + // endpoint. + PacketOut PacketLooping = 1 << iota + + // PacketLoop indicates that the packet should be handled locally. + PacketLoop +) + +// NetworkHeaderParams are the header parameters given as input by the +// transport endpoint to the network. +type NetworkHeaderParams struct { + // Protocol refers to the transport protocol number. + Protocol tcpip.TransportProtocolNumber + + // TTL refers to Time To Live field of the IP-header. + TTL uint8 + + // TOS refers to TypeOfService or TrafficClass field of the IP-header. + TOS uint8 + + // DF indicates whether the DF bit should be set. + DF bool + + // ExperimentOptionValue is a 16 bit value that is set for the IP experiment + // option headers if it is not zero. + ExperimentOptionValue uint16 +} + +// GroupAddressableEndpoint is an endpoint that supports group addressing. +// +// An endpoint is considered to support group addressing when one or more +// endpoints may associate themselves with the same identifier (group address). +type GroupAddressableEndpoint interface { + // JoinGroup joins the specified group. + JoinGroup(group tcpip.Address) tcpip.Error + + // LeaveGroup attempts to leave the specified group. + LeaveGroup(group tcpip.Address) tcpip.Error + + // IsInGroup returns true if the endpoint is a member of the specified group. + IsInGroup(group tcpip.Address) bool +} + +// PrimaryEndpointBehavior is an enumeration of an AddressEndpoint's primary +// behavior. +type PrimaryEndpointBehavior int + +const ( + // CanBePrimaryEndpoint indicates the endpoint can be used as a primary + // endpoint for new connections with no local address. + CanBePrimaryEndpoint PrimaryEndpointBehavior = iota + + // FirstPrimaryEndpoint indicates the endpoint should be the first + // primary endpoint considered. If there are multiple endpoints with + // this behavior, they are ordered by recency. + FirstPrimaryEndpoint + + // NeverPrimaryEndpoint indicates the endpoint should never be a + // primary endpoint. + NeverPrimaryEndpoint +) + +func (peb PrimaryEndpointBehavior) String() string { + switch peb { + case CanBePrimaryEndpoint: + return "CanBePrimaryEndpoint" + case FirstPrimaryEndpoint: + return "FirstPrimaryEndpoint" + case NeverPrimaryEndpoint: + return "NeverPrimaryEndpoint" + default: + panic(fmt.Sprintf("unknown primary endpoint behavior: %d", peb)) + } +} + +// AddressConfigType is the method used to add an address. +type AddressConfigType int + +const ( + // AddressConfigStatic is a statically configured address endpoint that was + // added by some user-specified action (adding an explicit address, joining a + // multicast group). + AddressConfigStatic AddressConfigType = iota + + // AddressConfigSlaac is an address endpoint added by SLAAC, as per RFC 4862 + // section 5.5.3. + AddressConfigSlaac +) + +// AddressLifetimes encodes an address' preferred and valid lifetimes, as well +// as if the address is deprecated. +// +// +stateify savable +type AddressLifetimes struct { + // Deprecated is whether the address is deprecated. + Deprecated bool + + // PreferredUntil is the time at which the address will be deprecated. + // + // Note that for certain addresses, deprecating the address at the + // PreferredUntil time is not handled as a scheduled job by the stack, but + // is information provided by the owner as an indication of when it will + // deprecate the address. + // + // PreferredUntil should be ignored if Deprecated is true. If Deprecated + // is false, and PreferredUntil is the zero value, no information about + // the preferred lifetime can be inferred. + PreferredUntil tcpip.MonotonicTime + + // ValidUntil is the time at which the address will be invalidated. + // + // Note that for certain addresses, invalidating the address at the + // ValidUntil time is not handled as a scheduled job by the stack, but + // is information provided by the owner as an indication of when it will + // invalidate the address. + // + // If ValidUntil is the zero value, no information about the valid lifetime + // can be inferred. + ValidUntil tcpip.MonotonicTime +} + +// AddressProperties contains additional properties that can be configured when +// adding an address. +type AddressProperties struct { + PEB PrimaryEndpointBehavior + ConfigType AddressConfigType + // Lifetimes encodes the address' lifetimes. + // + // Lifetimes.PreferredUntil and Lifetimes.ValidUntil are informational, i.e. + // the stack will not deprecated nor invalidate the address upon reaching + // these timestamps. + // + // If Lifetimes.Deprecated is true, the address will be added as deprecated. + Lifetimes AddressLifetimes + // Temporary is as defined in RFC 4941, but applies not only to addresses + // added via SLAAC, e.g. DHCPv6 can also add temporary addresses. Temporary + // addresses are short-lived and are not to be valid (or preferred) + // forever; hence the term temporary. + Temporary bool + Disp AddressDispatcher +} + +// AddressAssignmentState is an address' assignment state. +type AddressAssignmentState int + +const ( + _ AddressAssignmentState = iota + + // AddressDisabled indicates the NIC the address is assigned to is disabled. + AddressDisabled + + // AddressTentative indicates an address is yet to pass DAD (IPv4 addresses + // are never tentative). + AddressTentative + + // AddressAssigned indicates an address is assigned. + AddressAssigned +) + +func (state AddressAssignmentState) String() string { + switch state { + case AddressDisabled: + return "Disabled" + case AddressTentative: + return "Tentative" + case AddressAssigned: + return "Assigned" + default: + panic(fmt.Sprintf("unknown address assignment state: %d", state)) + } +} + +// AddressRemovalReason is the reason an address was removed. +type AddressRemovalReason int + +const ( + _ AddressRemovalReason = iota + + // AddressRemovalManualAction indicates the address was removed explicitly + // using the stack API. + AddressRemovalManualAction + + // AddressRemovalInterfaceRemoved indicates the address was removed because + // the NIC it is assigned to was removed. + AddressRemovalInterfaceRemoved + + // AddressRemovalDADFailed indicates the address was removed because DAD + // failed. + AddressRemovalDADFailed + + // AddressRemovalInvalidated indicates the address was removed because it + // was invalidated. + AddressRemovalInvalidated +) + +func (reason AddressRemovalReason) String() string { + switch reason { + case AddressRemovalManualAction: + return "ManualAction" + case AddressRemovalInterfaceRemoved: + return "InterfaceRemoved" + case AddressRemovalDADFailed: + return "DADFailed" + case AddressRemovalInvalidated: + return "Invalidated" + default: + panic(fmt.Sprintf("unknown address removal reason: %d", reason)) + } +} + +// AddressDispatcher is the interface integrators can implement to receive +// address-related events. +type AddressDispatcher interface { + // OnChanged is called with an address' properties when they change. + // + // OnChanged is called once when the address is added with the initial state, + // and every time a property changes. + // + // The PreferredUntil and ValidUntil fields in AddressLifetimes must be + // considered informational, i.e. one must not consider an address to be + // deprecated/invalid even if the monotonic clock timestamp is past these + // deadlines. The Deprecated field indicates whether an address is + // preferred or not; and OnRemoved will be called when an address is + // removed due to invalidation. + OnChanged(AddressLifetimes, AddressAssignmentState) + + // OnRemoved is called when an address is removed with the removal reason. + OnRemoved(AddressRemovalReason) +} + +// AssignableAddressEndpoint is a reference counted address endpoint that may be +// assigned to a NetworkEndpoint. +type AssignableAddressEndpoint interface { + // AddressWithPrefix returns the endpoint's address. + AddressWithPrefix() tcpip.AddressWithPrefix + + // Subnet returns the subnet of the endpoint's address. + Subnet() tcpip.Subnet + + // IsAssigned returns whether or not the endpoint is considered bound + // to its NetworkEndpoint. + IsAssigned(allowExpired bool) bool + + // TryIncRef tries to increment this endpoint's reference count. + // + // Returns true if it was successfully incremented. If it returns false, then + // the endpoint is considered expired and should no longer be used. + TryIncRef() bool + + // DecRef decrements this endpoint's reference count. + DecRef() +} + +// AddressEndpoint is an endpoint representing an address assigned to an +// AddressableEndpoint. +type AddressEndpoint interface { + AssignableAddressEndpoint + + // GetKind returns the address kind for this endpoint. + GetKind() AddressKind + + // SetKind sets the address kind for this endpoint. + SetKind(AddressKind) + + // ConfigType returns the method used to add the address. + ConfigType() AddressConfigType + + // Deprecated returns whether or not this endpoint is deprecated. + Deprecated() bool + + // SetDeprecated sets this endpoint's deprecated status. + SetDeprecated(bool) + + // Lifetimes returns this endpoint's lifetimes. + Lifetimes() AddressLifetimes + + // SetLifetimes sets this endpoint's lifetimes. + // + // Note that setting preferred-until and valid-until times do not result in + // deprecation/invalidation jobs to be scheduled by the stack. + SetLifetimes(AddressLifetimes) + + // Temporary returns whether or not this endpoint is temporary. + Temporary() bool + + // RegisterDispatcher registers an address dispatcher. + // + // OnChanged will be called immediately on the provided address dispatcher + // with this endpoint's current state. + RegisterDispatcher(AddressDispatcher) +} + +// AddressKind is the kind of an address. +// +// See the values of AddressKind for more details. +type AddressKind int + +const ( + // PermanentTentative is a permanent address endpoint that is not yet + // considered to be fully bound to an interface in the traditional + // sense. That is, the address is associated with a NIC, but packets + // destined to the address MUST NOT be accepted and MUST be silently + // dropped, and the address MUST NOT be used as a source address for + // outgoing packets. For IPv6, addresses are of this kind until NDP's + // Duplicate Address Detection (DAD) resolves. If DAD fails, the address + // is removed. + PermanentTentative AddressKind = iota + + // Permanent is a permanent endpoint (vs. a temporary one) assigned to the + // NIC. Its reference count is biased by 1 to avoid removal when no route + // holds a reference to it. It is removed by explicitly removing the address + // from the NIC. + Permanent + + // PermanentExpired is a permanent endpoint that had its address removed from + // the NIC, and it is waiting to be removed once no references to it are held. + // + // If the address is re-added before the endpoint is removed, its type + // changes back to Permanent. + PermanentExpired + + // Temporary is an endpoint, created on a one-off basis to temporarily + // consider the NIC bound an an address that it is not explicitly bound to + // (such as a permanent address). Its reference count must not be biased by 1 + // so that the address is removed immediately when references to it are no + // longer held. + // + // A temporary endpoint may be promoted to permanent if the address is added + // permanently. + Temporary +) + +// IsPermanent returns true if the AddressKind represents a permanent address. +func (k AddressKind) IsPermanent() bool { + switch k { + case Permanent, PermanentTentative: + return true + case Temporary, PermanentExpired: + return false + default: + panic(fmt.Sprintf("unrecognized address kind = %d", k)) + } +} + +// AddressableEndpoint is an endpoint that supports addressing. +// +// An endpoint is considered to support addressing when the endpoint may +// associate itself with an identifier (address). +type AddressableEndpoint interface { + // AddAndAcquirePermanentAddress adds the passed permanent address. + // + // Returns *tcpip.ErrDuplicateAddress if the address exists. + // + // Acquires and returns the AddressEndpoint for the added address. + AddAndAcquirePermanentAddress(addr tcpip.AddressWithPrefix, properties AddressProperties) (AddressEndpoint, tcpip.Error) + + // RemovePermanentAddress removes the passed address if it is a permanent + // address. + // + // Returns *tcpip.ErrBadLocalAddress if the endpoint does not have the passed + // permanent address. + RemovePermanentAddress(addr tcpip.Address) tcpip.Error + + // SetLifetimes sets an address' lifetimes (strictly informational) and + // whether it should be deprecated or preferred. + // + // Returns *tcpip.ErrBadLocalAddress if the endpoint does not have the passed + // address. + SetLifetimes(addr tcpip.Address, lifetimes AddressLifetimes) tcpip.Error + + // MainAddress returns the endpoint's primary permanent address. + MainAddress() tcpip.AddressWithPrefix + + // AcquireAssignedAddress returns an address endpoint for the passed address + // that is considered bound to the endpoint, optionally creating a temporary + // endpoint if requested and no existing address exists. + // + // The returned endpoint's reference count is incremented if readOnly is + // false. + // + // Returns nil if the specified address is not local to this endpoint. + AcquireAssignedAddress(localAddr tcpip.Address, allowTemp bool, tempPEB PrimaryEndpointBehavior, readOnly bool) AddressEndpoint + + // AcquireOutgoingPrimaryAddress returns a primary address that may be used as + // a source address when sending packets to the passed remote address. + // + // If allowExpired is true, expired addresses may be returned. + // + // The returned endpoint's reference count is incremented. + // + // Returns nil if a primary address is not available. + AcquireOutgoingPrimaryAddress(remoteAddr, srcHint tcpip.Address, allowExpired bool) AddressEndpoint + + // PrimaryAddresses returns the primary addresses. + PrimaryAddresses() []tcpip.AddressWithPrefix + + // PermanentAddresses returns all the permanent addresses. + PermanentAddresses() []tcpip.AddressWithPrefix +} + +// NDPEndpoint is a network endpoint that supports NDP. +type NDPEndpoint interface { + NetworkEndpoint + + // InvalidateDefaultRouter invalidates a default router discovered through + // NDP. + InvalidateDefaultRouter(tcpip.Address) +} + +// NetworkInterface is a network interface. +type NetworkInterface interface { + NetworkLinkEndpoint + + // ID returns the interface's ID. + ID() tcpip.NICID + + // IsLoopback returns true if the interface is a loopback interface. + IsLoopback() bool + + // Name returns the name of the interface. + // + // May return an empty string if the interface is not configured with a name. + Name() string + + // Enabled returns true if the interface is enabled. + Enabled() bool + + // Promiscuous returns true if the interface is in promiscuous mode. + // + // When in promiscuous mode, the interface should accept all packets. + Promiscuous() bool + + // Spoofing returns true if the interface is in spoofing mode. + // + // When in spoofing mode, the interface should consider all addresses as + // assigned to it. + Spoofing() bool + + // PrimaryAddress returns the primary address associated with the interface. + // + // PrimaryAddress will return the first non-deprecated address if such an + // address exists. If no non-deprecated addresses exist, the first deprecated + // address will be returned. If no deprecated addresses exist, the zero value + // will be returned. + PrimaryAddress(tcpip.NetworkProtocolNumber) (tcpip.AddressWithPrefix, tcpip.Error) + + // CheckLocalAddress returns true if the address exists on the interface. + CheckLocalAddress(tcpip.NetworkProtocolNumber, tcpip.Address) bool + + // WritePacketToRemote writes the packet to the given remote link address. + WritePacketToRemote(tcpip.LinkAddress, *PacketBuffer) tcpip.Error + + // WritePacket writes a packet through the given route. + // + // WritePacket may modify the packet buffer. The packet buffer's + // network and transport header must be set. + WritePacket(*Route, *PacketBuffer) tcpip.Error + + // HandleNeighborProbe processes an incoming neighbor probe (e.g. ARP + // request or NDP Neighbor Solicitation). + // + // HandleNeighborProbe assumes that the probe is valid for the network + // interface the probe was received on. + HandleNeighborProbe(tcpip.NetworkProtocolNumber, tcpip.Address, tcpip.LinkAddress) tcpip.Error + + // HandleNeighborConfirmation processes an incoming neighbor confirmation + // (e.g. ARP reply or NDP Neighbor Advertisement). + HandleNeighborConfirmation(tcpip.NetworkProtocolNumber, tcpip.Address, tcpip.LinkAddress, ReachabilityConfirmationFlags) tcpip.Error +} + +// LinkResolvableNetworkEndpoint handles link resolution events. +type LinkResolvableNetworkEndpoint interface { + // HandleLinkResolutionFailure is called when link resolution prevents the + // argument from having been sent. + HandleLinkResolutionFailure(*PacketBuffer) +} + +// NetworkEndpoint is the interface that needs to be implemented by endpoints +// of network layer protocols (e.g., ipv4, ipv6). +type NetworkEndpoint interface { + // Enable enables the endpoint. + // + // Must only be called when the stack is in a state that allows the endpoint + // to send and receive packets. + // + // Returns *tcpip.ErrNotPermitted if the endpoint cannot be enabled. + Enable() tcpip.Error + + // Enabled returns true if the endpoint is enabled. + Enabled() bool + + // Disable disables the endpoint. + Disable() + + // DefaultTTL is the default time-to-live value (or hop limit, in ipv6) + // for this endpoint. + DefaultTTL() uint8 + + // MTU is the maximum transmission unit for this endpoint. This is + // generally calculated as the MTU of the underlying data link endpoint + // minus the network endpoint max header length. + MTU() uint32 + + // MaxHeaderLength returns the maximum size the network (and lower + // level layers combined) headers can have. Higher levels use this + // information to reserve space in the front of the packets they're + // building. + MaxHeaderLength() uint16 + + // WritePacket writes a packet to the given destination address and + // protocol. It may modify pkt. pkt.TransportHeader must have + // already been set. + WritePacket(r *Route, params NetworkHeaderParams, pkt *PacketBuffer) tcpip.Error + + // WriteHeaderIncludedPacket writes a packet that includes a network + // header to the given destination address. It may modify pkt. + WriteHeaderIncludedPacket(r *Route, pkt *PacketBuffer) tcpip.Error + + // HandlePacket is called by the link layer when new packets arrive to + // this network endpoint. It sets pkt.NetworkHeader. + // + // HandlePacket may modify pkt. + HandlePacket(pkt *PacketBuffer) + + // Close is called when the endpoint is removed from a stack. + Close() + + // NetworkProtocolNumber returns the tcpip.NetworkProtocolNumber for + // this endpoint. + NetworkProtocolNumber() tcpip.NetworkProtocolNumber + + // Stats returns a reference to the network endpoint stats. + Stats() NetworkEndpointStats +} + +// NetworkEndpointStats is the interface implemented by each network endpoint +// stats struct. +type NetworkEndpointStats interface { + // IsNetworkEndpointStats is an empty method to implement the + // NetworkEndpointStats marker interface. + IsNetworkEndpointStats() +} + +// IPNetworkEndpointStats is a NetworkEndpointStats that tracks IP-related +// statistics. +type IPNetworkEndpointStats interface { + NetworkEndpointStats + + // IPStats returns the IP statistics of a network endpoint. + IPStats() *tcpip.IPStats +} + +// ForwardingNetworkEndpoint is a network endpoint that may forward packets. +type ForwardingNetworkEndpoint interface { + NetworkEndpoint + + // Forwarding returns the forwarding configuration. + Forwarding() bool + + // SetForwarding sets the forwarding configuration. + // + // Returns the previous forwarding configuration. + SetForwarding(bool) bool +} + +// MulticastForwardingNetworkEndpoint is a network endpoint that may forward +// multicast packets. +type MulticastForwardingNetworkEndpoint interface { + ForwardingNetworkEndpoint + + // MulticastForwarding returns true if multicast forwarding is enabled. + // Otherwise, returns false. + MulticastForwarding() bool + + // SetMulticastForwarding sets the multicast forwarding configuration. + // + // Returns the previous forwarding configuration. + SetMulticastForwarding(bool) bool +} + +// NetworkProtocol is the interface that needs to be implemented by network +// protocols (e.g., ipv4, ipv6) that want to be part of the networking stack. +type NetworkProtocol interface { + // Number returns the network protocol number. + Number() tcpip.NetworkProtocolNumber + + // MinimumPacketSize returns the minimum valid packet size of this + // network protocol. The stack automatically drops any packets smaller + // than this targeted at this protocol. + MinimumPacketSize() int + + // ParseAddresses returns the source and destination addresses stored in a + // packet of this protocol. + ParseAddresses(b []byte) (src, dst tcpip.Address) + + // NewEndpoint creates a new endpoint of this protocol. + NewEndpoint(nic NetworkInterface, dispatcher TransportDispatcher) NetworkEndpoint + + // SetOption allows enabling/disabling protocol specific features. + // SetOption returns an error if the option is not supported or the + // provided option value is invalid. + SetOption(option tcpip.SettableNetworkProtocolOption) tcpip.Error + + // Option allows retrieving protocol specific option values. + // Option returns an error if the option is not supported or the + // provided option value is invalid. + Option(option tcpip.GettableNetworkProtocolOption) tcpip.Error + + // Close requests that any worker goroutines owned by the protocol + // stop. + Close() + + // Wait waits for any worker goroutines owned by the protocol to stop. + Wait() + + // Parse sets pkt.NetworkHeader and trims pkt.Data appropriately. It + // returns: + // - The encapsulated protocol, if present. + // - Whether there is an encapsulated transport protocol payload (e.g. ARP + // does not encapsulate anything). + // - Whether pkt.Data was large enough to parse and set pkt.NetworkHeader. + Parse(pkt *PacketBuffer) (proto tcpip.TransportProtocolNumber, hasTransportHdr bool, ok bool) +} + +// UnicastSourceAndMulticastDestination is a tuple that represents a unicast +// source address and a multicast destination address. +// +// +stateify savable +type UnicastSourceAndMulticastDestination struct { + // Source represents a unicast source address. + Source tcpip.Address + // Destination represents a multicast destination address. + Destination tcpip.Address +} + +// MulticastRouteOutgoingInterface represents an outgoing interface in a +// multicast route. +type MulticastRouteOutgoingInterface struct { + // ID corresponds to the outgoing NIC. + ID tcpip.NICID + + // MinTTL represents the minimum TTL/HopLimit a multicast packet must have to + // be sent through the outgoing interface. + // + // Note: a value of 0 allows all packets to be forwarded. + MinTTL uint8 +} + +// MulticastRoute is a multicast route. +type MulticastRoute struct { + // ExpectedInputInterface is the interface on which packets using this route + // are expected to ingress. + ExpectedInputInterface tcpip.NICID + + // OutgoingInterfaces is the set of interfaces that a multicast packet should + // be forwarded out of. + // + // This field should not be empty. + OutgoingInterfaces []MulticastRouteOutgoingInterface +} + +// MulticastForwardingNetworkProtocol is the interface that needs to be +// implemented by the network protocols that support multicast forwarding. +type MulticastForwardingNetworkProtocol interface { + NetworkProtocol + + // AddMulticastRoute adds a route to the multicast routing table such that + // packets matching the addresses will be forwarded using the provided route. + // + // Returns an error if the addresses or route is invalid. + AddMulticastRoute(UnicastSourceAndMulticastDestination, MulticastRoute) tcpip.Error + + // RemoveMulticastRoute removes the route matching the provided addresses + // from the multicast routing table. + // + // Returns an error if the addresses are invalid or a matching route is not + // found. + RemoveMulticastRoute(UnicastSourceAndMulticastDestination) tcpip.Error + + // MulticastRouteLastUsedTime returns a monotonic timestamp that + // represents the last time that the route matching the provided addresses + // was used or updated. + // + // Returns an error if the addresses are invalid or a matching route was not + // found. + MulticastRouteLastUsedTime(UnicastSourceAndMulticastDestination) (tcpip.MonotonicTime, tcpip.Error) + + // EnableMulticastForwarding enables multicast forwarding for the protocol. + // + // Returns an error if the provided multicast forwarding event dispatcher is + // nil. Otherwise, returns true if the multicast forwarding was already + // enabled. + EnableMulticastForwarding(MulticastForwardingEventDispatcher) (bool, tcpip.Error) + + // DisableMulticastForwarding disables multicast forwarding for the protocol. + DisableMulticastForwarding() +} + +// MulticastPacketContext is the context in which a multicast packet triggered +// a multicast forwarding event. +type MulticastPacketContext struct { + // SourceAndDestination contains the unicast source address and the multicast + // destination address found in the relevant multicast packet. + SourceAndDestination UnicastSourceAndMulticastDestination + // InputInterface is the interface on which the relevant multicast packet + // arrived. + InputInterface tcpip.NICID +} + +// MulticastForwardingEventDispatcher is the interface that integrators should +// implement to handle multicast routing events. +type MulticastForwardingEventDispatcher interface { + // OnMissingRoute is called when an incoming multicast packet does not match + // any installed route. + // + // The packet that triggered this event may be queued so that it can be + // transmitted once a route is installed. Even then, it may still be dropped + // as per the routing table's GC/eviction policy. + OnMissingRoute(MulticastPacketContext) + + // OnUnexpectedInputInterface is called when a multicast packet arrives at an + // interface that does not match the installed route's expected input + // interface. + // + // This may be an indication of a routing loop. The packet that triggered + // this event is dropped without being forwarded. + OnUnexpectedInputInterface(context MulticastPacketContext, expectedInputInterface tcpip.NICID) +} + +// NetworkDispatcher contains the methods used by the network stack to deliver +// inbound/outbound packets to the appropriate network/packet(if any) endpoints. +type NetworkDispatcher interface { + // DeliverNetworkPacket finds the appropriate network protocol endpoint + // and hands the packet over for further processing. + // + // + // If the link-layer has a header, the packet's link header must be populated. + // + // DeliverNetworkPacket may modify pkt. + DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) + + // DeliverLinkPacket delivers a packet to any interested packet endpoints. + // + // This method should be called with both incoming and outgoing packets. + // + // If the link-layer has a header, the packet's link header must be populated. + DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) +} + +// LinkEndpointCapabilities is the type associated with the capabilities +// supported by a link-layer endpoint. It is a set of bitfields. +type LinkEndpointCapabilities uint + +// The following are the supported link endpoint capabilities. +const ( + CapabilityNone LinkEndpointCapabilities = 0 + // CapabilityTXChecksumOffload indicates that the link endpoint supports + // checksum computation for outgoing packets and the stack can skip + // computing checksums when sending packets. + CapabilityTXChecksumOffload LinkEndpointCapabilities = 1 << iota + // CapabilityRXChecksumOffload indicates that the link endpoint supports + // checksum verification on received packets and that it's safe for the + // stack to skip checksum verification. + CapabilityRXChecksumOffload + CapabilityResolutionRequired + CapabilitySaveRestore + CapabilityDisconnectOk + CapabilityLoopback +) + +// LinkWriter is an interface that supports sending packets via a data-link +// layer endpoint. It is used with QueueingDiscipline to batch writes from +// upper layer endpoints. +type LinkWriter interface { + // WritePackets writes packets. Must not be called with an empty list of + // packet buffers. + // + // Each packet must have the link-layer header set, if the link requires + // one. + // + // WritePackets may modify the packet buffers, and takes ownership of the PacketBufferList. + // it is not safe to use the PacketBufferList after a call to WritePackets. + WritePackets(PacketBufferList) (int, tcpip.Error) +} + +// NetworkLinkEndpoint is a data-link layer that supports sending network +// layer packets. +type NetworkLinkEndpoint interface { + // MTU is the maximum transmission unit for this endpoint. This is + // usually dictated by the backing physical network; when such a + // physical network doesn't exist, the limit is generally 64k, which + // includes the maximum size of an IP packet. + MTU() uint32 + + // SetMTU update the maximum transmission unit for the endpoint. + SetMTU(mtu uint32) + + // MaxHeaderLength returns the maximum size the data link (and + // lower level layers combined) headers can have. Higher levels use this + // information to reserve space in the front of the packets they're + // building. + MaxHeaderLength() uint16 + + // LinkAddress returns the link address (typically a MAC) of the + // endpoint. + LinkAddress() tcpip.LinkAddress + + // SetLinkAddress updated the endpoint's link address (typically a MAC). + SetLinkAddress(addr tcpip.LinkAddress) + + // Capabilities returns the set of capabilities supported by the + // endpoint. + Capabilities() LinkEndpointCapabilities + + // Attach attaches the data link layer endpoint to the network-layer + // dispatcher of the stack. + // + // Attach is called with a nil dispatcher when the endpoint's NIC is being + // removed. + Attach(dispatcher NetworkDispatcher) + + // IsAttached returns whether a NetworkDispatcher is attached to the + // endpoint. + IsAttached() bool + + // Wait waits for any worker goroutines owned by the endpoint to stop. + // + // For now, requesting that an endpoint's worker goroutine(s) stop is + // implementation specific. + // + // Wait will not block if the endpoint hasn't started any goroutines + // yet, even if it might later. + Wait() + + // ARPHardwareType returns the ARPHRD_TYPE of the link endpoint. + // + // See: + // https://github.com/torvalds/linux/blob/aa0c9086b40c17a7ad94425b3b70dd1fdd7497bf/include/uapi/linux/if_arp.h#L30 + ARPHardwareType() header.ARPHardwareType + + // AddHeader adds a link layer header to the packet if required. + AddHeader(*PacketBuffer) + + // ParseHeader parses the link layer header to the packet. + ParseHeader(*PacketBuffer) bool + + // Close is called when the endpoint is removed from a stack. + Close() + + // SetOnCloseAction sets the action that will be executed before closing the + // endpoint. It is used to destroy a network device when its endpoint + // is closed. Endpoints that are closed only after destroying their + // network devices can implement this method as no-op. + SetOnCloseAction(func()) +} + +// QueueingDiscipline provides a queueing strategy for outgoing packets (e.g +// FIFO, LIFO, Random Early Drop etc). +type QueueingDiscipline interface { + // WritePacket writes a packet. + // + // WritePacket may modify the packet buffer. The packet buffer's + // network and transport header must be set. + // + // To participate in transparent bridging, a LinkEndpoint implementation + // should call eth.Encode with header.EthernetFields.SrcAddr set to + // pkg.EgressRoute.LocalLinkAddress if it is provided. + WritePacket(*PacketBuffer) tcpip.Error + + Close() +} + +// LinkEndpoint is the interface implemented by data link layer protocols (e.g., +// ethernet, loopback, raw) and used by network layer protocols to send packets +// out through the implementer's data link endpoint. When a link header exists, +// it sets each PacketBuffer's LinkHeader field before passing it up the +// stack. +type LinkEndpoint interface { + NetworkLinkEndpoint + LinkWriter +} + +// InjectableLinkEndpoint is a LinkEndpoint where inbound packets are +// delivered via the Inject method. +type InjectableLinkEndpoint interface { + LinkEndpoint + + // InjectInbound injects an inbound packet. + InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) + + // InjectOutbound writes a fully formed outbound packet directly to the + // link. + // + // dest is used by endpoints with multiple raw destinations. + InjectOutbound(dest tcpip.Address, packet *buffer.View) tcpip.Error +} + +// DADResult is a marker interface for the result of a duplicate address +// detection process. +type DADResult interface { + isDADResult() +} + +var _ DADResult = (*DADSucceeded)(nil) + +// DADSucceeded indicates DAD completed without finding any duplicate addresses. +type DADSucceeded struct{} + +func (*DADSucceeded) isDADResult() {} + +var _ DADResult = (*DADError)(nil) + +// DADError indicates DAD hit an error. +type DADError struct { + Err tcpip.Error +} + +func (*DADError) isDADResult() {} + +var _ DADResult = (*DADAborted)(nil) + +// DADAborted indicates DAD was aborted. +type DADAborted struct{} + +func (*DADAborted) isDADResult() {} + +var _ DADResult = (*DADDupAddrDetected)(nil) + +// DADDupAddrDetected indicates DAD detected a duplicate address. +type DADDupAddrDetected struct { + // HolderLinkAddress is the link address of the node that holds the duplicate + // address. + HolderLinkAddress tcpip.LinkAddress +} + +func (*DADDupAddrDetected) isDADResult() {} + +// DADCompletionHandler is a handler for DAD completion. +type DADCompletionHandler func(DADResult) + +// DADCheckAddressDisposition enumerates the possible return values from +// DAD.CheckDuplicateAddress. +type DADCheckAddressDisposition int + +const ( + _ DADCheckAddressDisposition = iota + + // DADDisabled indicates that DAD is disabled. + DADDisabled + + // DADStarting indicates that DAD is starting for an address. + DADStarting + + // DADAlreadyRunning indicates that DAD was already started for an address. + DADAlreadyRunning +) + +const ( + // defaultDupAddrDetectTransmits is the default number of NDP Neighbor + // Solicitation messages to send when doing Duplicate Address Detection + // for a tentative address. + // + // Default = 1 (from RFC 4862 section 5.1) + defaultDupAddrDetectTransmits = 1 +) + +// DADConfigurations holds configurations for duplicate address detection. +// +// +stateify savable +type DADConfigurations struct { + // The number of Neighbor Solicitation messages to send when doing + // Duplicate Address Detection for a tentative address. + // + // Note, a value of zero effectively disables DAD. + DupAddrDetectTransmits uint8 + + // The amount of time to wait between sending Neighbor Solicitation + // messages. + // + // Must be greater than or equal to 1ms. + RetransmitTimer time.Duration +} + +// DefaultDADConfigurations returns the default DAD configurations. +func DefaultDADConfigurations() DADConfigurations { + return DADConfigurations{ + DupAddrDetectTransmits: defaultDupAddrDetectTransmits, + RetransmitTimer: defaultRetransmitTimer, + } +} + +// Validate modifies the configuration with valid values. If invalid values are +// present in the configurations, the corresponding default values are used +// instead. +func (c *DADConfigurations) Validate() { + if c.RetransmitTimer < minimumRetransmitTimer { + c.RetransmitTimer = defaultRetransmitTimer + } +} + +// DuplicateAddressDetector handles checking if an address is already assigned +// to some neighboring node on the link. +type DuplicateAddressDetector interface { + // CheckDuplicateAddress checks if an address is assigned to a neighbor. + // + // If DAD is already being performed for the address, the handler will be + // called with the result of the original DAD request. + CheckDuplicateAddress(tcpip.Address, DADCompletionHandler) DADCheckAddressDisposition + + // SetDADConfigurations sets the configurations for DAD. + SetDADConfigurations(c DADConfigurations) + + // DuplicateAddressProtocol returns the network protocol the receiver can + // perform duplicate address detection for. + DuplicateAddressProtocol() tcpip.NetworkProtocolNumber +} + +// LinkAddressResolver handles link address resolution for a network protocol. +type LinkAddressResolver interface { + // LinkAddressRequest sends a request for the link address of the target + // address. The request is broadcast on the local network if a remote link + // address is not provided. + LinkAddressRequest(targetAddr, localAddr tcpip.Address, remoteLinkAddr tcpip.LinkAddress) tcpip.Error + + // ResolveStaticAddress attempts to resolve address without sending + // requests. It either resolves the name immediately or returns the + // empty LinkAddress. + // + // It can be used to resolve broadcast addresses for example. + ResolveStaticAddress(addr tcpip.Address) (tcpip.LinkAddress, bool) + + // LinkAddressProtocol returns the network protocol of the + // addresses this resolver can resolve. + LinkAddressProtocol() tcpip.NetworkProtocolNumber +} + +// RawFactory produces endpoints for writing various types of raw packets. +type RawFactory interface { + // NewUnassociatedEndpoint produces endpoints for writing packets not + // associated with a particular transport protocol. Such endpoints can + // be used to write arbitrary packets that include the network header. + NewUnassociatedEndpoint(stack *Stack, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) + + // NewPacketEndpoint produces endpoints for reading and writing packets + // that include network and (when cooked is false) link layer headers. + NewPacketEndpoint(stack *Stack, cooked bool, netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) +} + +// GSOType is the type of GSO segments. +// +// +stateify savable +type GSOType int + +// Types of gso segments. +const ( + GSONone GSOType = iota + + // Hardware GSO types: + GSOTCPv4 + GSOTCPv6 + + // GSOGvisor is used for gVisor GSO segments which have to be sent by + // endpoint.WritePackets. + GSOGvisor +) + +// GSO contains generic segmentation offload properties. +// +// +stateify savable +type GSO struct { + // Type is one of GSONone, GSOTCPv4, etc. + Type GSOType + // NeedsCsum is set if the checksum offload is enabled. + NeedsCsum bool + // CsumOffset is offset after that to place checksum. + CsumOffset uint16 + + // Mss is maximum segment size. + MSS uint16 + // L3Len is L3 (IP) header length. + L3HdrLen uint16 + + // MaxSize is maximum GSO packet size. + MaxSize uint32 +} + +// SupportedGSO is the type of segmentation offloading supported. +type SupportedGSO int + +const ( + // GSONotSupported indicates that segmentation offloading is not supported. + GSONotSupported SupportedGSO = iota + + // HostGSOSupported indicates that segmentation offloading may be performed + // by the host. This is typically true when netstack is attached to a host + // AF_PACKET socket, and not true when attached to a unix socket or other + // non-networking data layer. + HostGSOSupported + + // GVisorGSOSupported indicates that segmentation offloading may be performed + // in gVisor. + GVisorGSOSupported +) + +// GSOEndpoint provides access to GSO properties. +type GSOEndpoint interface { + // GSOMaxSize returns the maximum GSO packet size. + GSOMaxSize() uint32 + + // SupportedGSO returns the supported segmentation offloading. + SupportedGSO() SupportedGSO +} + +// GVisorGSOMaxSize is a maximum allowed size of a software GSO segment. +// This isn't a hard limit, because it is never set into packet headers. +const GVisorGSOMaxSize = 1 << 16 diff --git a/pkg/tcpip/stack/route.go b/pkg/tcpip/stack/route.go new file mode 100644 index 0000000..ed2bc28 --- /dev/null +++ b/pkg/tcpip/stack/route.go @@ -0,0 +1,598 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +// Route represents a route through the networking stack to a given destination. +// +// It is safe to call Route's methods from multiple goroutines. +type Route struct { + routeInfo routeInfo + + // localAddressNIC is the interface the address is associated with. + // TODO(gvisor.dev/issue/4548): Remove this field once we can query the + // address's assigned status without the NIC. + localAddressNIC *nic + + // mu protects annotated fields below. + mu routeRWMutex + + // localAddressEndpoint is the local address this route is associated with. + // +checklocks:mu + localAddressEndpoint AssignableAddressEndpoint + + // remoteLinkAddress is the link-layer (MAC) address of the next hop. + // +checklocks:mu + remoteLinkAddress tcpip.LinkAddress + + // outgoingNIC is the interface this route uses to write packets. + outgoingNIC *nic + + // linkRes is set if link address resolution is enabled for this protocol on + // the route's NIC. + linkRes *linkResolver + + // neighborEntry is the cached result of fetching a neighbor entry from the + // neighbor cache. + // +checklocks:mu + neighborEntry *neighborEntry + + // mtu is the maximum transmission unit to use for this route. + // If mtu is 0, this field is ignored and the MTU of the outgoing NIC + // is used for egress packets. + mtu uint32 +} + +// +stateify savable +type routeInfo struct { + RemoteAddress tcpip.Address + + LocalAddress tcpip.Address + + LocalLinkAddress tcpip.LinkAddress + + NextHop tcpip.Address + + NetProto tcpip.NetworkProtocolNumber + + Loop PacketLooping +} + +// RemoteAddress returns the route's destination. +func (r *Route) RemoteAddress() tcpip.Address { + return r.routeInfo.RemoteAddress +} + +// LocalAddress returns the route's local address. +func (r *Route) LocalAddress() tcpip.Address { + return r.routeInfo.LocalAddress +} + +// LocalLinkAddress returns the route's local link-layer address. +func (r *Route) LocalLinkAddress() tcpip.LinkAddress { + return r.routeInfo.LocalLinkAddress +} + +// NextHop returns the next node in the route's path to the destination. +func (r *Route) NextHop() tcpip.Address { + return r.routeInfo.NextHop +} + +// NetProto returns the route's network-layer protocol number. +func (r *Route) NetProto() tcpip.NetworkProtocolNumber { + return r.routeInfo.NetProto +} + +// Loop returns the route's required packet looping. +func (r *Route) Loop() PacketLooping { + return r.routeInfo.Loop +} + +// OutgoingNIC returns the route's outgoing NIC. +func (r *Route) OutgoingNIC() tcpip.NICID { + return r.outgoingNIC.id +} + +// RouteInfo contains all of Route's exported fields. +// +// +stateify savable +type RouteInfo struct { + routeInfo + + // RemoteLinkAddress is the link-layer (MAC) address of the next hop in the + // route. + RemoteLinkAddress tcpip.LinkAddress +} + +// Fields returns a RouteInfo with all of the known values for the route's +// fields. +// +// If any fields are unknown (e.g. remote link address when it is waiting for +// link address resolution), they will be unset. +func (r *Route) Fields() RouteInfo { + r.mu.RLock() + defer r.mu.RUnlock() + return r.fieldsLocked() +} + +// +checklocksread:r.mu +func (r *Route) fieldsLocked() RouteInfo { + return RouteInfo{ + routeInfo: r.routeInfo, + RemoteLinkAddress: r.remoteLinkAddress, + } +} + +// constructAndValidateRoute validates and initializes a route. It takes +// ownership of the provided local address. +// +// Returns an empty route if validation fails. +func constructAndValidateRoute(netProto tcpip.NetworkProtocolNumber, addressEndpoint AssignableAddressEndpoint, localAddressNIC, outgoingNIC *nic, gateway, localAddr, remoteAddr tcpip.Address, handleLocal, multicastLoop bool, mtu uint32) *Route { + if localAddr.BitLen() == 0 { + localAddr = addressEndpoint.AddressWithPrefix().Address + } + + if localAddressNIC != outgoingNIC && header.IsV6LinkLocalUnicastAddress(localAddr) { + addressEndpoint.DecRef() + return nil + } + + // If no remote address is provided, use the local address. + if remoteAddr.BitLen() == 0 { + remoteAddr = localAddr + } + + r := makeRoute( + netProto, + gateway, + localAddr, + remoteAddr, + outgoingNIC, + localAddressNIC, + addressEndpoint, + handleLocal, + multicastLoop, + mtu, + ) + + return r +} + +// makeRoute initializes a new route. It takes ownership of the provided +// AssignableAddressEndpoint. +func makeRoute(netProto tcpip.NetworkProtocolNumber, gateway, localAddr, remoteAddr tcpip.Address, outgoingNIC, localAddressNIC *nic, localAddressEndpoint AssignableAddressEndpoint, handleLocal, multicastLoop bool, mtu uint32) *Route { + if localAddressNIC.stack != outgoingNIC.stack { + panic(fmt.Sprintf("cannot create a route with NICs from different stacks")) + } + + if localAddr.BitLen() == 0 { + localAddr = localAddressEndpoint.AddressWithPrefix().Address + } + + loop := PacketOut + + // Loopback interface loops back packets at the link endpoint level. We + // could remove this check if loopback interfaces looped back packets + // at the network layer. + if !outgoingNIC.IsLoopback() { + if handleLocal && localAddr != (tcpip.Address{}) && remoteAddr == localAddr { + loop = PacketLoop + } else if multicastLoop && (header.IsV4MulticastAddress(remoteAddr) || header.IsV6MulticastAddress(remoteAddr)) { + loop |= PacketLoop + } else if remoteAddr == header.IPv4Broadcast { + loop |= PacketLoop + } else if subnet := localAddressEndpoint.AddressWithPrefix().Subnet(); subnet.IsBroadcast(remoteAddr) { + loop |= PacketLoop + } + } + + r := makeRouteInner(netProto, localAddr, remoteAddr, outgoingNIC, localAddressNIC, localAddressEndpoint, loop, mtu) + if r.Loop()&PacketOut == 0 { + // Packet will not leave the stack, no need for a gateway or a remote link + // address. + return r + } + + if r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityResolutionRequired != 0 { + if linkRes, ok := r.outgoingNIC.linkAddrResolvers[r.NetProto()]; ok { + r.linkRes = linkRes + } + } + + if gateway.BitLen() > 0 { + r.routeInfo.NextHop = gateway + return r + } + + if r.linkRes == nil { + return r + } + + if linkAddr, ok := r.linkRes.resolver.ResolveStaticAddress(r.RemoteAddress()); ok { + r.ResolveWith(linkAddr) + return r + } + + if subnet := localAddressEndpoint.Subnet(); subnet.IsBroadcast(remoteAddr) { + r.ResolveWith(header.EthernetBroadcastAddress) + return r + } + + if r.RemoteAddress() == r.LocalAddress() { + // Local link address is already known. + r.ResolveWith(r.LocalLinkAddress()) + } + + return r +} + +func makeRouteInner(netProto tcpip.NetworkProtocolNumber, localAddr, remoteAddr tcpip.Address, outgoingNIC, localAddressNIC *nic, localAddressEndpoint AssignableAddressEndpoint, loop PacketLooping, mtu uint32) *Route { + r := &Route{ + routeInfo: routeInfo{ + NetProto: netProto, + LocalAddress: localAddr, + LocalLinkAddress: outgoingNIC.NetworkLinkEndpoint.LinkAddress(), + RemoteAddress: remoteAddr, + Loop: loop, + }, + localAddressNIC: localAddressNIC, + outgoingNIC: outgoingNIC, + mtu: mtu, + } + + r.mu.Lock() + r.localAddressEndpoint = localAddressEndpoint + r.mu.Unlock() + + return r +} + +// makeLocalRoute initializes a new local route. It takes ownership of the +// provided AssignableAddressEndpoint. +// +// A local route is a route to a destination that is local to the stack. +func makeLocalRoute(netProto tcpip.NetworkProtocolNumber, localAddr, remoteAddr tcpip.Address, outgoingNIC, localAddressNIC *nic, localAddressEndpoint AssignableAddressEndpoint) *Route { + loop := PacketLoop + // Loopback interface loops back packets at the link endpoint level. We + // could remove this check if loopback interfaces looped back packets + // at the network layer. + if outgoingNIC.IsLoopback() { + loop = PacketOut + } + return makeRouteInner(netProto, localAddr, remoteAddr, outgoingNIC, localAddressNIC, localAddressEndpoint, loop, 0 /* mtu */) +} + +// RemoteLinkAddress returns the link-layer (MAC) address of the next hop in +// the route. +func (r *Route) RemoteLinkAddress() tcpip.LinkAddress { + r.mu.RLock() + defer r.mu.RUnlock() + return r.remoteLinkAddress +} + +// NICID returns the id of the NIC from which this route originates. +func (r *Route) NICID() tcpip.NICID { + return r.outgoingNIC.ID() +} + +// MaxHeaderLength forwards the call to the network endpoint's implementation. +func (r *Route) MaxHeaderLength() uint16 { + return r.outgoingNIC.getNetworkEndpoint(r.NetProto()).MaxHeaderLength() +} + +// Stats returns a mutable copy of current stats. +func (r *Route) Stats() tcpip.Stats { + return r.outgoingNIC.stack.Stats() +} + +// PseudoHeaderChecksum forwards the call to the network endpoint's +// implementation. +func (r *Route) PseudoHeaderChecksum(protocol tcpip.TransportProtocolNumber, totalLen uint16) uint16 { + return header.PseudoHeaderChecksum(protocol, r.LocalAddress(), r.RemoteAddress(), totalLen) +} + +// RequiresTXTransportChecksum returns false if the route does not require +// transport checksums to be populated. +func (r *Route) RequiresTXTransportChecksum() bool { + if r.local() { + return false + } + return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityTXChecksumOffload == 0 +} + +// HasGVisorGSOCapability returns true if the route supports gVisor GSO. +func (r *Route) HasGVisorGSOCapability() bool { + if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok { + return gso.SupportedGSO() == GVisorGSOSupported + } + return false +} + +// HasHostGSOCapability returns true if the route supports host GSO. +func (r *Route) HasHostGSOCapability() bool { + if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok { + return gso.SupportedGSO() == HostGSOSupported + } + return false +} + +// HasSaveRestoreCapability returns true if the route supports save/restore. +func (r *Route) HasSaveRestoreCapability() bool { + return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilitySaveRestore != 0 +} + +// HasDisconnectOkCapability returns true if the route supports disconnecting. +func (r *Route) HasDisconnectOkCapability() bool { + return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityDisconnectOk != 0 +} + +// GSOMaxSize returns the maximum GSO packet size. +func (r *Route) GSOMaxSize() uint32 { + if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok { + return gso.GSOMaxSize() + } + return 0 +} + +// ResolveWith immediately resolves a route with the specified remote link +// address. +func (r *Route) ResolveWith(addr tcpip.LinkAddress) { + r.mu.Lock() + defer r.mu.Unlock() + r.remoteLinkAddress = addr +} + +// ResolvedFieldsResult is the result of a route resolution attempt. +type ResolvedFieldsResult struct { + RouteInfo RouteInfo + Err tcpip.Error +} + +// ResolvedFields attempts to resolve the remote link address if it is not +// known. +// +// If a callback is provided, it will be called before ResolvedFields returns +// when address resolution is not required. If address resolution is required, +// the callback will be called once address resolution is complete, regardless +// of success or failure. +// +// Note, the route will not cache the remote link address when address +// resolution completes. +func (r *Route) ResolvedFields(afterResolve func(ResolvedFieldsResult)) tcpip.Error { + _, _, err := r.resolvedFields(afterResolve) + return err +} + +// resolvedFields is like ResolvedFields but also returns a notification channel +// when address resolution is required. This channel will become readable once +// address resolution is complete. +// +// The route's fields will also be returned, regardless of whether address +// resolution is required or not. +func (r *Route) resolvedFields(afterResolve func(ResolvedFieldsResult)) (RouteInfo, <-chan struct{}, tcpip.Error) { + r.mu.RLock() + fields := r.fieldsLocked() + resolutionRequired := r.isResolutionRequiredRLocked() + r.mu.RUnlock() + if !resolutionRequired { + if afterResolve != nil { + afterResolve(ResolvedFieldsResult{RouteInfo: fields, Err: nil}) + } + return fields, nil, nil + } + + // If specified, the local address used for link address resolution must be an + // address on the outgoing interface. + var linkAddressResolutionRequestLocalAddr tcpip.Address + if r.localAddressNIC == r.outgoingNIC { + linkAddressResolutionRequestLocalAddr = r.LocalAddress() + } + + nEntry := r.getCachedNeighborEntry() + if nEntry != nil { + if addr, ok := nEntry.getRemoteLinkAddress(); ok { + fields.RemoteLinkAddress = addr + if afterResolve != nil { + afterResolve(ResolvedFieldsResult{RouteInfo: fields, Err: nil}) + } + return fields, nil, nil + } + } + afterResolveFields := fields + entry, ch, err := r.linkRes.neigh.entry(r.nextHop(), linkAddressResolutionRequestLocalAddr, func(lrr LinkResolutionResult) { + if afterResolve != nil { + if lrr.Err == nil { + afterResolveFields.RemoteLinkAddress = lrr.LinkAddress + } + + afterResolve(ResolvedFieldsResult{RouteInfo: afterResolveFields, Err: lrr.Err}) + } + }) + if err == nil { + fields.RemoteLinkAddress, _ = entry.getRemoteLinkAddress() + } + r.setCachedNeighborEntry(entry) + return fields, ch, err +} + +func (r *Route) getCachedNeighborEntry() *neighborEntry { + r.mu.RLock() + defer r.mu.RUnlock() + return r.neighborEntry +} + +func (r *Route) setCachedNeighborEntry(entry *neighborEntry) { + r.mu.Lock() + defer r.mu.Unlock() + r.neighborEntry = entry +} + +func (r *Route) nextHop() tcpip.Address { + if r.NextHop().BitLen() == 0 { + return r.RemoteAddress() + } + return r.NextHop() +} + +// local returns true if the route is a local route. +func (r *Route) local() bool { + return r.Loop() == PacketLoop || r.outgoingNIC.IsLoopback() +} + +// IsResolutionRequired returns true if Resolve() must be called to resolve +// the link address before the route can be written to. +// +// The NICs the route is associated with must not be locked. +func (r *Route) IsResolutionRequired() bool { + r.mu.RLock() + defer r.mu.RUnlock() + return r.isResolutionRequiredRLocked() +} + +// +checklocksread:r.mu +func (r *Route) isResolutionRequiredRLocked() bool { + return len(r.remoteLinkAddress) == 0 && r.linkRes != nil && r.isValidForOutgoingRLocked() && !r.local() +} + +func (r *Route) isValidForOutgoing() bool { + r.mu.RLock() + defer r.mu.RUnlock() + return r.isValidForOutgoingRLocked() +} + +// +checklocksread:r.mu +func (r *Route) isValidForOutgoingRLocked() bool { + if !r.outgoingNIC.Enabled() { + return false + } + + localAddressEndpoint := r.localAddressEndpoint + if localAddressEndpoint == nil || !r.localAddressNIC.isValidForOutgoing(localAddressEndpoint) { + return false + } + + // If the source NIC and outgoing NIC are different, make sure the stack has + // forwarding enabled, or the packet will be handled locally. + if r.outgoingNIC != r.localAddressNIC && !isNICForwarding(r.localAddressNIC, r.NetProto()) && (!r.outgoingNIC.stack.handleLocal || !r.outgoingNIC.hasAddress(r.NetProto(), r.RemoteAddress())) { + return false + } + + return true +} + +// WritePacket writes the packet through the given route. +func (r *Route) WritePacket(params NetworkHeaderParams, pkt *PacketBuffer) tcpip.Error { + if !r.isValidForOutgoing() { + return &tcpip.ErrInvalidEndpointState{} + } + + return r.outgoingNIC.getNetworkEndpoint(r.NetProto()).WritePacket(r, params, pkt) +} + +// WriteHeaderIncludedPacket writes a packet already containing a network +// header through the given route. +func (r *Route) WriteHeaderIncludedPacket(pkt *PacketBuffer) tcpip.Error { + if !r.isValidForOutgoing() { + return &tcpip.ErrInvalidEndpointState{} + } + + return r.outgoingNIC.getNetworkEndpoint(r.NetProto()).WriteHeaderIncludedPacket(r, pkt) +} + +// DefaultTTL returns the default TTL of the underlying network endpoint. +func (r *Route) DefaultTTL() uint8 { + return r.outgoingNIC.getNetworkEndpoint(r.NetProto()).DefaultTTL() +} + +// MTU returns the MTU of the route if present, otherwise the MTU of the underlying network endpoint. +func (r *Route) MTU() uint32 { + if r.mtu > 0 { + return r.mtu + } + return r.outgoingNIC.getNetworkEndpoint(r.NetProto()).MTU() +} + +// Release decrements the reference counter of the resources associated with the +// route. +func (r *Route) Release() { + r.mu.Lock() + defer r.mu.Unlock() + + if ep := r.localAddressEndpoint; ep != nil { + ep.DecRef() + } +} + +// Acquire increments the reference counter of the resources associated with the +// route. +func (r *Route) Acquire() { + r.mu.RLock() + defer r.mu.RUnlock() + r.acquireLocked() +} + +// +checklocksread:r.mu +func (r *Route) acquireLocked() { + if ep := r.localAddressEndpoint; ep != nil { + if !ep.TryIncRef() { + panic(fmt.Sprintf("failed to increment reference count for local address endpoint = %s", r.LocalAddress())) + } + } +} + +// Stack returns the instance of the Stack that owns this route. +func (r *Route) Stack() *Stack { + return r.outgoingNIC.stack +} + +func (r *Route) isV4Broadcast(addr tcpip.Address) bool { + if addr == header.IPv4Broadcast { + return true + } + + r.mu.RLock() + localAddressEndpoint := r.localAddressEndpoint + r.mu.RUnlock() + if localAddressEndpoint == nil { + return false + } + + subnet := localAddressEndpoint.Subnet() + return subnet.IsBroadcast(addr) +} + +// IsOutboundBroadcast returns true if the route is for an outbound broadcast +// packet. +func (r *Route) IsOutboundBroadcast() bool { + // Only IPv4 has a notion of broadcast. + return r.isV4Broadcast(r.RemoteAddress()) +} + +// ConfirmReachable informs the network/link layer that the neighbour used for +// the route is reachable. +// +// "Reachable" is defined as having full-duplex communication between the +// local and remote ends of the route. +func (r *Route) ConfirmReachable() { + if entry := r.getCachedNeighborEntry(); entry != nil { + entry.handleUpperLevelConfirmation() + } +} diff --git a/pkg/tcpip/stack/route_export.go b/pkg/tcpip/stack/route_export.go new file mode 100644 index 0000000..7e0fc5d --- /dev/null +++ b/pkg/tcpip/stack/route_export.go @@ -0,0 +1,14 @@ +package stack + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" +) + +type exportedEndpoint interface { + WritePacketDirect(r *Route, pkt *PacketBuffer) tcpip.Error +} + +func (r *Route) WritePacketDirect(pkt *PacketBuffer) tcpip.Error { + rawEndpoint := r.outgoingNIC.getNetworkEndpoint(r.NetProto()).(exportedEndpoint) + return rawEndpoint.WritePacketDirect(r, pkt) +} diff --git a/pkg/tcpip/stack/route_mutex.go b/pkg/tcpip/stack/route_mutex.go new file mode 100644 index 0000000..ca06e5c --- /dev/null +++ b/pkg/tcpip/stack/route_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type routeRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var routelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type routelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *routeRWMutex) Lock() { + locking.AddGLock(routeprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *routeRWMutex) NestedLock(i routelockNameIndex) { + locking.AddGLock(routeprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *routeRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(routeprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *routeRWMutex) NestedUnlock(i routelockNameIndex) { + m.mu.Unlock() + locking.DelGLock(routeprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *routeRWMutex) RLock() { + locking.AddGLock(routeprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *routeRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(routeprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *routeRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *routeRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *routeRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var routeprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func routeinitLockNames() {} + +func init() { + routeinitLockNames() + routeprefixIndex = locking.NewMutexClass(reflect.TypeOf(routeRWMutex{}), routelockNames) +} diff --git a/pkg/tcpip/stack/route_stack_mutex.go b/pkg/tcpip/stack/route_stack_mutex.go new file mode 100644 index 0000000..bfa849f --- /dev/null +++ b/pkg/tcpip/stack/route_stack_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type routeStackRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var routeStacklockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type routeStacklockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *routeStackRWMutex) Lock() { + locking.AddGLock(routeStackprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *routeStackRWMutex) NestedLock(i routeStacklockNameIndex) { + locking.AddGLock(routeStackprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *routeStackRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(routeStackprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *routeStackRWMutex) NestedUnlock(i routeStacklockNameIndex) { + m.mu.Unlock() + locking.DelGLock(routeStackprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *routeStackRWMutex) RLock() { + locking.AddGLock(routeStackprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *routeStackRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(routeStackprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *routeStackRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *routeStackRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *routeStackRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var routeStackprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func routeStackinitLockNames() {} + +func init() { + routeStackinitLockNames() + routeStackprefixIndex = locking.NewMutexClass(reflect.TypeOf(routeStackRWMutex{}), routeStacklockNames) +} diff --git a/pkg/tcpip/stack/save_restore.go b/pkg/tcpip/stack/save_restore.go new file mode 100644 index 0000000..5b38acd --- /dev/null +++ b/pkg/tcpip/stack/save_restore.go @@ -0,0 +1,29 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "context" + "math/rand" + "time" + + cryptorand "github.com/sagernet/gvisor/pkg/rand" +) + +// afterLoad is invoked by stateify. +func (s *Stack) afterLoad(context.Context) { + s.insecureRNG = rand.New(rand.NewSource(time.Now().UnixNano())) + s.secureRNG = cryptorand.RNGFrom(cryptorand.Reader) +} diff --git a/pkg/tcpip/stack/stack.go b/pkg/tcpip/stack/stack.go new file mode 100644 index 0000000..0d1d738 --- /dev/null +++ b/pkg/tcpip/stack/stack.go @@ -0,0 +1,2508 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package stack provides the glue between networking protocols and the +// consumers of the networking stack. +// +// For consumers, the only function of interest is New(), everything else is +// provided by the tcpip/public package. +package stack + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "math/rand" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/log" + cryptorand "github.com/sagernet/gvisor/pkg/rand" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" + "github.com/sagernet/gvisor/pkg/waiter" + "golang.org/x/time/rate" +) + +const ( + // DefaultTOS is the default type of service value for network endpoints. + DefaultTOS = 0 +) + +// +stateify savable +type transportProtocolState struct { + proto TransportProtocol + defaultHandler func(id TransportEndpointID, pkt *PacketBuffer) bool `state:"nosave"` +} + +// RestoredEndpoint is an endpoint that needs to be restored. +type RestoredEndpoint interface { + // Restore restores an endpoint. This can be used to restart background + // workers such as protocol goroutines. This must be called after all + // indirect dependencies of the endpoint has been restored, which + // generally implies at the end of the restore process. + Restore(*Stack) +} + +// ResumableEndpoint is an endpoint that needs to be resumed after save. +type ResumableEndpoint interface { + // Resume resumes an endpoint. + Resume() +} + +var netRawMissingLogger = log.BasicRateLimitedLogger(time.Minute) + +// Stack is a networking stack, with all supported protocols, NICs, and route +// table. +// +// LOCK ORDERING: mu > routeMu. +// +// +stateify savable +type Stack struct { + transportProtocols map[tcpip.TransportProtocolNumber]*transportProtocolState + networkProtocols map[tcpip.NetworkProtocolNumber]NetworkProtocol + + // rawFactory creates raw endpoints. If nil, raw endpoints are + // disabled. It is set during Stack creation and is immutable. + rawFactory RawFactory + packetEndpointWriteSupported bool + + demux *transportDemuxer + + stats tcpip.Stats + + // routeMu protects annotated fields below. + routeMu routeStackRWMutex `state:"nosave"` + + // routeTable is a list of routes sorted by prefix length, longest (most specific) first. + // +checklocks:routeMu + routeTable tcpip.RouteList `state:"nosave"` + + mu stackRWMutex `state:"nosave"` + // +checklocks:mu + nics map[tcpip.NICID]*nic `state:"nosave"` + // +checklocks:mu + loopbackNIC *nic + // +checklocks:mu + defaultForwardingEnabled map[tcpip.NetworkProtocolNumber]struct{} + + // nicIDGen is used to generate NIC IDs. + nicIDGen atomicbitops.Int32 `state:"nosave"` + + // cleanupEndpointsMu protects cleanupEndpoints. + cleanupEndpointsMu cleanupEndpointsMutex `state:"nosave"` + // +checklocks:cleanupEndpointsMu + cleanupEndpoints map[TransportEndpoint]struct{} + + *ports.PortManager + + // clock is used to generate user-visible times. + clock tcpip.Clock + + // handleLocal allows non-loopback interfaces to loop packets. + handleLocal bool + + // tables are the iptables packet filtering and manipulation rules. + // TODO(gvisor.dev/issue/4595): S/R this field. + tables *IPTables `state:"nosave"` + + // nftables is the nftables interface for packet filtering and manipulation rules. + nftables NFTablesInterface `state:"nosave"` + + // restoredEndpoints is a list of endpoints that need to be restored if the + // stack is being restored. + restoredEndpoints []RestoredEndpoint + + // resumableEndpoints is a list of endpoints that need to be resumed + // after save. + resumableEndpoints []ResumableEndpoint + + // icmpRateLimiter is a global rate limiter for all ICMP messages generated + // by the stack. + icmpRateLimiter *ICMPRateLimiter + + // seed is a one-time random value initialized at stack startup. + // + // TODO(gvisor.dev/issue/940): S/R this field. + seed uint32 + + // nudConfigs is the default NUD configurations used by interfaces. + nudConfigs NUDConfigurations + + // nudDisp is the NUD event dispatcher that is used to send the netstack + // integrator NUD related events. + nudDisp NUDDispatcher + + // randomGenerator is an injectable pseudo random generator that can be + // used when a random number is required. It must not be used in + // security-sensitive contexts. + insecureRNG *rand.Rand `state:"nosave"` + + // secureRNG is a cryptographically secure random number generator. + secureRNG cryptorand.RNG `state:"nosave"` + + // sendBufferSize holds the min/default/max send buffer sizes for + // endpoints other than TCP. + sendBufferSize tcpip.SendBufferSizeOption + + // receiveBufferSize holds the min/default/max receive buffer sizes for + // endpoints other than TCP. + receiveBufferSize tcpip.ReceiveBufferSizeOption + + // tcpInvalidRateLimit is the maximal rate for sending duplicate + // acknowledgements in response to incoming TCP packets that are for an existing + // connection but that are invalid due to any of the following reasons: + // + // a) out-of-window sequence number. + // b) out-of-window acknowledgement number. + // c) PAWS check failure (when implemented). + // + // This is required to prevent potential ACK loops. + // Setting this to 0 will disable all rate limiting. + tcpInvalidRateLimit time.Duration + + // tsOffsetSecret is the secret key for generating timestamp offsets + // initialized at stack startup. + tsOffsetSecret uint32 + + // saveRestoreEnabled indicates whether the stack is saved and restored. + saveRestoreEnabled bool +} + +// NetworkProtocolFactory instantiates a network protocol. +// +// NetworkProtocolFactory must not attempt to modify the stack, it may only +// query the stack. +type NetworkProtocolFactory func(*Stack) NetworkProtocol + +// TransportProtocolFactory instantiates a transport protocol. +// +// TransportProtocolFactory must not attempt to modify the stack, it may only +// query the stack. +type TransportProtocolFactory func(*Stack) TransportProtocol + +// Options contains optional Stack configuration. +type Options struct { + // NetworkProtocols lists the network protocols to enable. + NetworkProtocols []NetworkProtocolFactory + + // TransportProtocols lists the transport protocols to enable. + TransportProtocols []TransportProtocolFactory + + // Clock is an optional clock used for timekeeping. + // + // If Clock is nil, tcpip.NewStdClock() will be used. + Clock tcpip.Clock + + // Stats are optional statistic counters. + Stats tcpip.Stats + + // HandleLocal indicates whether packets destined to their source + // should be handled by the stack internally (true) or outside the + // stack (false). + HandleLocal bool + + // NUDConfigs is the default NUD configurations used by interfaces. + NUDConfigs NUDConfigurations + + // NUDDisp is the NUD event dispatcher that an integrator can provide to + // receive NUD related events. + NUDDisp NUDDispatcher + + // RawFactory produces raw endpoints. Raw endpoints are enabled only if + // this is non-nil. + RawFactory RawFactory + + // AllowPacketEndpointWrite determines if packet endpoints support write + // operations. + AllowPacketEndpointWrite bool + + // RandSource is an optional source to use to generate random + // numbers. If omitted it defaults to a Source seeded by the data + // returned by the stack secure RNG. + // + // RandSource must be thread-safe. + RandSource rand.Source + + // IPTables are the initial iptables rules. If nil, DefaultIPTables will be + // used to construct the initial iptables rules. + // all traffic. + IPTables *IPTables + + // NFTables is the nftables interface for packet filtering and manipulation rules. + NFTables NFTablesInterface + + // DefaultIPTables is an optional iptables rules constructor that is called + // if IPTables is nil. If both fields are nil, iptables will allow all + // traffic. + DefaultIPTables func(clock tcpip.Clock, rand *rand.Rand) *IPTables + + // SecureRNG is a cryptographically secure random number generator. + SecureRNG io.Reader +} + +// TransportEndpointInfo holds useful information about a transport endpoint +// which can be queried by monitoring tools. +// +// +stateify savable +type TransportEndpointInfo struct { + // The following fields are initialized at creation time and are + // immutable. + + NetProto tcpip.NetworkProtocolNumber + TransProto tcpip.TransportProtocolNumber + + // The following fields are protected by endpoint mu. + + ID TransportEndpointID + // BindNICID and bindAddr are set via calls to Bind(). They are used to + // reject attempts to send data or connect via a different NIC or + // address + BindNICID tcpip.NICID + BindAddr tcpip.Address + // RegisterNICID is the default NICID registered as a side-effect of + // connect or datagram write. + RegisterNICID tcpip.NICID +} + +// AddrNetProtoLocked unwraps the specified address if it is a V4-mapped V6 +// address and returns the network protocol number to be used to communicate +// with the specified address. It returns an error if the passed address is +// incompatible with the receiver. +// +// Preconditon: the parent endpoint mu must be held while calling this method. +func (t *TransportEndpointInfo) AddrNetProtoLocked(addr tcpip.FullAddress, v6only bool, bind bool) (tcpip.FullAddress, tcpip.NetworkProtocolNumber, tcpip.Error) { + netProto := t.NetProto + switch addr.Addr.BitLen() { + case header.IPv4AddressSizeBits: + netProto = header.IPv4ProtocolNumber + case header.IPv6AddressSizeBits: + if header.IsV4MappedAddress(addr.Addr) { + netProto = header.IPv4ProtocolNumber + addr.Addr = tcpip.AddrFrom4Slice(addr.Addr.AsSlice()[header.IPv6AddressSize-header.IPv4AddressSize:]) + if addr.Addr == header.IPv4Any { + addr.Addr = tcpip.Address{} + } + } + } + + switch t.ID.LocalAddress.BitLen() { + case header.IPv4AddressSizeBits: + if addr.Addr.BitLen() == header.IPv6AddressSizeBits { + return tcpip.FullAddress{}, 0, &tcpip.ErrInvalidEndpointState{} + } + case header.IPv6AddressSizeBits: + if addr.Addr.BitLen() == header.IPv4AddressSizeBits { + return tcpip.FullAddress{}, 0, &tcpip.ErrNetworkUnreachable{} + } + } + + if !bind && addr.Addr.Unspecified() { + // If the destination address isn't set, Linux sets it to the + // source address. If a source address isn't set either, it + // sets both to the loopback address. + if t.ID.LocalAddress.Unspecified() { + switch netProto { + case header.IPv4ProtocolNumber: + addr.Addr = header.IPv4Loopback + case header.IPv6ProtocolNumber: + addr.Addr = header.IPv6Loopback + } + } else { + addr.Addr = t.ID.LocalAddress + } + } + + switch { + case netProto == t.NetProto: + case netProto == header.IPv4ProtocolNumber && t.NetProto == header.IPv6ProtocolNumber: + if v6only { + return tcpip.FullAddress{}, 0, &tcpip.ErrHostUnreachable{} + } + default: + return tcpip.FullAddress{}, 0, &tcpip.ErrInvalidEndpointState{} + } + + return addr, netProto, nil +} + +// IsEndpointInfo is an empty method to implement the tcpip.EndpointInfo +// marker interface. +func (*TransportEndpointInfo) IsEndpointInfo() {} + +// New allocates a new networking stack with only the requested networking and +// transport protocols configured with default options. +// +// Note, NDPConfigurations will be fixed before being used by the Stack. That +// is, if an invalid value was provided, it will be reset to the default value. +// +// Protocol options can be changed by calling the +// SetNetworkProtocolOption/SetTransportProtocolOption methods provided by the +// stack. Please refer to individual protocol implementations as to what options +// are supported. +func New(opts Options) *Stack { + clock := opts.Clock + if clock == nil { + clock = tcpip.NewStdClock() + } + + if opts.SecureRNG == nil { + opts.SecureRNG = cryptorand.Reader + } + secureRNG := cryptorand.RNGFrom(opts.SecureRNG) + + randSrc := opts.RandSource + if randSrc == nil { + var v int64 + if err := binary.Read(opts.SecureRNG, binary.LittleEndian, &v); err != nil { + panic(err) + } + // Source provided by rand.NewSource is not thread-safe so + // we wrap it in a simple thread-safe version. + randSrc = &lockedRandomSource{src: rand.NewSource(v)} + } + insecureRNG := rand.New(randSrc) + + if opts.IPTables == nil { + if opts.DefaultIPTables == nil { + opts.DefaultIPTables = DefaultTables + } + opts.IPTables = opts.DefaultIPTables(clock, insecureRNG) + } + + opts.NUDConfigs.resetInvalidFields() + + s := &Stack{ + transportProtocols: make(map[tcpip.TransportProtocolNumber]*transportProtocolState), + networkProtocols: make(map[tcpip.NetworkProtocolNumber]NetworkProtocol), + nics: make(map[tcpip.NICID]*nic), + packetEndpointWriteSupported: opts.AllowPacketEndpointWrite, + defaultForwardingEnabled: make(map[tcpip.NetworkProtocolNumber]struct{}), + cleanupEndpoints: make(map[TransportEndpoint]struct{}), + PortManager: ports.NewPortManager(), + clock: clock, + stats: opts.Stats.FillIn(), + handleLocal: opts.HandleLocal, + tables: opts.IPTables, + nftables: opts.NFTables, + icmpRateLimiter: NewICMPRateLimiter(clock), + seed: secureRNG.Uint32(), + nudConfigs: opts.NUDConfigs, + nudDisp: opts.NUDDisp, + insecureRNG: insecureRNG, + secureRNG: secureRNG, + sendBufferSize: tcpip.SendBufferSizeOption{ + Min: MinBufferSize, + Default: DefaultBufferSize, + Max: DefaultMaxBufferSize, + }, + receiveBufferSize: tcpip.ReceiveBufferSizeOption{ + Min: MinBufferSize, + Default: DefaultBufferSize, + Max: DefaultMaxBufferSize, + }, + tcpInvalidRateLimit: defaultTCPInvalidRateLimit, + tsOffsetSecret: secureRNG.Uint32(), + } + + // Add specified network protocols. + for _, netProtoFactory := range opts.NetworkProtocols { + netProto := netProtoFactory(s) + s.networkProtocols[netProto.Number()] = netProto + } + + // Add specified transport protocols. + for _, transProtoFactory := range opts.TransportProtocols { + transProto := transProtoFactory(s) + s.transportProtocols[transProto.Number()] = &transportProtocolState{ + proto: transProto, + } + } + + // Add the factory for raw endpoints, if present. + s.rawFactory = opts.RawFactory + + // Create the global transport demuxer. + s.demux = newTransportDemuxer(s) + + return s +} + +// NextNICID allocates the next available NIC ID and returns it. +func (s *Stack) NextNICID() tcpip.NICID { + next := s.nicIDGen.Add(1) + if next < 0 { + panic("NICID overflow") + } + return tcpip.NICID(next) +} + +// SetNetworkProtocolOption allows configuring individual protocol level +// options. This method returns an error if the protocol is not supported or +// option is not supported by the protocol implementation or the provided value +// is incorrect. +func (s *Stack) SetNetworkProtocolOption(network tcpip.NetworkProtocolNumber, option tcpip.SettableNetworkProtocolOption) tcpip.Error { + netProto, ok := s.networkProtocols[network] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + return netProto.SetOption(option) +} + +// NetworkProtocolOption allows retrieving individual protocol level option +// values. This method returns an error if the protocol is not supported or +// option is not supported by the protocol implementation. E.g.: +// +// var v ipv4.MyOption +// err := s.NetworkProtocolOption(tcpip.IPv4ProtocolNumber, &v) +// if err != nil { +// ... +// } +func (s *Stack) NetworkProtocolOption(network tcpip.NetworkProtocolNumber, option tcpip.GettableNetworkProtocolOption) tcpip.Error { + netProto, ok := s.networkProtocols[network] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + return netProto.Option(option) +} + +// SetTransportProtocolOption allows configuring individual protocol level +// options. This method returns an error if the protocol is not supported or +// option is not supported by the protocol implementation or the provided value +// is incorrect. +func (s *Stack) SetTransportProtocolOption(transport tcpip.TransportProtocolNumber, option tcpip.SettableTransportProtocolOption) tcpip.Error { + transProtoState, ok := s.transportProtocols[transport] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + return transProtoState.proto.SetOption(option) +} + +// TransportProtocolOption allows retrieving individual protocol level option +// values. This method returns an error if the protocol is not supported or +// option is not supported by the protocol implementation. +// +// var v tcp.SACKEnabled +// if err := s.TransportProtocolOption(tcpip.TCPProtocolNumber, &v); err != nil { +// ... +// } +func (s *Stack) TransportProtocolOption(transport tcpip.TransportProtocolNumber, option tcpip.GettableTransportProtocolOption) tcpip.Error { + transProtoState, ok := s.transportProtocols[transport] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + return transProtoState.proto.Option(option) +} + +// SendBufSizeProto is a protocol that can return its send buffer size. +type SendBufSizeProto interface { + SendBufferSize() tcpip.TCPSendBufferSizeRangeOption +} + +// TCPSendBufferLimits returns the TCP send buffer size limit. +func (s *Stack) TCPSendBufferLimits() tcpip.TCPSendBufferSizeRangeOption { + return s.transportProtocols[header.TCPProtocolNumber].proto.(SendBufSizeProto).SendBufferSize() +} + +// SetTransportProtocolHandler sets the per-stack default handler for the given +// protocol. +// +// It must be called only during initialization of the stack. Changing it as the +// stack is operating is not supported. +func (s *Stack) SetTransportProtocolHandler(p tcpip.TransportProtocolNumber, h func(TransportEndpointID, *PacketBuffer) bool) { + state := s.transportProtocols[p] + if state != nil { + state.defaultHandler = h + } +} + +// Clock returns the Stack's clock for retrieving the current time and +// scheduling work. +func (s *Stack) Clock() tcpip.Clock { + return s.clock +} + +// Stats returns a mutable copy of the current stats. +// +// This is not generally exported via the public interface, but is available +// internally. +func (s *Stack) Stats() tcpip.Stats { + return s.stats +} + +// SetNICForwarding enables or disables packet forwarding on the specified NIC +// for the passed protocol. +// +// Returns the previous configuration on the NIC. +func (s *Stack) SetNICForwarding(id tcpip.NICID, protocol tcpip.NetworkProtocolNumber, enable bool) (bool, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return false, &tcpip.ErrUnknownNICID{} + } + + return nic.setForwarding(protocol, enable) +} + +// NICForwarding returns the forwarding configuration for the specified NIC. +func (s *Stack) NICForwarding(id tcpip.NICID, protocol tcpip.NetworkProtocolNumber) (bool, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return false, &tcpip.ErrUnknownNICID{} + } + + return nic.forwarding(protocol) +} + +// SetForwardingDefaultAndAllNICs sets packet forwarding for all NICs for the +// passed protocol and sets the default setting for newly created NICs. +func (s *Stack) SetForwardingDefaultAndAllNICs(protocol tcpip.NetworkProtocolNumber, enable bool) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + doneOnce := false + for id, nic := range s.nics { + if _, err := nic.setForwarding(protocol, enable); err != nil { + // Expect forwarding to be settable on all interfaces if it was set on + // one. + if doneOnce { + panic(fmt.Sprintf("nic(id=%d).setForwarding(%d, %t): %s", id, protocol, enable, err)) + } + + return err + } + + doneOnce = true + } + + if enable { + s.defaultForwardingEnabled[protocol] = struct{}{} + } else { + delete(s.defaultForwardingEnabled, protocol) + } + + return nil +} + +// AddMulticastRoute adds a multicast route to be used for the specified +// addresses and protocol. +func (s *Stack) AddMulticastRoute(protocol tcpip.NetworkProtocolNumber, addresses UnicastSourceAndMulticastDestination, route MulticastRoute) tcpip.Error { + netProto, ok := s.networkProtocols[protocol] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + + forwardingNetProto, ok := netProto.(MulticastForwardingNetworkProtocol) + if !ok { + return &tcpip.ErrNotSupported{} + } + + return forwardingNetProto.AddMulticastRoute(addresses, route) +} + +// RemoveMulticastRoute removes a multicast route that matches the specified +// addresses and protocol. +func (s *Stack) RemoveMulticastRoute(protocol tcpip.NetworkProtocolNumber, addresses UnicastSourceAndMulticastDestination) tcpip.Error { + netProto, ok := s.networkProtocols[protocol] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + + forwardingNetProto, ok := netProto.(MulticastForwardingNetworkProtocol) + if !ok { + return &tcpip.ErrNotSupported{} + } + + return forwardingNetProto.RemoveMulticastRoute(addresses) +} + +// MulticastRouteLastUsedTime returns a monotonic timestamp that represents the +// last time that the route that matches the provided addresses and protocol +// was used or updated. +func (s *Stack) MulticastRouteLastUsedTime(protocol tcpip.NetworkProtocolNumber, addresses UnicastSourceAndMulticastDestination) (tcpip.MonotonicTime, tcpip.Error) { + netProto, ok := s.networkProtocols[protocol] + if !ok { + return tcpip.MonotonicTime{}, &tcpip.ErrUnknownProtocol{} + } + + forwardingNetProto, ok := netProto.(MulticastForwardingNetworkProtocol) + if !ok { + return tcpip.MonotonicTime{}, &tcpip.ErrNotSupported{} + } + + return forwardingNetProto.MulticastRouteLastUsedTime(addresses) +} + +// EnableMulticastForwardingForProtocol enables multicast forwarding for the +// provided protocol. +// +// Returns true if forwarding was already enabled on the protocol. +// Additionally, returns an error if: +// +// - The protocol is not found. +// - The protocol doesn't support multicast forwarding. +// - The multicast forwarding event dispatcher is nil. +// +// If successful, future multicast forwarding events will be sent to the +// provided event dispatcher. +func (s *Stack) EnableMulticastForwardingForProtocol(protocol tcpip.NetworkProtocolNumber, disp MulticastForwardingEventDispatcher) (bool, tcpip.Error) { + netProto, ok := s.networkProtocols[protocol] + if !ok { + return false, &tcpip.ErrUnknownProtocol{} + } + + forwardingNetProto, ok := netProto.(MulticastForwardingNetworkProtocol) + if !ok { + return false, &tcpip.ErrNotSupported{} + } + + return forwardingNetProto.EnableMulticastForwarding(disp) +} + +// DisableMulticastForwardingForProtocol disables multicast forwarding for the +// provided protocol. +// +// Returns an error if the provided protocol is not found or if it does not +// support multicast forwarding. +func (s *Stack) DisableMulticastForwardingForProtocol(protocol tcpip.NetworkProtocolNumber) tcpip.Error { + netProto, ok := s.networkProtocols[protocol] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + + forwardingNetProto, ok := netProto.(MulticastForwardingNetworkProtocol) + if !ok { + return &tcpip.ErrNotSupported{} + } + + forwardingNetProto.DisableMulticastForwarding() + return nil +} + +// SetNICMulticastForwarding enables or disables multicast packet forwarding on +// the specified NIC for the passed protocol. +// +// Returns the previous configuration on the NIC. +func (s *Stack) SetNICMulticastForwarding(id tcpip.NICID, protocol tcpip.NetworkProtocolNumber, enable bool) (bool, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return false, &tcpip.ErrUnknownNICID{} + } + + return nic.setMulticastForwarding(protocol, enable) +} + +// NICMulticastForwarding returns the multicast forwarding configuration for +// the specified NIC. +func (s *Stack) NICMulticastForwarding(id tcpip.NICID, protocol tcpip.NetworkProtocolNumber) (bool, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return false, &tcpip.ErrUnknownNICID{} + } + + return nic.multicastForwarding(protocol) +} + +// PortRange returns the UDP and TCP inclusive range of ephemeral ports used in +// both IPv4 and IPv6. +func (s *Stack) PortRange() (uint16, uint16) { + return s.PortManager.PortRange() +} + +// SetPortRange sets the UDP and TCP IPv4 and IPv6 ephemeral port range +// (inclusive). +func (s *Stack) SetPortRange(start uint16, end uint16) tcpip.Error { + return s.PortManager.SetPortRange(start, end) +} + +// SetRouteTable assigns the route table to be used by this stack. It +// specifies which NIC to use for given destination address ranges. +// +// This method takes ownership of the table. +func (s *Stack) SetRouteTable(table []tcpip.Route) { + s.routeMu.Lock() + defer s.routeMu.Unlock() + s.routeTable.Reset() + for _, r := range table { + s.addRouteLocked(&r) + } +} + +// GetRouteTable returns the route table which is currently in use. +func (s *Stack) GetRouteTable() []tcpip.Route { + s.routeMu.RLock() + defer s.routeMu.RUnlock() + table := make([]tcpip.Route, 0) + for r := s.routeTable.Front(); r != nil; r = r.Next() { + table = append(table, *r) + } + return table +} + +// AddRoute appends a route to the route table. +func (s *Stack) AddRoute(route tcpip.Route) { + s.routeMu.Lock() + defer s.routeMu.Unlock() + s.addRouteLocked(&route) +} + +// +checklocks:s.routeMu +func (s *Stack) addRouteLocked(route *tcpip.Route) { + routePrefix := route.Destination.Prefix() + n := s.routeTable.Front() + for ; n != nil; n = n.Next() { + if n.Destination.Prefix() < routePrefix { + s.routeTable.InsertBefore(n, route) + return + } + } + s.routeTable.PushBack(route) +} + +// RemoveRoutes removes matching routes from the route table, it +// returns the number of routes that are removed. +func (s *Stack) RemoveRoutes(match func(tcpip.Route) bool) int { + s.routeMu.Lock() + defer s.routeMu.Unlock() + + return s.removeRoutesLocked(match) +} + +// +checklocks:s.routeMu +func (s *Stack) removeRoutesLocked(match func(tcpip.Route) bool) int { + count := 0 + for route := s.routeTable.Front(); route != nil; { + next := route.Next() + if match(*route) { + s.routeTable.Remove(route) + count++ + } + route = next + } + return count +} + +// ReplaceRoute replaces the route in the routing table which matches +// the lookup key for the routing table. If there is no match, the given +// route will still be added to the routing table. +// The lookup key consists of destination, ToS, scope and output interface. +func (s *Stack) ReplaceRoute(route tcpip.Route) { + s.routeMu.Lock() + defer s.routeMu.Unlock() + + s.removeRoutesLocked(func(rt tcpip.Route) bool { + return rt.Equal(route) + }) + s.addRouteLocked(&route) +} + +// NewEndpoint creates a new transport layer endpoint of the given protocol. +func (s *Stack) NewEndpoint(transport tcpip.TransportProtocolNumber, network tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + t, ok := s.transportProtocols[transport] + if !ok { + return nil, &tcpip.ErrUnknownProtocol{} + } + + return t.proto.NewEndpoint(network, waiterQueue) +} + +// NewRawEndpoint creates a new raw transport layer endpoint of the given +// protocol. Raw endpoints receive all traffic for a given protocol regardless +// of address. +func (s *Stack) NewRawEndpoint(transport tcpip.TransportProtocolNumber, network tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue, associated bool) (tcpip.Endpoint, tcpip.Error) { + if s.rawFactory == nil { + netRawMissingLogger.Infof("A process tried to create a raw socket, but --net-raw was not specified. Should runsc be run with --net-raw?") + return nil, &tcpip.ErrNotPermitted{} + } + + if !associated { + return s.rawFactory.NewUnassociatedEndpoint(s, network, transport, waiterQueue) + } + + t, ok := s.transportProtocols[transport] + if !ok { + return nil, &tcpip.ErrUnknownProtocol{} + } + + return t.proto.NewRawEndpoint(network, waiterQueue) +} + +// NewPacketEndpoint creates a new packet endpoint listening for the given +// netProto. +func (s *Stack) NewPacketEndpoint(cooked bool, netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + if s.rawFactory == nil { + return nil, &tcpip.ErrNotPermitted{} + } + + return s.rawFactory.NewPacketEndpoint(s, cooked, netProto, waiterQueue) +} + +// NICContext is an opaque pointer used to store client-supplied NIC metadata. +type NICContext any + +// NICOptions specifies the configuration of a NIC as it is being created. +// The zero value creates an enabled, unnamed NIC. +type NICOptions struct { + // Name specifies the name of the NIC. + Name string + + // Disabled specifies whether to avoid calling Attach on the passed + // LinkEndpoint. + Disabled bool + + // Context specifies user-defined data that will be returned in stack.NICInfo + // for the NIC. Clients of this library can use it to add metadata that + // should be tracked alongside a NIC, to avoid having to keep a + // map[tcpip.NICID]metadata mirroring stack.Stack's nic map. + Context NICContext + + // QDisc is the queue discipline to use for this NIC. + QDisc QueueingDiscipline + + // DeliverLinkPackets specifies whether the NIC is responsible for + // delivering raw packets to packet sockets. + DeliverLinkPackets bool + + // EnableExperimentIPOption specifies whether the NIC is responsible for + // passing the experiment IP option. + EnableExperimentIPOption bool +} + +// GetNICByID return a network device associated with the specified ID. +func (s *Stack) GetNICByID(id tcpip.NICID) (*nic, tcpip.Error) { + s.mu.Lock() + defer s.mu.Unlock() + + n, ok := s.nics[id] + if !ok { + return nil, &tcpip.ErrNoSuchFile{} + } + return n, nil +} + +// CreateNICWithOptions creates a NIC with the provided id, LinkEndpoint, and +// NICOptions. See the documentation on type NICOptions for details on how +// NICs can be configured. +// +// LinkEndpoint.Attach will be called to bind ep with a NetworkDispatcher. +func (s *Stack) CreateNICWithOptions(id tcpip.NICID, ep LinkEndpoint, opts NICOptions) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + if id == 0 { + return &tcpip.ErrInvalidNICID{} + } + // Make sure id is unique. + if _, ok := s.nics[id]; ok { + return &tcpip.ErrDuplicateNICID{} + } + + // Make sure name is unique, unless unnamed. + if opts.Name != "" { + for _, n := range s.nics { + if n.Name() == opts.Name { + return &tcpip.ErrDuplicateNICID{} + } + } + } + + n := newNIC(s, id, ep, opts) + for proto := range s.defaultForwardingEnabled { + if _, err := n.setForwarding(proto, true); err != nil { + panic(fmt.Sprintf("newNIC(%d, ...).setForwarding(%d, true): %s", id, proto, err)) + } + } + s.nics[id] = n + if n.IsLoopback() { + s.loopbackNIC = n + } + ep.SetOnCloseAction(func() { + s.RemoveNIC(id) + }) + if !opts.Disabled { + return n.enable() + } + + return nil +} + +// CreateNIC creates a NIC with the provided id and LinkEndpoint and calls +// LinkEndpoint.Attach to bind ep with a NetworkDispatcher. +func (s *Stack) CreateNIC(id tcpip.NICID, ep LinkEndpoint) tcpip.Error { + return s.CreateNICWithOptions(id, ep, NICOptions{}) +} + +// GetLinkEndpointByName gets the link endpoint specified by name. +func (s *Stack) GetLinkEndpointByName(name string) LinkEndpoint { + s.mu.RLock() + defer s.mu.RUnlock() + for _, nic := range s.nics { + if nic.Name() == name { + linkEP, ok := nic.NetworkLinkEndpoint.(LinkEndpoint) + if !ok { + panic(fmt.Sprintf("unexpected NetworkLinkEndpoint(%#v) is not a LinkEndpoint", nic.NetworkLinkEndpoint)) + } + return linkEP + } + } + return nil +} + +// EnableNIC enables the given NIC so that the link-layer endpoint can start +// delivering packets to it. +func (s *Stack) EnableNIC(id tcpip.NICID) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.enable() +} + +// DisableNIC disables the given NIC. +func (s *Stack) DisableNIC(id tcpip.NICID) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + nic.disable() + return nil +} + +// CheckNIC checks if a NIC is usable. +func (s *Stack) CheckNIC(id tcpip.NICID) bool { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return false + } + + return nic.Enabled() +} + +// RemoveNIC removes NIC and all related routes from the network stack. +func (s *Stack) RemoveNIC(id tcpip.NICID) tcpip.Error { + s.mu.Lock() + deferAct, err := s.removeNICLocked(id) + s.mu.Unlock() + if deferAct != nil { + deferAct() + } + return err +} + +// removeNICLocked removes NIC and all related routes from the network stack. +// +// +checklocks:s.mu +func (s *Stack) removeNICLocked(id tcpip.NICID) (func(), tcpip.Error) { + nic, ok := s.nics[id] + if !ok { + return nil, &tcpip.ErrUnknownNICID{} + } + delete(s.nics, id) + + if nic.Primary != nil { + b := nic.Primary.NetworkLinkEndpoint.(CoordinatorNIC) + if err := b.DelNIC(nic); err != nil { + return nil, err + } + } + + // Remove routes in-place. n tracks the number of routes written. + s.routeMu.Lock() + for r := s.routeTable.Front(); r != nil; { + next := r.Next() + if r.NIC == id { + s.routeTable.Remove(r) + } + r = next + } + s.routeMu.Unlock() + + if s.loopbackNIC == nic { + s.loopbackNIC = nil + } + return nic.remove(true /* closeLinkEndpoint */) +} + +// SetNICCoordinator sets a coordinator device. +func (s *Stack) SetNICCoordinator(id tcpip.NICID, mid tcpip.NICID) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + // Setting a coordinator for a coordinator NIC is not allowed. + if _, ok := nic.NetworkLinkEndpoint.(CoordinatorNIC); ok { + return &tcpip.ErrNoSuchFile{} + } + m, ok := s.nics[mid] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + b, ok := m.NetworkLinkEndpoint.(CoordinatorNIC) + if !ok { + return &tcpip.ErrNotSupported{} + } + if err := b.AddNIC(nic); err != nil { + return err + } + nic.Primary = m + return nil +} + +// SetNICAddress sets the hardware address which is identified by the nic ID. +func (s *Stack) SetNICAddress(id tcpip.NICID, addr tcpip.LinkAddress) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + nic.NetworkLinkEndpoint.SetLinkAddress(addr) + return nil +} + +// SetNICName sets a NIC's name. +func (s *Stack) SetNICName(id tcpip.NICID, name string) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + nic.name = name + return nil +} + +// SetNICMTU sets a NIC's MTU. +func (s *Stack) SetNICMTU(id tcpip.NICID, mtu uint32) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + nic.NetworkLinkEndpoint.SetMTU(mtu) + return nil +} + +// NICInfo captures the name and addresses assigned to a NIC. +type NICInfo struct { + Name string + LinkAddress tcpip.LinkAddress + ProtocolAddresses []tcpip.ProtocolAddress + + // Flags indicate the state of the NIC. + Flags NICStateFlags + + // MTU is the maximum transmission unit. + MTU uint32 + + Stats tcpip.NICStats + + // NetworkStats holds the stats of each NetworkEndpoint bound to the NIC. + NetworkStats map[tcpip.NetworkProtocolNumber]NetworkEndpointStats + + // Context is user-supplied data optionally supplied in CreateNICWithOptions. + // See type NICOptions for more details. + Context NICContext + + // ARPHardwareType holds the ARP Hardware type of the NIC. This is the + // value sent in haType field of an ARP Request sent by this NIC and the + // value expected in the haType field of an ARP response. + ARPHardwareType header.ARPHardwareType + + // Forwarding holds the forwarding status for each network endpoint that + // supports forwarding. + Forwarding map[tcpip.NetworkProtocolNumber]bool + + // MulticastForwarding holds the forwarding status for each network endpoint + // that supports multicast forwarding. + MulticastForwarding map[tcpip.NetworkProtocolNumber]bool +} + +// HasNIC returns true if the NICID is defined in the stack. +func (s *Stack) HasNIC(id tcpip.NICID) bool { + s.mu.RLock() + _, ok := s.nics[id] + s.mu.RUnlock() + return ok +} + +// NICInfo returns a map of NICIDs to their associated information. +func (s *Stack) NICInfo() map[tcpip.NICID]NICInfo { + s.mu.RLock() + defer s.mu.RUnlock() + + type forwardingFn func(tcpip.NetworkProtocolNumber) (bool, tcpip.Error) + forwardingValue := func(forwardingFn forwardingFn, proto tcpip.NetworkProtocolNumber, nicID tcpip.NICID, fnName string) (forward bool, ok bool) { + switch forwarding, err := forwardingFn(proto); err.(type) { + case nil: + return forwarding, true + case *tcpip.ErrUnknownProtocol: + panic(fmt.Sprintf("expected network protocol %d to be available on NIC %d", proto, nicID)) + case *tcpip.ErrNotSupported: + // Not all network protocols support forwarding. + default: + panic(fmt.Sprintf("nic(id=%d).%s(%d): %s", nicID, fnName, proto, err)) + } + return false, false + } + + nics := make(map[tcpip.NICID]NICInfo) + for id, nic := range s.nics { + flags := NICStateFlags{ + Up: true, // Netstack interfaces are always up. + Running: nic.Enabled(), + Promiscuous: nic.Promiscuous(), + Loopback: nic.IsLoopback(), + } + + netStats := make(map[tcpip.NetworkProtocolNumber]NetworkEndpointStats) + for proto, netEP := range nic.networkEndpoints { + netStats[proto] = netEP.Stats() + } + + info := NICInfo{ + Name: nic.name, + LinkAddress: nic.NetworkLinkEndpoint.LinkAddress(), + ProtocolAddresses: nic.primaryAddresses(), + Flags: flags, + MTU: nic.NetworkLinkEndpoint.MTU(), + Stats: nic.stats.local, + NetworkStats: netStats, + Context: nic.context, + ARPHardwareType: nic.NetworkLinkEndpoint.ARPHardwareType(), + Forwarding: make(map[tcpip.NetworkProtocolNumber]bool), + MulticastForwarding: make(map[tcpip.NetworkProtocolNumber]bool), + } + + for proto := range s.networkProtocols { + if forwarding, ok := forwardingValue(nic.forwarding, proto, id, "forwarding"); ok { + info.Forwarding[proto] = forwarding + } + + if multicastForwarding, ok := forwardingValue(nic.multicastForwarding, proto, id, "multicastForwarding"); ok { + info.MulticastForwarding[proto] = multicastForwarding + } + } + + nics[id] = info + } + return nics +} + +// NICStateFlags holds information about the state of an NIC. +type NICStateFlags struct { + // Up indicates whether the interface is running. + Up bool + + // Running indicates whether resources are allocated. + Running bool + + // Promiscuous indicates whether the interface is in promiscuous mode. + Promiscuous bool + + // Loopback indicates whether the interface is a loopback. + Loopback bool +} + +// AddProtocolAddress adds an address to the specified NIC, possibly with extra +// properties. +func (s *Stack) AddProtocolAddress(id tcpip.NICID, protocolAddress tcpip.ProtocolAddress, properties AddressProperties) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.addAddress(protocolAddress, properties) +} + +// RemoveAddress removes an existing network-layer address from the specified +// NIC. +func (s *Stack) RemoveAddress(id tcpip.NICID, addr tcpip.Address) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + if nic, ok := s.nics[id]; ok { + return nic.removeAddress(addr) + } + + return &tcpip.ErrUnknownNICID{} +} + +// SetAddressLifetimes sets informational preferred and valid lifetimes, and +// whether the address should be preferred or deprecated. +func (s *Stack) SetAddressLifetimes(id tcpip.NICID, addr tcpip.Address, lifetimes AddressLifetimes) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + if nic, ok := s.nics[id]; ok { + return nic.setAddressLifetimes(addr, lifetimes) + } + + return &tcpip.ErrUnknownNICID{} +} + +// AllAddresses returns a map of NICIDs to their protocol addresses (primary +// and non-primary). +func (s *Stack) AllAddresses() map[tcpip.NICID][]tcpip.ProtocolAddress { + s.mu.RLock() + defer s.mu.RUnlock() + + nics := make(map[tcpip.NICID][]tcpip.ProtocolAddress) + for id, nic := range s.nics { + nics[id] = nic.allPermanentAddresses() + } + return nics +} + +// GetMainNICAddress returns the first non-deprecated primary address and prefix +// for the given NIC and protocol. If no non-deprecated primary addresses exist, +// a deprecated address will be returned. If no deprecated addresses exist, the +// zero value will be returned. +func (s *Stack) GetMainNICAddress(id tcpip.NICID, protocol tcpip.NetworkProtocolNumber) (tcpip.AddressWithPrefix, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return tcpip.AddressWithPrefix{}, &tcpip.ErrUnknownNICID{} + } + + return nic.PrimaryAddress(protocol) +} + +func (s *Stack) getAddressEP(nic *nic, localAddr, remoteAddr, srcHint tcpip.Address, netProto tcpip.NetworkProtocolNumber) AssignableAddressEndpoint { + if localAddr.BitLen() == 0 { + return nic.primaryEndpoint(netProto, remoteAddr, srcHint) + } + return nic.findEndpoint(netProto, localAddr, CanBePrimaryEndpoint) +} + +// NewRouteForMulticast returns a Route that may be used to forward multicast +// packets. +// +// Returns nil if validation fails. +func (s *Stack) NewRouteForMulticast(nicID tcpip.NICID, remoteAddr tcpip.Address, netProto tcpip.NetworkProtocolNumber) *Route { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[nicID] + if !ok || !nic.Enabled() { + return nil + } + + if addressEndpoint := s.getAddressEP(nic, tcpip.Address{} /* localAddr */, remoteAddr, tcpip.Address{} /* srcHint */, netProto); addressEndpoint != nil { + return constructAndValidateRoute(netProto, addressEndpoint, nic, nic, tcpip.Address{} /* gateway */, tcpip.Address{} /* localAddr */, remoteAddr, s.handleLocal, false /* multicastLoop */, 0 /* mtu */) + } + return nil +} + +// findLocalRouteFromNICRLocked is like findLocalRouteRLocked but finds a route +// from the specified NIC. +// +// +checklocksread:s.mu +func (s *Stack) findLocalRouteFromNICRLocked(localAddressNIC *nic, localAddr, remoteAddr tcpip.Address, netProto tcpip.NetworkProtocolNumber) *Route { + localAddressEndpoint := localAddressNIC.getAddressOrCreateTempInner(netProto, localAddr, false /* createTemp */, NeverPrimaryEndpoint) + if localAddressEndpoint == nil { + return nil + } + + var outgoingNIC *nic + // Prefer a local route to the same interface as the local address. + if localAddressNIC.hasAddress(netProto, remoteAddr) { + outgoingNIC = localAddressNIC + } + + // If the remote address isn't owned by the local address's NIC, check all + // NICs. + if outgoingNIC == nil { + for _, nic := range s.nics { + if nic.hasAddress(netProto, remoteAddr) { + outgoingNIC = nic + break + } + } + } + + // If the remote address is not owned by the stack, we can't return a local + // route. + if outgoingNIC == nil { + localAddressEndpoint.DecRef() + return nil + } + + r := makeLocalRoute( + netProto, + localAddr, + remoteAddr, + outgoingNIC, + localAddressNIC, + localAddressEndpoint, + ) + + if r.IsOutboundBroadcast() { + r.Release() + return nil + } + + return r +} + +func (s *Stack) loopbackLocalRoute(localAddressNIC *nic, localAddr, remoteAddr tcpip.Address, netProto tcpip.NetworkProtocolNumber) *Route { + localAddressEndpoint := localAddressNIC.getAddressOrCreateTempInner(netProto, localAddr, true /* createTemp */, NeverPrimaryEndpoint) + if localAddressEndpoint == nil { + return nil + } + + r := makeLocalRoute( + netProto, + localAddr, + remoteAddr, + localAddressNIC, + localAddressNIC, + localAddressEndpoint, + ) + + if r.IsOutboundBroadcast() { + r.Release() + return nil + } + + return r +} + +// findLocalRouteRLocked returns a local route. +// +// A local route is a route to some remote address which the stack owns. That +// is, a local route is a route where packets never have to leave the stack. +// +// +checklocksread:s.mu +func (s *Stack) findLocalRouteRLocked(localAddressNICID tcpip.NICID, localAddr, remoteAddr tcpip.Address, netProto tcpip.NetworkProtocolNumber) *Route { + if localAddr.BitLen() == 0 { + localAddr = remoteAddr + } + + if localAddressNICID == 0 { + if s.loopbackNIC != nil { + // Send all packets directed to local ip addresses through the loopback device. + for _, nic := range s.nics { + if !nic.hasAddress(netProto, remoteAddr) { + continue + } + if isSubnetBroadcastOnNIC(nic, netProto, remoteAddr) { + break + } + if r := s.loopbackLocalRoute(s.loopbackNIC, localAddr, remoteAddr, netProto); r != nil { + return r + } + break + } + } + + for _, localAddressNIC := range s.nics { + if r := s.findLocalRouteFromNICRLocked(localAddressNIC, localAddr, remoteAddr, netProto); r != nil { + return r + } + } + + return nil + } + + if localAddressNIC, ok := s.nics[localAddressNICID]; ok { + return s.findLocalRouteFromNICRLocked(localAddressNIC, localAddr, remoteAddr, netProto) + } + + return nil +} + +// HandleLocal returns true if non-loopback interfaces are allowed to loop packets. +func (s *Stack) HandleLocal() bool { + return s.handleLocal +} + +func isNICForwarding(nic *nic, proto tcpip.NetworkProtocolNumber) bool { + switch forwarding, err := nic.forwarding(proto); err.(type) { + case nil: + return forwarding + case *tcpip.ErrUnknownProtocol: + panic(fmt.Sprintf("expected network protocol %d to be available on NIC %d", proto, nic.ID())) + case *tcpip.ErrNotSupported: + // Not all network protocols support forwarding. + return false + default: + panic(fmt.Sprintf("nic(id=%d).forwarding(%d): %s", nic.ID(), proto, err)) + } +} + +// findRouteWithLocalAddrFromAnyInterfaceRLocked returns a route to the given +// destination address, leaving through the given NIC. +// +// Rather than preferring to find a route that uses a local address assigned to +// the outgoing interface, it finds any NIC that holds a matching local address +// endpoint. +// +// +checklocksread:s.mu +func (s *Stack) findRouteWithLocalAddrFromAnyInterfaceRLocked(outgoingNIC *nic, localAddr, remoteAddr, srcHint, gateway tcpip.Address, netProto tcpip.NetworkProtocolNumber, multicastLoop bool, mtu uint32) *Route { + for _, aNIC := range s.nics { + addressEndpoint := s.getAddressEP(aNIC, localAddr, remoteAddr, srcHint, netProto) + if addressEndpoint == nil { + continue + } + + if r := constructAndValidateRoute(netProto, addressEndpoint, aNIC /* localAddressNIC */, outgoingNIC, gateway, localAddr, remoteAddr, s.handleLocal, multicastLoop, mtu); r != nil { + return r + } + } + return nil +} + +// FindRoute creates a route to the given destination address, leaving through +// the given NIC and local address (if provided). +// +// If a NIC is not specified, the returned route will leave through the same +// NIC as the NIC that has the local address assigned when forwarding is +// disabled. If forwarding is enabled and the NIC is unspecified, the route may +// leave through any interface unless the route is link-local. +// +// If no local address is provided, the stack will select a local address. If no +// remote address is provided, the stack will use a remote address equal to the +// local address. +func (s *Stack) FindRoute(id tcpip.NICID, localAddr, remoteAddr tcpip.Address, netProto tcpip.NetworkProtocolNumber, multicastLoop bool) (*Route, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + // Reject attempts to use unsupported protocols. + if !s.CheckNetworkProtocol(netProto) { + return nil, &tcpip.ErrUnknownProtocol{} + } + + isLinkLocal := header.IsV6LinkLocalUnicastAddress(remoteAddr) || header.IsV6LinkLocalMulticastAddress(remoteAddr) + isLocalBroadcast := remoteAddr == header.IPv4Broadcast + isMulticast := header.IsV4MulticastAddress(remoteAddr) || header.IsV6MulticastAddress(remoteAddr) + isLoopback := header.IsV4LoopbackAddress(remoteAddr) || header.IsV6LoopbackAddress(remoteAddr) + needRoute := !(isLocalBroadcast || isMulticast || isLinkLocal || isLoopback) + + if s.handleLocal && !isMulticast && !isLocalBroadcast { + if r := s.findLocalRouteRLocked(id, localAddr, remoteAddr, netProto); r != nil { + return r, nil + } + } + + // If the interface is specified and we do not need a route, return a route + // through the interface if the interface is valid and enabled. + if id != 0 && !needRoute { + if nic, ok := s.nics[id]; ok && nic.Enabled() { + if addressEndpoint := s.getAddressEP(nic, localAddr, remoteAddr, tcpip.Address{} /* srcHint */, netProto); addressEndpoint != nil { + return makeRoute( + netProto, + tcpip.Address{}, /* gateway */ + localAddr, + remoteAddr, + nic, /* outgoingNIC */ + nic, /* localAddressNIC*/ + addressEndpoint, + s.handleLocal, + multicastLoop, + 0, /* mtu */ + ), nil + } + } + + if isLoopback { + return nil, &tcpip.ErrBadLocalAddress{} + } + return nil, &tcpip.ErrNetworkUnreachable{} + } + + onlyGlobalAddresses := !header.IsV6LinkLocalUnicastAddress(localAddr) && !isLinkLocal + + // Find a route to the remote with the route table. + var chosenRoute tcpip.Route + if r := func() *Route { + s.routeMu.RLock() + defer s.routeMu.RUnlock() + + for route := s.routeTable.Front(); route != nil; route = route.Next() { + if remoteAddr.BitLen() != 0 && !route.Destination.Contains(remoteAddr) { + continue + } + + nic, ok := s.nics[route.NIC] + if !ok || !nic.Enabled() { + continue + } + + if id == 0 || id == route.NIC { + if addressEndpoint := s.getAddressEP(nic, localAddr, remoteAddr, route.SourceHint, netProto); addressEndpoint != nil { + var gateway tcpip.Address + if needRoute { + gateway = route.Gateway + } + r := constructAndValidateRoute(netProto, addressEndpoint, nic /* outgoingNIC */, nic /* outgoingNIC */, gateway, localAddr, remoteAddr, s.handleLocal, multicastLoop, route.MTU) + if r == nil { + panic(fmt.Sprintf("non-forwarding route validation failed with route table entry = %#v, id = %d, localAddr = %s, remoteAddr = %s", route, id, localAddr, remoteAddr)) + } + return r + } + } + + // If the stack has forwarding enabled, we haven't found a valid route to + // the remote address yet, and we are routing locally generated traffic, + // keep track of the first valid route. We keep iterating because we + // prefer routes that let us use a local address that is assigned to the + // outgoing interface. There is no requirement to do this from any RFC + // but simply a choice made to better follow a strong host model which + // the netstack follows at the time of writing. + // + // Note that for incoming traffic that we are forwarding (for which the + // NIC and local address are unspecified), we do not keep iterating, as + // there is no reason to prefer routes that let us use a local address + // when routing forwarded (as opposed to locally-generated) traffic. + locallyGenerated := (id != 0 || localAddr != tcpip.Address{}) + if onlyGlobalAddresses && chosenRoute.Equal(tcpip.Route{}) && isNICForwarding(nic, netProto) { + if locallyGenerated { + chosenRoute = *route + continue + } + + if r := s.findRouteWithLocalAddrFromAnyInterfaceRLocked(nic, localAddr, remoteAddr, route.SourceHint, route.Gateway, netProto, multicastLoop, route.MTU); r != nil { + return r + } + } + } + + return nil + }(); r != nil { + return r, nil + } + + if !chosenRoute.Equal(tcpip.Route{}) { + // At this point we know the stack has forwarding enabled since chosenRoute is + // only set when forwarding is enabled. + nic, ok := s.nics[chosenRoute.NIC] + if !ok { + // If the route's NIC was invalid, we should not have chosen the route. + panic(fmt.Sprintf("chosen route must have a valid NIC with ID = %d", chosenRoute.NIC)) + } + + var gateway tcpip.Address + if needRoute { + gateway = chosenRoute.Gateway + } + + // Use the specified NIC to get the local address endpoint. + if id != 0 { + if aNIC, ok := s.nics[id]; ok { + if addressEndpoint := s.getAddressEP(aNIC, localAddr, remoteAddr, chosenRoute.SourceHint, netProto); addressEndpoint != nil { + if r := constructAndValidateRoute(netProto, addressEndpoint, aNIC /* localAddressNIC */, nic /* outgoingNIC */, gateway, localAddr, remoteAddr, s.handleLocal, multicastLoop, chosenRoute.MTU); r != nil { + return r, nil + } + } + } + + return nil, &tcpip.ErrNetworkUnreachable{} + } + + if id == 0 { + // If an interface is not specified, try to find a NIC that holds the local + // address endpoint to construct a route. + if r := s.findRouteWithLocalAddrFromAnyInterfaceRLocked(nic, localAddr, remoteAddr, chosenRoute.SourceHint, gateway, netProto, multicastLoop, chosenRoute.MTU); r != nil { + return r, nil + } + } + } + + if needRoute { + return nil, &tcpip.ErrNetworkUnreachable{} + } + if header.IsV6LoopbackAddress(remoteAddr) { + return nil, &tcpip.ErrBadLocalAddress{} + } + return nil, &tcpip.ErrNetworkUnreachable{} +} + +// CheckNetworkProtocol checks if a given network protocol is enabled in the +// stack. +func (s *Stack) CheckNetworkProtocol(protocol tcpip.NetworkProtocolNumber) bool { + _, ok := s.networkProtocols[protocol] + return ok +} + +// CheckDuplicateAddress performs duplicate address detection for the address on +// the specified interface. +func (s *Stack) CheckDuplicateAddress(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber, addr tcpip.Address, h DADCompletionHandler) (DADCheckAddressDisposition, tcpip.Error) { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + + if !ok { + return 0, &tcpip.ErrUnknownNICID{} + } + + return nic.checkDuplicateAddress(protocol, addr, h) +} + +// CheckLocalAddress determines if the given local address exists, and if it +// does, returns the id of the NIC it's bound to. Returns 0 if the address +// does not exist. +func (s *Stack) CheckLocalAddress(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) tcpip.NICID { + s.mu.RLock() + defer s.mu.RUnlock() + + // If a NIC is specified, use its NIC id. + if nicID != 0 { + nic, ok := s.nics[nicID] + if !ok { + return 0 + } + // In IPv4, linux only checks the interface. If it matches, then it does + // not bother with the address. + // https://github.com/torvalds/linux/blob/15205c2829ca2cbb5ece5ceaafe1171a8470e62b/net/ipv4/igmp.c#L1829-L1837 + if protocol == header.IPv4ProtocolNumber { + return nic.id + } + if nic.CheckLocalAddress(protocol, addr) { + return nic.id + } + return 0 + } + + // Go through all the NICs. + for _, nic := range s.nics { + if nic.CheckLocalAddress(protocol, addr) { + return nic.id + } + } + + return 0 +} + +// SetPromiscuousMode enables or disables promiscuous mode in the given NIC. +func (s *Stack) SetPromiscuousMode(nicID tcpip.NICID, enable bool) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[nicID] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + nic.setPromiscuousMode(enable) + + return nil +} + +// SetSpoofing enables or disables address spoofing in the given NIC, allowing +// endpoints to bind to any address in the NIC. +func (s *Stack) SetSpoofing(nicID tcpip.NICID, enable bool) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[nicID] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + nic.setSpoofing(enable) + + return nil +} + +// LinkResolutionResult is the result of a link address resolution attempt. +type LinkResolutionResult struct { + LinkAddress tcpip.LinkAddress + Err tcpip.Error +} + +// GetLinkAddress finds the link address corresponding to a network address. +// +// Returns ErrNotSupported if the stack is not configured with a link address +// resolver for the specified network protocol. +// +// Returns ErrWouldBlock if the link address is not readily available, along +// with a notification channel for the caller to block on. Triggers address +// resolution asynchronously. +// +// onResolve will be called either immediately, if resolution is not required, +// or when address resolution is complete, with the resolved link address and +// whether resolution succeeded. +// +// If specified, the local address must be an address local to the interface +// the neighbor cache belongs to. The local address is the source address of +// a packet prompting NUD/link address resolution. +func (s *Stack) GetLinkAddress(nicID tcpip.NICID, addr, localAddr tcpip.Address, protocol tcpip.NetworkProtocolNumber, onResolve func(LinkResolutionResult)) tcpip.Error { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.getLinkAddress(addr, localAddr, protocol, onResolve) +} + +// Neighbors returns all IP to MAC address associations. +func (s *Stack) Neighbors(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber) ([]NeighborEntry, tcpip.Error) { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + + if !ok { + return nil, &tcpip.ErrUnknownNICID{} + } + + return nic.neighbors(protocol) +} + +// AddStaticNeighbor statically associates an IP address to a MAC address. +func (s *Stack) AddStaticNeighbor(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber, addr tcpip.Address, linkAddr tcpip.LinkAddress) tcpip.Error { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.addStaticNeighbor(addr, protocol, linkAddr) +} + +// RemoveNeighbor removes an IP to MAC address association previously created +// either automatically or by AddStaticNeighbor. Returns ErrBadAddress if there +// is no association with the provided address. +func (s *Stack) RemoveNeighbor(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) tcpip.Error { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.removeNeighbor(protocol, addr) +} + +// ClearNeighbors removes all IP to MAC address associations. +func (s *Stack) ClearNeighbors(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber) tcpip.Error { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.clearNeighbors(protocol) +} + +// RegisterTransportEndpoint registers the given endpoint with the stack +// transport dispatcher. Received packets that match the provided id will be +// delivered to the given endpoint; specifying a nic is optional, but +// nic-specific IDs have precedence over global ones. +func (s *Stack) RegisterTransportEndpoint(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + return s.demux.registerEndpoint(netProtos, protocol, id, ep, flags, bindToDevice) +} + +// CheckRegisterTransportEndpoint checks if an endpoint can be registered with +// the stack transport dispatcher. +func (s *Stack) CheckRegisterTransportEndpoint(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + return s.demux.checkEndpoint(netProtos, protocol, id, flags, bindToDevice) +} + +// UnregisterTransportEndpoint removes the endpoint with the given id from the +// stack transport dispatcher. +func (s *Stack) UnregisterTransportEndpoint(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) { + s.demux.unregisterEndpoint(netProtos, protocol, id, ep, flags, bindToDevice) +} + +// StartTransportEndpointCleanup removes the endpoint with the given id from +// the stack transport dispatcher. It also transitions it to the cleanup stage. +func (s *Stack) StartTransportEndpointCleanup(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) { + s.cleanupEndpointsMu.Lock() + s.cleanupEndpoints[ep] = struct{}{} + s.cleanupEndpointsMu.Unlock() + + s.demux.unregisterEndpoint(netProtos, protocol, id, ep, flags, bindToDevice) +} + +// CompleteTransportEndpointCleanup removes the endpoint from the cleanup +// stage. +func (s *Stack) CompleteTransportEndpointCleanup(ep TransportEndpoint) { + s.cleanupEndpointsMu.Lock() + delete(s.cleanupEndpoints, ep) + s.cleanupEndpointsMu.Unlock() +} + +// FindTransportEndpoint finds an endpoint that most closely matches the provided +// id. If no endpoint is found it returns nil. +func (s *Stack) FindTransportEndpoint(netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, id TransportEndpointID, nicID tcpip.NICID) TransportEndpoint { + return s.demux.findTransportEndpoint(netProto, transProto, id, nicID) +} + +// RegisterRawTransportEndpoint registers the given endpoint with the stack +// transport dispatcher. Received packets that match the provided transport +// protocol will be delivered to the given endpoint. +func (s *Stack) RegisterRawTransportEndpoint(netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, ep RawTransportEndpoint) tcpip.Error { + return s.demux.registerRawEndpoint(netProto, transProto, ep) +} + +// UnregisterRawTransportEndpoint removes the endpoint for the transport +// protocol from the stack transport dispatcher. +func (s *Stack) UnregisterRawTransportEndpoint(netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, ep RawTransportEndpoint) { + s.demux.unregisterRawEndpoint(netProto, transProto, ep) +} + +// RegisterRestoredEndpoint records e as an endpoint that has been restored on +// this stack. +func (s *Stack) RegisterRestoredEndpoint(e RestoredEndpoint) { + s.mu.Lock() + defer s.mu.Unlock() + + s.restoredEndpoints = append(s.restoredEndpoints, e) +} + +// RegisterResumableEndpoint records e as an endpoint that has to be resumed. +func (s *Stack) RegisterResumableEndpoint(e ResumableEndpoint) { + s.mu.Lock() + defer s.mu.Unlock() + + s.resumableEndpoints = append(s.resumableEndpoints, e) +} + +// RegisteredEndpoints returns all endpoints which are currently registered. +func (s *Stack) RegisteredEndpoints() []TransportEndpoint { + s.mu.Lock() + defer s.mu.Unlock() + + var es []TransportEndpoint + for _, e := range s.demux.protocol { + es = append(es, e.transportEndpoints()...) + } + return es +} + +// CleanupEndpoints returns endpoints currently in the cleanup state. +func (s *Stack) CleanupEndpoints() []TransportEndpoint { + s.cleanupEndpointsMu.Lock() + defer s.cleanupEndpointsMu.Unlock() + + es := make([]TransportEndpoint, 0, len(s.cleanupEndpoints)) + for e := range s.cleanupEndpoints { + es = append(es, e) + } + return es +} + +// RestoreCleanupEndpoints adds endpoints to cleanup tracking. This is useful +// for restoring a stack after a save. +func (s *Stack) RestoreCleanupEndpoints(es []TransportEndpoint) { + s.cleanupEndpointsMu.Lock() + defer s.cleanupEndpointsMu.Unlock() + + for _, e := range es { + s.cleanupEndpoints[e] = struct{}{} + } +} + +// Close closes all currently registered transport endpoints. +// +// Endpoints created or modified during this call may not get closed. +func (s *Stack) Close() { + for _, e := range s.RegisteredEndpoints() { + e.Abort() + } + for _, p := range s.transportProtocols { + p.proto.Close() + } + for _, p := range s.networkProtocols { + p.Close() + } +} + +// Wait waits for all transport and link endpoints to halt their worker +// goroutines. +// +// Endpoints created or modified during this call may not get waited on. +// +// Note that link endpoints must be stopped via an implementation specific +// mechanism. +func (s *Stack) Wait() { + for _, e := range s.RegisteredEndpoints() { + e.Wait() + } + for _, e := range s.CleanupEndpoints() { + e.Wait() + } + for _, p := range s.transportProtocols { + p.proto.Wait() + } + for _, p := range s.networkProtocols { + p.Wait() + } + + deferActs := make([]func(), 0) + + s.mu.Lock() + for id, n := range s.nics { + // Remove NIC to ensure that qDisc goroutines are correctly + // terminated on stack teardown. + act, _ := s.removeNICLocked(id) + n.NetworkLinkEndpoint.Wait() + if act != nil { + deferActs = append(deferActs, act) + } + } + s.mu.Unlock() + + for _, act := range deferActs { + act() + } +} + +// Destroy destroys the stack with all endpoints. +func (s *Stack) Destroy() { + s.Close() + s.Wait() +} + +// Pause pauses any protocol level background workers. +func (s *Stack) Pause() { + for _, p := range s.transportProtocols { + p.proto.Pause() + } +} + +func (s *Stack) getNICs() map[tcpip.NICID]*nic { + s.mu.RLock() + defer s.mu.RUnlock() + + nics := s.nics + return nics +} + +// ReplaceConfig replaces config in the loaded stack. +func (s *Stack) ReplaceConfig(st *Stack) { + if st == nil { + panic("stack.Stack cannot be nil when netstack s/r is enabled") + } + + // Update route table. + s.SetRouteTable(st.GetRouteTable()) + + // Update NICs. + nics := st.getNICs() + s.mu.Lock() + defer s.mu.Unlock() + s.nics = make(map[tcpip.NICID]*nic) + s.loopbackNIC = nil + for id, nic := range nics { + nic.stack = s + s.nics[id] = nic + if nic.IsLoopback() { + s.loopbackNIC = nic + } + _ = s.NextNICID() + } + s.tables = st.tables + s.nftables = st.nftables +} + +// Restore restarts the stack after a restore. This must be called after the +// entire system has been restored. +func (s *Stack) Restore() { + // RestoredEndpoint.Restore() may call other methods on s, so we can't hold + // s.mu while restoring the endpoints. + s.mu.Lock() + eps := s.restoredEndpoints + s.restoredEndpoints = nil + saveRestoreEnabled := s.saveRestoreEnabled + s.mu.Unlock() + for _, e := range eps { + e.Restore(s) + } + + // Make sure all the endpoints are loaded correctly before resuming the + // protocol level background workers. + tcpip.AsyncLoading.Wait() + + // Now resume any protocol level background workers. + for _, p := range s.transportProtocols { + if saveRestoreEnabled { + p.proto.Restore() + } else { + p.proto.Resume() + } + } +} + +// Resume resumes the stack after a save. +func (s *Stack) Resume() { + s.mu.Lock() + eps := s.resumableEndpoints + s.resumableEndpoints = nil + s.mu.Unlock() + for _, e := range eps { + e.Resume() + } + // Now resume any protocol level background workers. + for _, p := range s.transportProtocols { + p.proto.Resume() + } +} + +// RegisterPacketEndpoint registers ep with the stack, causing it to receive +// all traffic of the specified netProto on the given NIC. If nicID is 0, it +// receives traffic from every NIC. +func (s *Stack) RegisterPacketEndpoint(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, ep PacketEndpoint) tcpip.Error { + s.mu.Lock() + defer s.mu.Unlock() + + // If no NIC is specified, capture on all devices. + if nicID == 0 { + // Register with each NIC. + for _, nic := range s.nics { + nic.registerPacketEndpoint(netProto, ep) + } + return nil + } + + // Capture on a specific device. + nic, ok := s.nics[nicID] + if !ok { + return &tcpip.ErrUnknownNICID{} + } + nic.registerPacketEndpoint(netProto, ep) + + return nil +} + +// UnregisterPacketEndpoint unregisters ep for packets of the specified +// netProto from the specified NIC. If nicID is 0, ep is unregistered from all +// NICs. +func (s *Stack) UnregisterPacketEndpoint(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, ep PacketEndpoint) { + s.mu.Lock() + defer s.mu.Unlock() + s.unregisterPacketEndpointLocked(nicID, netProto, ep) +} + +// +checklocks:s.mu +func (s *Stack) unregisterPacketEndpointLocked(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, ep PacketEndpoint) { + // If no NIC is specified, unregister on all devices. + if nicID == 0 { + // Unregister with each NIC. + for _, nic := range s.nics { + nic.unregisterPacketEndpoint(netProto, ep) + } + return + } + + // Unregister in a single device. + nic, ok := s.nics[nicID] + if !ok { + return + } + nic.unregisterPacketEndpoint(netProto, ep) +} + +// WritePacketToRemote writes a payload on the specified NIC using the provided +// network protocol and remote link address. +func (s *Stack) WritePacketToRemote(nicID tcpip.NICID, remote tcpip.LinkAddress, netProto tcpip.NetworkProtocolNumber, payload buffer.Buffer) tcpip.Error { + s.mu.Lock() + nic, ok := s.nics[nicID] + s.mu.Unlock() + if !ok { + return &tcpip.ErrUnknownDevice{} + } + pkt := NewPacketBuffer(PacketBufferOptions{ + ReserveHeaderBytes: int(nic.MaxHeaderLength()), + Payload: payload, + }) + defer pkt.DecRef() + pkt.NetworkProtocolNumber = netProto + return nic.WritePacketToRemote(remote, pkt) +} + +// WriteRawPacket writes data directly to the specified NIC without adding any +// headers. +func (s *Stack) WriteRawPacket(nicID tcpip.NICID, proto tcpip.NetworkProtocolNumber, payload buffer.Buffer) tcpip.Error { + s.mu.RLock() + nic, ok := s.nics[nicID] + s.mu.RUnlock() + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + pkt := NewPacketBuffer(PacketBufferOptions{ + Payload: payload, + }) + defer pkt.DecRef() + pkt.NetworkProtocolNumber = proto + return nic.writeRawPacketWithLinkHeaderInPayload(pkt) +} + +// NetworkProtocolInstance returns the protocol instance in the stack for the +// specified network protocol. This method is public for protocol implementers +// and tests to use. +func (s *Stack) NetworkProtocolInstance(num tcpip.NetworkProtocolNumber) NetworkProtocol { + if p, ok := s.networkProtocols[num]; ok { + return p + } + return nil +} + +// TransportProtocolInstance returns the protocol instance in the stack for the +// specified transport protocol. This method is public for protocol implementers +// and tests to use. +func (s *Stack) TransportProtocolInstance(num tcpip.TransportProtocolNumber) TransportProtocol { + if pState, ok := s.transportProtocols[num]; ok { + return pState.proto + } + return nil +} + +// JoinGroup joins the given multicast group on the given NIC. +func (s *Stack) JoinGroup(protocol tcpip.NetworkProtocolNumber, nicID tcpip.NICID, multicastAddr tcpip.Address) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + if nic, ok := s.nics[nicID]; ok { + return nic.joinGroup(protocol, multicastAddr) + } + return &tcpip.ErrUnknownNICID{} +} + +// LeaveGroup leaves the given multicast group on the given NIC. +func (s *Stack) LeaveGroup(protocol tcpip.NetworkProtocolNumber, nicID tcpip.NICID, multicastAddr tcpip.Address) tcpip.Error { + s.mu.RLock() + defer s.mu.RUnlock() + + if nic, ok := s.nics[nicID]; ok { + return nic.leaveGroup(protocol, multicastAddr) + } + return &tcpip.ErrUnknownNICID{} +} + +// IsInGroup returns true if the NIC with ID nicID has joined the multicast +// group multicastAddr. +func (s *Stack) IsInGroup(nicID tcpip.NICID, multicastAddr tcpip.Address) (bool, tcpip.Error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if nic, ok := s.nics[nicID]; ok { + return nic.isInGroup(multicastAddr), nil + } + return false, &tcpip.ErrUnknownNICID{} +} + +// IPTables returns the stack's iptables. +func (s *Stack) IPTables() *IPTables { + return s.tables +} + +// NFTables returns the stack's nftables. +func (s *Stack) NFTables() NFTablesInterface { + return s.nftables +} + +// SetNFTables sets the stack's nftables. +func (s *Stack) SetNFTables(nft NFTablesInterface) { + s.nftables = nft +} + +// ICMPLimit returns the maximum number of ICMP messages that can be sent +// in one second. +func (s *Stack) ICMPLimit() rate.Limit { + return s.icmpRateLimiter.Limit() +} + +// SetICMPLimit sets the maximum number of ICMP messages that be sent +// in one second. +func (s *Stack) SetICMPLimit(newLimit rate.Limit) { + s.icmpRateLimiter.SetLimit(newLimit) +} + +// ICMPBurst returns the maximum number of ICMP messages that can be sent +// in a single burst. +func (s *Stack) ICMPBurst() int { + return s.icmpRateLimiter.Burst() +} + +// SetICMPBurst sets the maximum number of ICMP messages that can be sent +// in a single burst. +func (s *Stack) SetICMPBurst(burst int) { + s.icmpRateLimiter.SetBurst(burst) +} + +// AllowICMPMessage returns true if we the rate limiter allows at least one +// ICMP message to be sent at this instant. +func (s *Stack) AllowICMPMessage() bool { + return s.icmpRateLimiter.Allow() +} + +// GetNetworkEndpoint returns the NetworkEndpoint with the specified protocol +// number installed on the specified NIC. +func (s *Stack) GetNetworkEndpoint(nicID tcpip.NICID, proto tcpip.NetworkProtocolNumber) (NetworkEndpoint, tcpip.Error) { + s.mu.Lock() + defer s.mu.Unlock() + + nic, ok := s.nics[nicID] + if !ok { + return nil, &tcpip.ErrUnknownNICID{} + } + + return nic.getNetworkEndpoint(proto), nil +} + +// NUDConfigurations gets the per-interface NUD configurations. +func (s *Stack) NUDConfigurations(id tcpip.NICID, proto tcpip.NetworkProtocolNumber) (NUDConfigurations, tcpip.Error) { + s.mu.RLock() + nic, ok := s.nics[id] + s.mu.RUnlock() + + if !ok { + return NUDConfigurations{}, &tcpip.ErrUnknownNICID{} + } + + return nic.nudConfigs(proto) +} + +// SetNUDConfigurations sets the per-interface NUD configurations. +// +// Note, if c contains invalid NUD configuration values, it will be fixed to +// use default values for the erroneous values. +func (s *Stack) SetNUDConfigurations(id tcpip.NICID, proto tcpip.NetworkProtocolNumber, c NUDConfigurations) tcpip.Error { + s.mu.RLock() + nic, ok := s.nics[id] + s.mu.RUnlock() + + if !ok { + return &tcpip.ErrUnknownNICID{} + } + + return nic.setNUDConfigs(proto, c) +} + +// Seed returns a 32 bit value that can be used as a seed value. +// +// NOTE: The seed is generated once during stack initialization only. +func (s *Stack) Seed() uint32 { + return s.seed +} + +// InsecureRNG returns a reference to a pseudo random generator that can be used +// to generate random numbers as required. It is not cryptographically secure +// and should not be used for security sensitive work. +func (s *Stack) InsecureRNG() *rand.Rand { + return s.insecureRNG +} + +// SecureRNG returns the stack's cryptographically secure random number +// generator. +func (s *Stack) SecureRNG() cryptorand.RNG { + return s.secureRNG +} + +// FindNICNameFromID returns the name of the NIC for the given NICID. +func (s *Stack) FindNICNameFromID(id tcpip.NICID) string { + s.mu.RLock() + defer s.mu.RUnlock() + + nic, ok := s.nics[id] + if !ok { + return "" + } + + return nic.Name() +} + +// ParseResult indicates the result of a parsing attempt. +type ParseResult int + +const ( + // ParsedOK indicates that a packet was successfully parsed. + ParsedOK ParseResult = iota + + // UnknownTransportProtocol indicates that the transport protocol is unknown. + UnknownTransportProtocol + + // TransportLayerParseError indicates that the transport packet was not + // successfully parsed. + TransportLayerParseError +) + +// ParsePacketBufferTransport parses the provided packet buffer's transport +// header. +func (s *Stack) ParsePacketBufferTransport(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) ParseResult { + pkt.TransportProtocolNumber = protocol + // Parse the transport header if present. + state, ok := s.transportProtocols[protocol] + if !ok { + return UnknownTransportProtocol + } + + if !state.proto.Parse(pkt) { + return TransportLayerParseError + } + + return ParsedOK +} + +// networkProtocolNumbers returns the network protocol numbers the stack is +// configured with. +func (s *Stack) networkProtocolNumbers() []tcpip.NetworkProtocolNumber { + protos := make([]tcpip.NetworkProtocolNumber, 0, len(s.networkProtocols)) + for p := range s.networkProtocols { + protos = append(protos, p) + } + return protos +} + +func isSubnetBroadcastOnNIC(nic *nic, protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) bool { + addressEndpoint := nic.getAddressOrCreateTempInner(protocol, addr, false /* createTemp */, NeverPrimaryEndpoint) + if addressEndpoint == nil { + return false + } + + subnet := addressEndpoint.Subnet() + addressEndpoint.DecRef() + return subnet.IsBroadcast(addr) +} + +// IsSubnetBroadcast returns true if the provided address is a subnet-local +// broadcast address on the specified NIC and protocol. +// +// Returns false if the NIC is unknown or if the protocol is unknown or does +// not support addressing. +// +// If the NIC is not specified, the stack will check all NICs. +func (s *Stack) IsSubnetBroadcast(nicID tcpip.NICID, protocol tcpip.NetworkProtocolNumber, addr tcpip.Address) bool { + s.mu.RLock() + defer s.mu.RUnlock() + + if nicID != 0 { + nic, ok := s.nics[nicID] + if !ok { + return false + } + + return isSubnetBroadcastOnNIC(nic, protocol, addr) + } + + for _, nic := range s.nics { + if isSubnetBroadcastOnNIC(nic, protocol, addr) { + return true + } + } + + return false +} + +// PacketEndpointWriteSupported returns true iff packet endpoints support write +// operations. +func (s *Stack) PacketEndpointWriteSupported() bool { + return s.packetEndpointWriteSupported +} + +// SetNICStack moves the network device to the specified network namespace. +func (s *Stack) SetNICStack(id tcpip.NICID, peer *Stack) (tcpip.NICID, tcpip.Error) { + s.mu.Lock() + nic, ok := s.nics[id] + if !ok { + s.mu.Unlock() + return 0, &tcpip.ErrUnknownNICID{} + } + if s == peer { + s.mu.Unlock() + return id, nil + } + delete(s.nics, id) + + // Remove routes in-place. n tracks the number of routes written. + s.RemoveRoutes(func(r tcpip.Route) bool { return r.NIC == id }) + ne := nic.NetworkLinkEndpoint.(LinkEndpoint) + deferAct, err := nic.remove(false /* closeLinkEndpoint */) + s.mu.Unlock() + if deferAct != nil { + deferAct() + } + if err != nil { + return 0, err + } + + id = tcpip.NICID(peer.NextNICID()) + return id, peer.CreateNICWithOptions(id, ne, NICOptions{Name: nic.Name()}) +} + +// EnableSaveRestore marks the saveRestoreEnabled to true. +func (s *Stack) EnableSaveRestore() { + s.mu.Lock() + defer s.mu.Unlock() + + s.saveRestoreEnabled = true +} + +// IsSaveRestoreEnabled returns true if save restore is enabled for the stack. +func (s *Stack) IsSaveRestoreEnabled() bool { + s.mu.Lock() + defer s.mu.Unlock() + + return s.saveRestoreEnabled +} + +// contextID is this package's type for context.Context.Value keys. +type contextID int + +const ( + // CtxRestoreStack is a Context.Value key for the stack to be used in restore. + CtxRestoreStack contextID = iota +) + +// RestoreStackFromContext returns the stack to be used during restore. +func RestoreStackFromContext(ctx context.Context) *Stack { + return ctx.Value(CtxRestoreStack).(*Stack) +} diff --git a/pkg/tcpip/stack/stack_mutex.go b/pkg/tcpip/stack/stack_mutex.go new file mode 100644 index 0000000..811c819 --- /dev/null +++ b/pkg/tcpip/stack/stack_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type stackRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var stacklockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type stacklockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *stackRWMutex) Lock() { + locking.AddGLock(stackprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *stackRWMutex) NestedLock(i stacklockNameIndex) { + locking.AddGLock(stackprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *stackRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(stackprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *stackRWMutex) NestedUnlock(i stacklockNameIndex) { + m.mu.Unlock() + locking.DelGLock(stackprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *stackRWMutex) RLock() { + locking.AddGLock(stackprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *stackRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(stackprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *stackRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *stackRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *stackRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var stackprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func stackinitLockNames() {} + +func init() { + stackinitLockNames() + stackprefixIndex = locking.NewMutexClass(reflect.TypeOf(stackRWMutex{}), stacklockNames) +} diff --git a/pkg/tcpip/stack/stack_options.go b/pkg/tcpip/stack/stack_options.go new file mode 100644 index 0000000..02dfebe --- /dev/null +++ b/pkg/tcpip/stack/stack_options.go @@ -0,0 +1,125 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +const ( + // MinBufferSize is the smallest size of a receive or send buffer. + MinBufferSize = 4 << 10 // 4 KiB + + // DefaultBufferSize is the default size of the send/recv buffer for a + // transport endpoint. + DefaultBufferSize = 212 << 10 // 212 KiB + + // DefaultMaxBufferSize is the default maximum permitted size of a + // send/receive buffer. + DefaultMaxBufferSize = 4 << 20 // 4 MiB + + // defaultTCPInvalidRateLimit is the default value for + // stack.TCPInvalidRateLimit. + defaultTCPInvalidRateLimit = 500 * time.Millisecond +) + +// ReceiveBufferSizeOption is used by stack.(Stack*).Option/SetOption to +// get/set the default, min and max receive buffer sizes. +type ReceiveBufferSizeOption struct { + Min int + Default int + Max int +} + +// TCPInvalidRateLimitOption is used by stack.(Stack*).Option/SetOption to get/set +// stack.tcpInvalidRateLimit. +type TCPInvalidRateLimitOption time.Duration + +// SetOption allows setting stack wide options. +func (s *Stack) SetOption(option any) tcpip.Error { + switch v := option.(type) { + case tcpip.SendBufferSizeOption: + // Make sure we don't allow lowering the buffer below minimum + // required for stack to work. + if v.Min < MinBufferSize { + return &tcpip.ErrInvalidOptionValue{} + } + + if v.Default < v.Min || v.Default > v.Max { + return &tcpip.ErrInvalidOptionValue{} + } + + s.mu.Lock() + s.sendBufferSize = v + s.mu.Unlock() + return nil + + case tcpip.ReceiveBufferSizeOption: + // Make sure we don't allow lowering the buffer below minimum + // required for stack to work. + if v.Min < MinBufferSize { + return &tcpip.ErrInvalidOptionValue{} + } + + if v.Default < v.Min || v.Default > v.Max { + return &tcpip.ErrInvalidOptionValue{} + } + + s.mu.Lock() + s.receiveBufferSize = v + s.mu.Unlock() + return nil + + case TCPInvalidRateLimitOption: + if v < 0 { + return &tcpip.ErrInvalidOptionValue{} + } + s.mu.Lock() + s.tcpInvalidRateLimit = time.Duration(v) + s.mu.Unlock() + return nil + + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// Option allows retrieving stack wide options. +func (s *Stack) Option(option any) tcpip.Error { + switch v := option.(type) { + case *tcpip.SendBufferSizeOption: + s.mu.RLock() + *v = s.sendBufferSize + s.mu.RUnlock() + return nil + + case *tcpip.ReceiveBufferSizeOption: + s.mu.RLock() + *v = s.receiveBufferSize + s.mu.RUnlock() + return nil + + case *TCPInvalidRateLimitOption: + s.mu.RLock() + *v = TCPInvalidRateLimitOption(s.tcpInvalidRateLimit) + s.mu.RUnlock() + return nil + + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} diff --git a/pkg/tcpip/stack/stack_state_autogen.go b/pkg/tcpip/stack/stack_state_autogen.go new file mode 100644 index 0000000..a9cd134 --- /dev/null +++ b/pkg/tcpip/stack/stack_state_autogen.go @@ -0,0 +1,2496 @@ +// automatically generated by stateify. + +package stack + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (r *addressStateRefs) StateTypeName() string { + return "pkg/tcpip/stack.addressStateRefs" +} + +func (r *addressStateRefs) StateFields() []string { + return []string{ + "refCount", + } +} + +func (r *addressStateRefs) beforeSave() {} + +// +checklocksignore +func (r *addressStateRefs) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.refCount) +} + +// +checklocksignore +func (r *addressStateRefs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.refCount) + stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) }) +} + +func (a *AddressableEndpointState) StateTypeName() string { + return "pkg/tcpip/stack.AddressableEndpointState" +} + +func (a *AddressableEndpointState) StateFields() []string { + return []string{ + "networkEndpoint", + "options", + } +} + +func (a *AddressableEndpointState) beforeSave() {} + +// +checklocksignore +func (a *AddressableEndpointState) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.networkEndpoint) + stateSinkObject.Save(1, &a.options) +} + +func (a *AddressableEndpointState) afterLoad(context.Context) {} + +// +checklocksignore +func (a *AddressableEndpointState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.networkEndpoint) + stateSourceObject.Load(1, &a.options) +} + +func (a *AddressableEndpointStateOptions) StateTypeName() string { + return "pkg/tcpip/stack.AddressableEndpointStateOptions" +} + +func (a *AddressableEndpointStateOptions) StateFields() []string { + return []string{ + "HiddenWhileDisabled", + } +} + +func (a *AddressableEndpointStateOptions) beforeSave() {} + +// +checklocksignore +func (a *AddressableEndpointStateOptions) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.HiddenWhileDisabled) +} + +func (a *AddressableEndpointStateOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (a *AddressableEndpointStateOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.HiddenWhileDisabled) +} + +func (p *bridgePort) StateTypeName() string { + return "pkg/tcpip/stack.bridgePort" +} + +func (p *bridgePort) StateFields() []string { + return []string{ + "bridge", + "nic", + } +} + +func (p *bridgePort) beforeSave() {} + +// +checklocksignore +func (p *bridgePort) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.bridge) + stateSinkObject.Save(1, &p.nic) +} + +func (p *bridgePort) afterLoad(context.Context) {} + +// +checklocksignore +func (p *bridgePort) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.bridge) + stateSourceObject.Load(1, &p.nic) +} + +func (b *BridgeEndpoint) StateTypeName() string { + return "pkg/tcpip/stack.BridgeEndpoint" +} + +func (b *BridgeEndpoint) StateFields() []string { + return []string{ + "ports", + "dispatcher", + "addr", + "attached", + "mtu", + "fdbTable", + "maxHeaderLength", + } +} + +func (b *BridgeEndpoint) beforeSave() {} + +// +checklocksignore +func (b *BridgeEndpoint) StateSave(stateSinkObject state.Sink) { + b.beforeSave() + stateSinkObject.Save(0, &b.ports) + stateSinkObject.Save(1, &b.dispatcher) + stateSinkObject.Save(2, &b.addr) + stateSinkObject.Save(3, &b.attached) + stateSinkObject.Save(4, &b.mtu) + stateSinkObject.Save(5, &b.fdbTable) + stateSinkObject.Save(6, &b.maxHeaderLength) +} + +func (b *BridgeEndpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (b *BridgeEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &b.ports) + stateSourceObject.Load(1, &b.dispatcher) + stateSourceObject.Load(2, &b.addr) + stateSourceObject.Load(3, &b.attached) + stateSourceObject.Load(4, &b.mtu) + stateSourceObject.Load(5, &b.fdbTable) + stateSourceObject.Load(6, &b.maxHeaderLength) +} + +func (t *tuple) StateTypeName() string { + return "pkg/tcpip/stack.tuple" +} + +func (t *tuple) StateFields() []string { + return []string{ + "tupleEntry", + "conn", + "reply", + "tupleID", + } +} + +func (t *tuple) beforeSave() {} + +// +checklocksignore +func (t *tuple) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.tupleEntry) + stateSinkObject.Save(1, &t.conn) + stateSinkObject.Save(2, &t.reply) + stateSinkObject.Save(3, &t.tupleID) +} + +func (t *tuple) afterLoad(context.Context) {} + +// +checklocksignore +func (t *tuple) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.tupleEntry) + stateSourceObject.Load(1, &t.conn) + stateSourceObject.Load(2, &t.reply) + stateSourceObject.Load(3, &t.tupleID) +} + +func (ti *tupleID) StateTypeName() string { + return "pkg/tcpip/stack.tupleID" +} + +func (ti *tupleID) StateFields() []string { + return []string{ + "srcAddr", + "srcPortOrEchoRequestIdent", + "dstAddr", + "dstPortOrEchoReplyIdent", + "transProto", + "netProto", + } +} + +func (ti *tupleID) beforeSave() {} + +// +checklocksignore +func (ti *tupleID) StateSave(stateSinkObject state.Sink) { + ti.beforeSave() + stateSinkObject.Save(0, &ti.srcAddr) + stateSinkObject.Save(1, &ti.srcPortOrEchoRequestIdent) + stateSinkObject.Save(2, &ti.dstAddr) + stateSinkObject.Save(3, &ti.dstPortOrEchoReplyIdent) + stateSinkObject.Save(4, &ti.transProto) + stateSinkObject.Save(5, &ti.netProto) +} + +func (ti *tupleID) afterLoad(context.Context) {} + +// +checklocksignore +func (ti *tupleID) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ti.srcAddr) + stateSourceObject.Load(1, &ti.srcPortOrEchoRequestIdent) + stateSourceObject.Load(2, &ti.dstAddr) + stateSourceObject.Load(3, &ti.dstPortOrEchoReplyIdent) + stateSourceObject.Load(4, &ti.transProto) + stateSourceObject.Load(5, &ti.netProto) +} + +func (cn *conn) StateTypeName() string { + return "pkg/tcpip/stack.conn" +} + +func (cn *conn) StateFields() []string { + return []string{ + "ct", + "original", + "reply", + "finalizeResult", + "sourceManip", + "destinationManip", + "tcb", + "lastUsed", + } +} + +func (cn *conn) beforeSave() {} + +// +checklocksignore +func (cn *conn) StateSave(stateSinkObject state.Sink) { + cn.beforeSave() + stateSinkObject.Save(0, &cn.ct) + stateSinkObject.Save(1, &cn.original) + stateSinkObject.Save(2, &cn.reply) + stateSinkObject.Save(3, &cn.finalizeResult) + stateSinkObject.Save(4, &cn.sourceManip) + stateSinkObject.Save(5, &cn.destinationManip) + stateSinkObject.Save(6, &cn.tcb) + stateSinkObject.Save(7, &cn.lastUsed) +} + +func (cn *conn) afterLoad(context.Context) {} + +// +checklocksignore +func (cn *conn) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &cn.ct) + stateSourceObject.Load(1, &cn.original) + stateSourceObject.Load(2, &cn.reply) + stateSourceObject.Load(3, &cn.finalizeResult) + stateSourceObject.Load(4, &cn.sourceManip) + stateSourceObject.Load(5, &cn.destinationManip) + stateSourceObject.Load(6, &cn.tcb) + stateSourceObject.Load(7, &cn.lastUsed) +} + +func (ct *ConnTrack) StateTypeName() string { + return "pkg/tcpip/stack.ConnTrack" +} + +func (ct *ConnTrack) StateFields() []string { + return []string{ + "seed", + "clock", + "buckets", + } +} + +func (ct *ConnTrack) beforeSave() {} + +// +checklocksignore +func (ct *ConnTrack) StateSave(stateSinkObject state.Sink) { + ct.beforeSave() + stateSinkObject.Save(0, &ct.seed) + stateSinkObject.Save(1, &ct.clock) + stateSinkObject.Save(2, &ct.buckets) +} + +func (ct *ConnTrack) afterLoad(context.Context) {} + +// +checklocksignore +func (ct *ConnTrack) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ct.seed) + stateSourceObject.Load(1, &ct.clock) + stateSourceObject.Load(2, &ct.buckets) +} + +func (bkt *bucket) StateTypeName() string { + return "pkg/tcpip/stack.bucket" +} + +func (bkt *bucket) StateFields() []string { + return []string{ + "tuples", + } +} + +func (bkt *bucket) beforeSave() {} + +// +checklocksignore +func (bkt *bucket) StateSave(stateSinkObject state.Sink) { + bkt.beforeSave() + stateSinkObject.Save(0, &bkt.tuples) +} + +func (bkt *bucket) afterLoad(context.Context) {} + +// +checklocksignore +func (bkt *bucket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &bkt.tuples) +} + +func (l *ICMPRateLimiter) StateTypeName() string { + return "pkg/tcpip/stack.ICMPRateLimiter" +} + +func (l *ICMPRateLimiter) StateFields() []string { + return []string{ + "clock", + } +} + +func (l *ICMPRateLimiter) beforeSave() {} + +// +checklocksignore +func (l *ICMPRateLimiter) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.clock) +} + +func (l *ICMPRateLimiter) afterLoad(context.Context) {} + +// +checklocksignore +func (l *ICMPRateLimiter) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.clock) +} + +func (a *AcceptTarget) StateTypeName() string { + return "pkg/tcpip/stack.AcceptTarget" +} + +func (a *AcceptTarget) StateFields() []string { + return []string{ + "NetworkProtocol", + } +} + +func (a *AcceptTarget) beforeSave() {} + +// +checklocksignore +func (a *AcceptTarget) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.NetworkProtocol) +} + +func (a *AcceptTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (a *AcceptTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.NetworkProtocol) +} + +func (d *DropTarget) StateTypeName() string { + return "pkg/tcpip/stack.DropTarget" +} + +func (d *DropTarget) StateFields() []string { + return []string{ + "NetworkProtocol", + } +} + +func (d *DropTarget) beforeSave() {} + +// +checklocksignore +func (d *DropTarget) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.NetworkProtocol) +} + +func (d *DropTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (d *DropTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.NetworkProtocol) +} + +func (rt *RejectIPv4Target) StateTypeName() string { + return "pkg/tcpip/stack.RejectIPv4Target" +} + +func (rt *RejectIPv4Target) StateFields() []string { + return []string{ + "Handler", + "RejectWith", + } +} + +func (rt *RejectIPv4Target) beforeSave() {} + +// +checklocksignore +func (rt *RejectIPv4Target) StateSave(stateSinkObject state.Sink) { + rt.beforeSave() + stateSinkObject.Save(0, &rt.Handler) + stateSinkObject.Save(1, &rt.RejectWith) +} + +func (rt *RejectIPv4Target) afterLoad(context.Context) {} + +// +checklocksignore +func (rt *RejectIPv4Target) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &rt.Handler) + stateSourceObject.Load(1, &rt.RejectWith) +} + +func (rt *RejectIPv6Target) StateTypeName() string { + return "pkg/tcpip/stack.RejectIPv6Target" +} + +func (rt *RejectIPv6Target) StateFields() []string { + return []string{ + "Handler", + "RejectWith", + } +} + +func (rt *RejectIPv6Target) beforeSave() {} + +// +checklocksignore +func (rt *RejectIPv6Target) StateSave(stateSinkObject state.Sink) { + rt.beforeSave() + stateSinkObject.Save(0, &rt.Handler) + stateSinkObject.Save(1, &rt.RejectWith) +} + +func (rt *RejectIPv6Target) afterLoad(context.Context) {} + +// +checklocksignore +func (rt *RejectIPv6Target) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &rt.Handler) + stateSourceObject.Load(1, &rt.RejectWith) +} + +func (e *ErrorTarget) StateTypeName() string { + return "pkg/tcpip/stack.ErrorTarget" +} + +func (e *ErrorTarget) StateFields() []string { + return []string{ + "NetworkProtocol", + } +} + +func (e *ErrorTarget) beforeSave() {} + +// +checklocksignore +func (e *ErrorTarget) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.NetworkProtocol) +} + +func (e *ErrorTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrorTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.NetworkProtocol) +} + +func (u *UserChainTarget) StateTypeName() string { + return "pkg/tcpip/stack.UserChainTarget" +} + +func (u *UserChainTarget) StateFields() []string { + return []string{ + "Name", + "NetworkProtocol", + } +} + +func (u *UserChainTarget) beforeSave() {} + +// +checklocksignore +func (u *UserChainTarget) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.Name) + stateSinkObject.Save(1, &u.NetworkProtocol) +} + +func (u *UserChainTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (u *UserChainTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.Name) + stateSourceObject.Load(1, &u.NetworkProtocol) +} + +func (r *ReturnTarget) StateTypeName() string { + return "pkg/tcpip/stack.ReturnTarget" +} + +func (r *ReturnTarget) StateFields() []string { + return []string{ + "NetworkProtocol", + } +} + +func (r *ReturnTarget) beforeSave() {} + +// +checklocksignore +func (r *ReturnTarget) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.NetworkProtocol) +} + +func (r *ReturnTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (r *ReturnTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.NetworkProtocol) +} + +func (rt *DNATTarget) StateTypeName() string { + return "pkg/tcpip/stack.DNATTarget" +} + +func (rt *DNATTarget) StateFields() []string { + return []string{ + "Addr", + "Port", + "NetworkProtocol", + "ChangeAddress", + "ChangePort", + } +} + +func (rt *DNATTarget) beforeSave() {} + +// +checklocksignore +func (rt *DNATTarget) StateSave(stateSinkObject state.Sink) { + rt.beforeSave() + stateSinkObject.Save(0, &rt.Addr) + stateSinkObject.Save(1, &rt.Port) + stateSinkObject.Save(2, &rt.NetworkProtocol) + stateSinkObject.Save(3, &rt.ChangeAddress) + stateSinkObject.Save(4, &rt.ChangePort) +} + +func (rt *DNATTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (rt *DNATTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &rt.Addr) + stateSourceObject.Load(1, &rt.Port) + stateSourceObject.Load(2, &rt.NetworkProtocol) + stateSourceObject.Load(3, &rt.ChangeAddress) + stateSourceObject.Load(4, &rt.ChangePort) +} + +func (rt *RedirectTarget) StateTypeName() string { + return "pkg/tcpip/stack.RedirectTarget" +} + +func (rt *RedirectTarget) StateFields() []string { + return []string{ + "Port", + "NetworkProtocol", + } +} + +func (rt *RedirectTarget) beforeSave() {} + +// +checklocksignore +func (rt *RedirectTarget) StateSave(stateSinkObject state.Sink) { + rt.beforeSave() + stateSinkObject.Save(0, &rt.Port) + stateSinkObject.Save(1, &rt.NetworkProtocol) +} + +func (rt *RedirectTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (rt *RedirectTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &rt.Port) + stateSourceObject.Load(1, &rt.NetworkProtocol) +} + +func (st *SNATTarget) StateTypeName() string { + return "pkg/tcpip/stack.SNATTarget" +} + +func (st *SNATTarget) StateFields() []string { + return []string{ + "Addr", + "Port", + "NetworkProtocol", + "ChangeAddress", + "ChangePort", + } +} + +func (st *SNATTarget) beforeSave() {} + +// +checklocksignore +func (st *SNATTarget) StateSave(stateSinkObject state.Sink) { + st.beforeSave() + stateSinkObject.Save(0, &st.Addr) + stateSinkObject.Save(1, &st.Port) + stateSinkObject.Save(2, &st.NetworkProtocol) + stateSinkObject.Save(3, &st.ChangeAddress) + stateSinkObject.Save(4, &st.ChangePort) +} + +func (st *SNATTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (st *SNATTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &st.Addr) + stateSourceObject.Load(1, &st.Port) + stateSourceObject.Load(2, &st.NetworkProtocol) + stateSourceObject.Load(3, &st.ChangeAddress) + stateSourceObject.Load(4, &st.ChangePort) +} + +func (mt *MasqueradeTarget) StateTypeName() string { + return "pkg/tcpip/stack.MasqueradeTarget" +} + +func (mt *MasqueradeTarget) StateFields() []string { + return []string{ + "NetworkProtocol", + } +} + +func (mt *MasqueradeTarget) beforeSave() {} + +// +checklocksignore +func (mt *MasqueradeTarget) StateSave(stateSinkObject state.Sink) { + mt.beforeSave() + stateSinkObject.Save(0, &mt.NetworkProtocol) +} + +func (mt *MasqueradeTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (mt *MasqueradeTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &mt.NetworkProtocol) +} + +func (it *IPTables) StateTypeName() string { + return "pkg/tcpip/stack.IPTables" +} + +func (it *IPTables) StateFields() []string { + return []string{ + "connections", + "reaper", + "v4Tables", + "v6Tables", + "modified", + } +} + +// +checklocksignore +func (it *IPTables) StateSave(stateSinkObject state.Sink) { + it.beforeSave() + stateSinkObject.Save(0, &it.connections) + stateSinkObject.Save(1, &it.reaper) + stateSinkObject.Save(2, &it.v4Tables) + stateSinkObject.Save(3, &it.v6Tables) + stateSinkObject.Save(4, &it.modified) +} + +// +checklocksignore +func (it *IPTables) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &it.connections) + stateSourceObject.Load(1, &it.reaper) + stateSourceObject.Load(2, &it.v4Tables) + stateSourceObject.Load(3, &it.v6Tables) + stateSourceObject.Load(4, &it.modified) + stateSourceObject.AfterLoad(func() { it.afterLoad(ctx) }) +} + +func (table *Table) StateTypeName() string { + return "pkg/tcpip/stack.Table" +} + +func (table *Table) StateFields() []string { + return []string{ + "Rules", + "BuiltinChains", + "Underflows", + } +} + +func (table *Table) beforeSave() {} + +// +checklocksignore +func (table *Table) StateSave(stateSinkObject state.Sink) { + table.beforeSave() + stateSinkObject.Save(0, &table.Rules) + stateSinkObject.Save(1, &table.BuiltinChains) + stateSinkObject.Save(2, &table.Underflows) +} + +func (table *Table) afterLoad(context.Context) {} + +// +checklocksignore +func (table *Table) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &table.Rules) + stateSourceObject.Load(1, &table.BuiltinChains) + stateSourceObject.Load(2, &table.Underflows) +} + +func (r *Rule) StateTypeName() string { + return "pkg/tcpip/stack.Rule" +} + +func (r *Rule) StateFields() []string { + return []string{ + "Filter", + "Matchers", + "Target", + } +} + +func (r *Rule) beforeSave() {} + +// +checklocksignore +func (r *Rule) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.Filter) + stateSinkObject.Save(1, &r.Matchers) + stateSinkObject.Save(2, &r.Target) +} + +func (r *Rule) afterLoad(context.Context) {} + +// +checklocksignore +func (r *Rule) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.Filter) + stateSourceObject.Load(1, &r.Matchers) + stateSourceObject.Load(2, &r.Target) +} + +func (fl *IPHeaderFilter) StateTypeName() string { + return "pkg/tcpip/stack.IPHeaderFilter" +} + +func (fl *IPHeaderFilter) StateFields() []string { + return []string{ + "Protocol", + "CheckProtocol", + "Dst", + "DstMask", + "DstInvert", + "Src", + "SrcMask", + "SrcInvert", + "InputInterface", + "InputInterfaceMask", + "InputInterfaceInvert", + "OutputInterface", + "OutputInterfaceMask", + "OutputInterfaceInvert", + } +} + +func (fl *IPHeaderFilter) beforeSave() {} + +// +checklocksignore +func (fl *IPHeaderFilter) StateSave(stateSinkObject state.Sink) { + fl.beforeSave() + stateSinkObject.Save(0, &fl.Protocol) + stateSinkObject.Save(1, &fl.CheckProtocol) + stateSinkObject.Save(2, &fl.Dst) + stateSinkObject.Save(3, &fl.DstMask) + stateSinkObject.Save(4, &fl.DstInvert) + stateSinkObject.Save(5, &fl.Src) + stateSinkObject.Save(6, &fl.SrcMask) + stateSinkObject.Save(7, &fl.SrcInvert) + stateSinkObject.Save(8, &fl.InputInterface) + stateSinkObject.Save(9, &fl.InputInterfaceMask) + stateSinkObject.Save(10, &fl.InputInterfaceInvert) + stateSinkObject.Save(11, &fl.OutputInterface) + stateSinkObject.Save(12, &fl.OutputInterfaceMask) + stateSinkObject.Save(13, &fl.OutputInterfaceInvert) +} + +func (fl *IPHeaderFilter) afterLoad(context.Context) {} + +// +checklocksignore +func (fl *IPHeaderFilter) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &fl.Protocol) + stateSourceObject.Load(1, &fl.CheckProtocol) + stateSourceObject.Load(2, &fl.Dst) + stateSourceObject.Load(3, &fl.DstMask) + stateSourceObject.Load(4, &fl.DstInvert) + stateSourceObject.Load(5, &fl.Src) + stateSourceObject.Load(6, &fl.SrcMask) + stateSourceObject.Load(7, &fl.SrcInvert) + stateSourceObject.Load(8, &fl.InputInterface) + stateSourceObject.Load(9, &fl.InputInterfaceMask) + stateSourceObject.Load(10, &fl.InputInterfaceInvert) + stateSourceObject.Load(11, &fl.OutputInterface) + stateSourceObject.Load(12, &fl.OutputInterfaceMask) + stateSourceObject.Load(13, &fl.OutputInterfaceInvert) +} + +func (d *dynamicCacheEntry) StateTypeName() string { + return "pkg/tcpip/stack.dynamicCacheEntry" +} + +func (d *dynamicCacheEntry) StateFields() []string { + return []string{ + "lru", + "count", + } +} + +func (d *dynamicCacheEntry) beforeSave() {} + +// +checklocksignore +func (d *dynamicCacheEntry) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.lru) + stateSinkObject.Save(1, &d.count) +} + +func (d *dynamicCacheEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (d *dynamicCacheEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.lru) + stateSourceObject.Load(1, &d.count) +} + +func (n *neighborCacheMu) StateTypeName() string { + return "pkg/tcpip/stack.neighborCacheMu" +} + +func (n *neighborCacheMu) StateFields() []string { + return []string{ + "cache", + "dynamic", + } +} + +func (n *neighborCacheMu) beforeSave() {} + +// +checklocksignore +func (n *neighborCacheMu) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.cache) + stateSinkObject.Save(1, &n.dynamic) +} + +func (n *neighborCacheMu) afterLoad(context.Context) {} + +// +checklocksignore +func (n *neighborCacheMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.cache) + stateSourceObject.Load(1, &n.dynamic) +} + +func (n *neighborCache) StateTypeName() string { + return "pkg/tcpip/stack.neighborCache" +} + +func (n *neighborCache) StateFields() []string { + return []string{ + "nic", + "state", + "linkRes", + "mu", + } +} + +func (n *neighborCache) beforeSave() {} + +// +checklocksignore +func (n *neighborCache) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.nic) + stateSinkObject.Save(1, &n.state) + stateSinkObject.Save(2, &n.linkRes) + stateSinkObject.Save(3, &n.mu) +} + +func (n *neighborCache) afterLoad(context.Context) {} + +// +checklocksignore +func (n *neighborCache) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.nic) + stateSourceObject.Load(1, &n.state) + stateSourceObject.Load(2, &n.linkRes) + stateSourceObject.Load(3, &n.mu) +} + +func (n *NeighborEntry) StateTypeName() string { + return "pkg/tcpip/stack.NeighborEntry" +} + +func (n *NeighborEntry) StateFields() []string { + return []string{ + "Addr", + "LinkAddr", + "State", + "UpdatedAt", + } +} + +func (n *NeighborEntry) beforeSave() {} + +// +checklocksignore +func (n *NeighborEntry) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.Addr) + stateSinkObject.Save(1, &n.LinkAddr) + stateSinkObject.Save(2, &n.State) + stateSinkObject.Save(3, &n.UpdatedAt) +} + +func (n *NeighborEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NeighborEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.Addr) + stateSourceObject.Load(1, &n.LinkAddr) + stateSourceObject.Load(2, &n.State) + stateSourceObject.Load(3, &n.UpdatedAt) +} + +func (t *timer) StateTypeName() string { + return "pkg/tcpip/stack.timer" +} + +func (t *timer) StateFields() []string { + return []string{ + "done", + } +} + +func (t *timer) beforeSave() {} + +// +checklocksignore +func (t *timer) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.done) +} + +func (t *timer) afterLoad(context.Context) {} + +// +checklocksignore +func (t *timer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.done) +} + +func (n *neighborEntryMu) StateTypeName() string { + return "pkg/tcpip/stack.neighborEntryMu" +} + +func (n *neighborEntryMu) StateFields() []string { + return []string{ + "neigh", + "isRouter", + "timer", + } +} + +func (n *neighborEntryMu) beforeSave() {} + +// +checklocksignore +func (n *neighborEntryMu) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.neigh) + stateSinkObject.Save(1, &n.isRouter) + stateSinkObject.Save(2, &n.timer) +} + +func (n *neighborEntryMu) afterLoad(context.Context) {} + +// +checklocksignore +func (n *neighborEntryMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.neigh) + stateSourceObject.Load(1, &n.isRouter) + stateSourceObject.Load(2, &n.timer) +} + +func (e *neighborEntry) StateTypeName() string { + return "pkg/tcpip/stack.neighborEntry" +} + +func (e *neighborEntry) StateFields() []string { + return []string{ + "neighborEntryEntry", + "cache", + "nudState", + "mu", + } +} + +func (e *neighborEntry) beforeSave() {} + +// +checklocksignore +func (e *neighborEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.neighborEntryEntry) + stateSinkObject.Save(1, &e.cache) + stateSinkObject.Save(2, &e.nudState) + stateSinkObject.Save(3, &e.mu) +} + +func (e *neighborEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *neighborEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.neighborEntryEntry) + stateSourceObject.Load(1, &e.cache) + stateSourceObject.Load(2, &e.nudState) + stateSourceObject.Load(3, &e.mu) +} + +func (l *neighborEntryList) StateTypeName() string { + return "pkg/tcpip/stack.neighborEntryList" +} + +func (l *neighborEntryList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *neighborEntryList) beforeSave() {} + +// +checklocksignore +func (l *neighborEntryList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *neighborEntryList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *neighborEntryList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *neighborEntryEntry) StateTypeName() string { + return "pkg/tcpip/stack.neighborEntryEntry" +} + +func (e *neighborEntryEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *neighborEntryEntry) beforeSave() {} + +// +checklocksignore +func (e *neighborEntryEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *neighborEntryEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *neighborEntryEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func (l *linkResolver) StateTypeName() string { + return "pkg/tcpip/stack.linkResolver" +} + +func (l *linkResolver) StateFields() []string { + return []string{ + "resolver", + "neigh", + } +} + +func (l *linkResolver) beforeSave() {} + +// +checklocksignore +func (l *linkResolver) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.resolver) + stateSinkObject.Save(1, &l.neigh) +} + +func (l *linkResolver) afterLoad(context.Context) {} + +// +checklocksignore +func (l *linkResolver) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.resolver) + stateSourceObject.Load(1, &l.neigh) +} + +func (n *nic) StateTypeName() string { + return "pkg/tcpip/stack.nic" +} + +func (n *nic) StateFields() []string { + return []string{ + "NetworkLinkEndpoint", + "stack", + "id", + "name", + "context", + "stats", + "networkEndpoints", + "linkAddrResolvers", + "duplicateAddressDetectors", + "enabled", + "spoofing", + "promiscuous", + "linkResQueue", + "packetEPs", + "qDisc", + "deliverLinkPackets", + "Primary", + "experimentIPOptionEnabled", + } +} + +func (n *nic) beforeSave() {} + +// +checklocksignore +func (n *nic) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.NetworkLinkEndpoint) + stateSinkObject.Save(1, &n.stack) + stateSinkObject.Save(2, &n.id) + stateSinkObject.Save(3, &n.name) + stateSinkObject.Save(4, &n.context) + stateSinkObject.Save(5, &n.stats) + stateSinkObject.Save(6, &n.networkEndpoints) + stateSinkObject.Save(7, &n.linkAddrResolvers) + stateSinkObject.Save(8, &n.duplicateAddressDetectors) + stateSinkObject.Save(9, &n.enabled) + stateSinkObject.Save(10, &n.spoofing) + stateSinkObject.Save(11, &n.promiscuous) + stateSinkObject.Save(12, &n.linkResQueue) + stateSinkObject.Save(13, &n.packetEPs) + stateSinkObject.Save(14, &n.qDisc) + stateSinkObject.Save(15, &n.deliverLinkPackets) + stateSinkObject.Save(16, &n.Primary) + stateSinkObject.Save(17, &n.experimentIPOptionEnabled) +} + +func (n *nic) afterLoad(context.Context) {} + +// +checklocksignore +func (n *nic) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.NetworkLinkEndpoint) + stateSourceObject.Load(1, &n.stack) + stateSourceObject.Load(2, &n.id) + stateSourceObject.Load(3, &n.name) + stateSourceObject.Load(4, &n.context) + stateSourceObject.Load(5, &n.stats) + stateSourceObject.Load(6, &n.networkEndpoints) + stateSourceObject.Load(7, &n.linkAddrResolvers) + stateSourceObject.Load(8, &n.duplicateAddressDetectors) + stateSourceObject.Load(9, &n.enabled) + stateSourceObject.Load(10, &n.spoofing) + stateSourceObject.Load(11, &n.promiscuous) + stateSourceObject.Load(12, &n.linkResQueue) + stateSourceObject.Load(13, &n.packetEPs) + stateSourceObject.Load(14, &n.qDisc) + stateSourceObject.Load(15, &n.deliverLinkPackets) + stateSourceObject.Load(16, &n.Primary) + stateSourceObject.Load(17, &n.experimentIPOptionEnabled) +} + +func (p *packetEndpointList) StateTypeName() string { + return "pkg/tcpip/stack.packetEndpointList" +} + +func (p *packetEndpointList) StateFields() []string { + return []string{ + "eps", + } +} + +func (p *packetEndpointList) beforeSave() {} + +// +checklocksignore +func (p *packetEndpointList) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.eps) +} + +func (p *packetEndpointList) afterLoad(context.Context) {} + +// +checklocksignore +func (p *packetEndpointList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.eps) +} + +func (qDisc *delegatingQueueingDiscipline) StateTypeName() string { + return "pkg/tcpip/stack.delegatingQueueingDiscipline" +} + +func (qDisc *delegatingQueueingDiscipline) StateFields() []string { + return []string{ + "LinkWriter", + } +} + +func (qDisc *delegatingQueueingDiscipline) beforeSave() {} + +// +checklocksignore +func (qDisc *delegatingQueueingDiscipline) StateSave(stateSinkObject state.Sink) { + qDisc.beforeSave() + stateSinkObject.Save(0, &qDisc.LinkWriter) +} + +func (qDisc *delegatingQueueingDiscipline) afterLoad(context.Context) {} + +// +checklocksignore +func (qDisc *delegatingQueueingDiscipline) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &qDisc.LinkWriter) +} + +func (s *sharedStats) StateTypeName() string { + return "pkg/tcpip/stack.sharedStats" +} + +func (s *sharedStats) StateFields() []string { + return []string{ + "local", + "multiCounterNICStats", + } +} + +func (s *sharedStats) beforeSave() {} + +// +checklocksignore +func (s *sharedStats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.local) + stateSinkObject.Save(1, &s.multiCounterNICStats) +} + +func (s *sharedStats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *sharedStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.local) + stateSourceObject.Load(1, &s.multiCounterNICStats) +} + +func (m *multiCounterNICPacketStats) StateTypeName() string { + return "pkg/tcpip/stack.multiCounterNICPacketStats" +} + +func (m *multiCounterNICPacketStats) StateFields() []string { + return []string{ + "packets", + "bytes", + } +} + +func (m *multiCounterNICPacketStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterNICPacketStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.packets) + stateSinkObject.Save(1, &m.bytes) +} + +func (m *multiCounterNICPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterNICPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.packets) + stateSourceObject.Load(1, &m.bytes) +} + +func (m *multiCounterNICNeighborStats) StateTypeName() string { + return "pkg/tcpip/stack.multiCounterNICNeighborStats" +} + +func (m *multiCounterNICNeighborStats) StateFields() []string { + return []string{ + "unreachableEntryLookups", + "droppedConfirmationForNoninitiatedNeighbor", + "droppedInvalidLinkAddressConfirmations", + } +} + +func (m *multiCounterNICNeighborStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterNICNeighborStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.unreachableEntryLookups) + stateSinkObject.Save(1, &m.droppedConfirmationForNoninitiatedNeighbor) + stateSinkObject.Save(2, &m.droppedInvalidLinkAddressConfirmations) +} + +func (m *multiCounterNICNeighborStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterNICNeighborStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.unreachableEntryLookups) + stateSourceObject.Load(1, &m.droppedConfirmationForNoninitiatedNeighbor) + stateSourceObject.Load(2, &m.droppedInvalidLinkAddressConfirmations) +} + +func (m *multiCounterNICStats) StateTypeName() string { + return "pkg/tcpip/stack.multiCounterNICStats" +} + +func (m *multiCounterNICStats) StateFields() []string { + return []string{ + "unknownL3ProtocolRcvdPacketCounts", + "unknownL4ProtocolRcvdPacketCounts", + "malformedL4RcvdPackets", + "tx", + "txPacketsDroppedNoBufferSpace", + "rx", + "disabledRx", + "neighbor", + } +} + +func (m *multiCounterNICStats) beforeSave() {} + +// +checklocksignore +func (m *multiCounterNICStats) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.unknownL3ProtocolRcvdPacketCounts) + stateSinkObject.Save(1, &m.unknownL4ProtocolRcvdPacketCounts) + stateSinkObject.Save(2, &m.malformedL4RcvdPackets) + stateSinkObject.Save(3, &m.tx) + stateSinkObject.Save(4, &m.txPacketsDroppedNoBufferSpace) + stateSinkObject.Save(5, &m.rx) + stateSinkObject.Save(6, &m.disabledRx) + stateSinkObject.Save(7, &m.neighbor) +} + +func (m *multiCounterNICStats) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multiCounterNICStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.unknownL3ProtocolRcvdPacketCounts) + stateSourceObject.Load(1, &m.unknownL4ProtocolRcvdPacketCounts) + stateSourceObject.Load(2, &m.malformedL4RcvdPackets) + stateSourceObject.Load(3, &m.tx) + stateSourceObject.Load(4, &m.txPacketsDroppedNoBufferSpace) + stateSourceObject.Load(5, &m.rx) + stateSourceObject.Load(6, &m.disabledRx) + stateSourceObject.Load(7, &m.neighbor) +} + +func (c *NUDConfigurations) StateTypeName() string { + return "pkg/tcpip/stack.NUDConfigurations" +} + +func (c *NUDConfigurations) StateFields() []string { + return []string{ + "BaseReachableTime", + "LearnBaseReachableTime", + "MinRandomFactor", + "MaxRandomFactor", + "RetransmitTimer", + "LearnRetransmitTimer", + "DelayFirstProbeTime", + "MaxMulticastProbes", + "MaxUnicastProbes", + "MaxAnycastDelayTime", + "MaxReachabilityConfirmations", + } +} + +func (c *NUDConfigurations) beforeSave() {} + +// +checklocksignore +func (c *NUDConfigurations) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.BaseReachableTime) + stateSinkObject.Save(1, &c.LearnBaseReachableTime) + stateSinkObject.Save(2, &c.MinRandomFactor) + stateSinkObject.Save(3, &c.MaxRandomFactor) + stateSinkObject.Save(4, &c.RetransmitTimer) + stateSinkObject.Save(5, &c.LearnRetransmitTimer) + stateSinkObject.Save(6, &c.DelayFirstProbeTime) + stateSinkObject.Save(7, &c.MaxMulticastProbes) + stateSinkObject.Save(8, &c.MaxUnicastProbes) + stateSinkObject.Save(9, &c.MaxAnycastDelayTime) + stateSinkObject.Save(10, &c.MaxReachabilityConfirmations) +} + +func (c *NUDConfigurations) afterLoad(context.Context) {} + +// +checklocksignore +func (c *NUDConfigurations) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.BaseReachableTime) + stateSourceObject.Load(1, &c.LearnBaseReachableTime) + stateSourceObject.Load(2, &c.MinRandomFactor) + stateSourceObject.Load(3, &c.MaxRandomFactor) + stateSourceObject.Load(4, &c.RetransmitTimer) + stateSourceObject.Load(5, &c.LearnRetransmitTimer) + stateSourceObject.Load(6, &c.DelayFirstProbeTime) + stateSourceObject.Load(7, &c.MaxMulticastProbes) + stateSourceObject.Load(8, &c.MaxUnicastProbes) + stateSourceObject.Load(9, &c.MaxAnycastDelayTime) + stateSourceObject.Load(10, &c.MaxReachabilityConfirmations) +} + +func (n *nudStateMu) StateTypeName() string { + return "pkg/tcpip/stack.nudStateMu" +} + +func (n *nudStateMu) StateFields() []string { + return []string{ + "config", + "reachableTime", + "expiration", + "prevBaseReachableTime", + "prevMinRandomFactor", + "prevMaxRandomFactor", + } +} + +func (n *nudStateMu) beforeSave() {} + +// +checklocksignore +func (n *nudStateMu) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.config) + stateSinkObject.Save(1, &n.reachableTime) + stateSinkObject.Save(2, &n.expiration) + stateSinkObject.Save(3, &n.prevBaseReachableTime) + stateSinkObject.Save(4, &n.prevMinRandomFactor) + stateSinkObject.Save(5, &n.prevMaxRandomFactor) +} + +func (n *nudStateMu) afterLoad(context.Context) {} + +// +checklocksignore +func (n *nudStateMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.config) + stateSourceObject.Load(1, &n.reachableTime) + stateSourceObject.Load(2, &n.expiration) + stateSourceObject.Load(3, &n.prevBaseReachableTime) + stateSourceObject.Load(4, &n.prevMinRandomFactor) + stateSourceObject.Load(5, &n.prevMaxRandomFactor) +} + +func (s *NUDState) StateTypeName() string { + return "pkg/tcpip/stack.NUDState" +} + +func (s *NUDState) StateFields() []string { + return []string{ + "clock", + "mu", + } +} + +func (s *NUDState) beforeSave() {} + +// +checklocksignore +func (s *NUDState) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.clock) + stateSinkObject.Save(1, &s.mu) +} + +func (s *NUDState) afterLoad(context.Context) {} + +// +checklocksignore +func (s *NUDState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.clock) + stateSourceObject.Load(1, &s.mu) +} + +func (pk *PacketBuffer) StateTypeName() string { + return "pkg/tcpip/stack.PacketBuffer" +} + +func (pk *PacketBuffer) StateFields() []string { + return []string{ + "packetBufferRefs", + "buf", + "reserved", + "pushed", + "consumed", + "headers", + "NetworkProtocolNumber", + "TransportProtocolNumber", + "Hash", + "Owner", + "EgressRoute", + "GSOOptions", + "snatDone", + "dnatDone", + "PktType", + "NICID", + "RXChecksumValidated", + "NetworkPacketInfo", + "tuple", + } +} + +func (pk *PacketBuffer) beforeSave() {} + +// +checklocksignore +func (pk *PacketBuffer) StateSave(stateSinkObject state.Sink) { + pk.beforeSave() + stateSinkObject.Save(0, &pk.packetBufferRefs) + stateSinkObject.Save(1, &pk.buf) + stateSinkObject.Save(2, &pk.reserved) + stateSinkObject.Save(3, &pk.pushed) + stateSinkObject.Save(4, &pk.consumed) + stateSinkObject.Save(5, &pk.headers) + stateSinkObject.Save(6, &pk.NetworkProtocolNumber) + stateSinkObject.Save(7, &pk.TransportProtocolNumber) + stateSinkObject.Save(8, &pk.Hash) + stateSinkObject.Save(9, &pk.Owner) + stateSinkObject.Save(10, &pk.EgressRoute) + stateSinkObject.Save(11, &pk.GSOOptions) + stateSinkObject.Save(12, &pk.snatDone) + stateSinkObject.Save(13, &pk.dnatDone) + stateSinkObject.Save(14, &pk.PktType) + stateSinkObject.Save(15, &pk.NICID) + stateSinkObject.Save(16, &pk.RXChecksumValidated) + stateSinkObject.Save(17, &pk.NetworkPacketInfo) + stateSinkObject.Save(18, &pk.tuple) +} + +func (pk *PacketBuffer) afterLoad(context.Context) {} + +// +checklocksignore +func (pk *PacketBuffer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &pk.packetBufferRefs) + stateSourceObject.Load(1, &pk.buf) + stateSourceObject.Load(2, &pk.reserved) + stateSourceObject.Load(3, &pk.pushed) + stateSourceObject.Load(4, &pk.consumed) + stateSourceObject.Load(5, &pk.headers) + stateSourceObject.Load(6, &pk.NetworkProtocolNumber) + stateSourceObject.Load(7, &pk.TransportProtocolNumber) + stateSourceObject.Load(8, &pk.Hash) + stateSourceObject.Load(9, &pk.Owner) + stateSourceObject.Load(10, &pk.EgressRoute) + stateSourceObject.Load(11, &pk.GSOOptions) + stateSourceObject.Load(12, &pk.snatDone) + stateSourceObject.Load(13, &pk.dnatDone) + stateSourceObject.Load(14, &pk.PktType) + stateSourceObject.Load(15, &pk.NICID) + stateSourceObject.Load(16, &pk.RXChecksumValidated) + stateSourceObject.Load(17, &pk.NetworkPacketInfo) + stateSourceObject.Load(18, &pk.tuple) +} + +func (h *headerInfo) StateTypeName() string { + return "pkg/tcpip/stack.headerInfo" +} + +func (h *headerInfo) StateFields() []string { + return []string{ + "offset", + "length", + } +} + +func (h *headerInfo) beforeSave() {} + +// +checklocksignore +func (h *headerInfo) StateSave(stateSinkObject state.Sink) { + h.beforeSave() + stateSinkObject.Save(0, &h.offset) + stateSinkObject.Save(1, &h.length) +} + +func (h *headerInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (h *headerInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &h.offset) + stateSourceObject.Load(1, &h.length) +} + +func (d *PacketData) StateTypeName() string { + return "pkg/tcpip/stack.PacketData" +} + +func (d *PacketData) StateFields() []string { + return []string{ + "pk", + } +} + +func (d *PacketData) beforeSave() {} + +// +checklocksignore +func (d *PacketData) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.pk) +} + +func (d *PacketData) afterLoad(context.Context) {} + +// +checklocksignore +func (d *PacketData) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.pk) +} + +func (pl *PacketBufferList) StateTypeName() string { + return "pkg/tcpip/stack.PacketBufferList" +} + +func (pl *PacketBufferList) StateFields() []string { + return []string{ + "pbs", + } +} + +func (pl *PacketBufferList) beforeSave() {} + +// +checklocksignore +func (pl *PacketBufferList) StateSave(stateSinkObject state.Sink) { + pl.beforeSave() + stateSinkObject.Save(0, &pl.pbs) +} + +func (pl *PacketBufferList) afterLoad(context.Context) {} + +// +checklocksignore +func (pl *PacketBufferList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &pl.pbs) +} + +func (r *packetBufferRefs) StateTypeName() string { + return "pkg/tcpip/stack.packetBufferRefs" +} + +func (r *packetBufferRefs) StateFields() []string { + return []string{ + "refCount", + } +} + +func (r *packetBufferRefs) beforeSave() {} + +// +checklocksignore +func (r *packetBufferRefs) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.refCount) +} + +// +checklocksignore +func (r *packetBufferRefs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.refCount) + stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) }) +} + +func (p *pendingPacket) StateTypeName() string { + return "pkg/tcpip/stack.pendingPacket" +} + +func (p *pendingPacket) StateFields() []string { + return []string{ + "routeInfo", + "pkt", + } +} + +func (p *pendingPacket) beforeSave() {} + +// +checklocksignore +func (p *pendingPacket) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.routeInfo) + stateSinkObject.Save(1, &p.pkt) +} + +func (p *pendingPacket) afterLoad(context.Context) {} + +// +checklocksignore +func (p *pendingPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.routeInfo) + stateSourceObject.Load(1, &p.pkt) +} + +func (f *packetsPendingLinkResolution) StateTypeName() string { + return "pkg/tcpip/stack.packetsPendingLinkResolution" +} + +func (f *packetsPendingLinkResolution) StateFields() []string { + return []string{ + "nic", + } +} + +func (f *packetsPendingLinkResolution) beforeSave() {} + +// +checklocksignore +func (f *packetsPendingLinkResolution) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.nic) +} + +func (f *packetsPendingLinkResolution) afterLoad(context.Context) {} + +// +checklocksignore +func (f *packetsPendingLinkResolution) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.nic) +} + +func (t *TransportEndpointID) StateTypeName() string { + return "pkg/tcpip/stack.TransportEndpointID" +} + +func (t *TransportEndpointID) StateFields() []string { + return []string{ + "LocalPort", + "LocalAddress", + "RemotePort", + "RemoteAddress", + } +} + +func (t *TransportEndpointID) beforeSave() {} + +// +checklocksignore +func (t *TransportEndpointID) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.LocalPort) + stateSinkObject.Save(1, &t.LocalAddress) + stateSinkObject.Save(2, &t.RemotePort) + stateSinkObject.Save(3, &t.RemoteAddress) +} + +func (t *TransportEndpointID) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TransportEndpointID) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.LocalPort) + stateSourceObject.Load(1, &t.LocalAddress) + stateSourceObject.Load(2, &t.RemotePort) + stateSourceObject.Load(3, &t.RemoteAddress) +} + +func (n *NetworkPacketInfo) StateTypeName() string { + return "pkg/tcpip/stack.NetworkPacketInfo" +} + +func (n *NetworkPacketInfo) StateFields() []string { + return []string{ + "LocalAddressBroadcast", + "LocalAddressTemporary", + "IsForwardedPacket", + } +} + +func (n *NetworkPacketInfo) beforeSave() {} + +// +checklocksignore +func (n *NetworkPacketInfo) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.LocalAddressBroadcast) + stateSinkObject.Save(1, &n.LocalAddressTemporary) + stateSinkObject.Save(2, &n.IsForwardedPacket) +} + +func (n *NetworkPacketInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NetworkPacketInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.LocalAddressBroadcast) + stateSourceObject.Load(1, &n.LocalAddressTemporary) + stateSourceObject.Load(2, &n.IsForwardedPacket) +} + +func (p *PacketMMapOpts) StateTypeName() string { + return "pkg/tcpip/stack.PacketMMapOpts" +} + +func (p *PacketMMapOpts) StateFields() []string { + return []string{ + "Req", + "IsRx", + "Cooked", + "Stack", + "Wq", + "PacketEndpoint", + "Version", + "Reserve", + } +} + +func (p *PacketMMapOpts) beforeSave() {} + +// +checklocksignore +func (p *PacketMMapOpts) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.Req) + stateSinkObject.Save(1, &p.IsRx) + stateSinkObject.Save(2, &p.Cooked) + stateSinkObject.Save(3, &p.Stack) + stateSinkObject.Save(4, &p.Wq) + stateSinkObject.Save(5, &p.PacketEndpoint) + stateSinkObject.Save(6, &p.Version) + stateSinkObject.Save(7, &p.Reserve) +} + +func (p *PacketMMapOpts) afterLoad(context.Context) {} + +// +checklocksignore +func (p *PacketMMapOpts) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.Req) + stateSourceObject.Load(1, &p.IsRx) + stateSourceObject.Load(2, &p.Cooked) + stateSourceObject.Load(3, &p.Stack) + stateSourceObject.Load(4, &p.Wq) + stateSourceObject.Load(5, &p.PacketEndpoint) + stateSourceObject.Load(6, &p.Version) + stateSourceObject.Load(7, &p.Reserve) +} + +func (lifetimes *AddressLifetimes) StateTypeName() string { + return "pkg/tcpip/stack.AddressLifetimes" +} + +func (lifetimes *AddressLifetimes) StateFields() []string { + return []string{ + "Deprecated", + "PreferredUntil", + "ValidUntil", + } +} + +func (lifetimes *AddressLifetimes) beforeSave() {} + +// +checklocksignore +func (lifetimes *AddressLifetimes) StateSave(stateSinkObject state.Sink) { + lifetimes.beforeSave() + stateSinkObject.Save(0, &lifetimes.Deprecated) + stateSinkObject.Save(1, &lifetimes.PreferredUntil) + stateSinkObject.Save(2, &lifetimes.ValidUntil) +} + +func (lifetimes *AddressLifetimes) afterLoad(context.Context) {} + +// +checklocksignore +func (lifetimes *AddressLifetimes) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &lifetimes.Deprecated) + stateSourceObject.Load(1, &lifetimes.PreferredUntil) + stateSourceObject.Load(2, &lifetimes.ValidUntil) +} + +func (u *UnicastSourceAndMulticastDestination) StateTypeName() string { + return "pkg/tcpip/stack.UnicastSourceAndMulticastDestination" +} + +func (u *UnicastSourceAndMulticastDestination) StateFields() []string { + return []string{ + "Source", + "Destination", + } +} + +func (u *UnicastSourceAndMulticastDestination) beforeSave() {} + +// +checklocksignore +func (u *UnicastSourceAndMulticastDestination) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.Source) + stateSinkObject.Save(1, &u.Destination) +} + +func (u *UnicastSourceAndMulticastDestination) afterLoad(context.Context) {} + +// +checklocksignore +func (u *UnicastSourceAndMulticastDestination) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.Source) + stateSourceObject.Load(1, &u.Destination) +} + +func (c *DADConfigurations) StateTypeName() string { + return "pkg/tcpip/stack.DADConfigurations" +} + +func (c *DADConfigurations) StateFields() []string { + return []string{ + "DupAddrDetectTransmits", + "RetransmitTimer", + } +} + +func (c *DADConfigurations) beforeSave() {} + +// +checklocksignore +func (c *DADConfigurations) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.DupAddrDetectTransmits) + stateSinkObject.Save(1, &c.RetransmitTimer) +} + +func (c *DADConfigurations) afterLoad(context.Context) {} + +// +checklocksignore +func (c *DADConfigurations) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.DupAddrDetectTransmits) + stateSourceObject.Load(1, &c.RetransmitTimer) +} + +func (g *GSOType) StateTypeName() string { + return "pkg/tcpip/stack.GSOType" +} + +func (g *GSOType) StateFields() []string { + return nil +} + +func (g *GSO) StateTypeName() string { + return "pkg/tcpip/stack.GSO" +} + +func (g *GSO) StateFields() []string { + return []string{ + "Type", + "NeedsCsum", + "CsumOffset", + "MSS", + "L3HdrLen", + "MaxSize", + } +} + +func (g *GSO) beforeSave() {} + +// +checklocksignore +func (g *GSO) StateSave(stateSinkObject state.Sink) { + g.beforeSave() + stateSinkObject.Save(0, &g.Type) + stateSinkObject.Save(1, &g.NeedsCsum) + stateSinkObject.Save(2, &g.CsumOffset) + stateSinkObject.Save(3, &g.MSS) + stateSinkObject.Save(4, &g.L3HdrLen) + stateSinkObject.Save(5, &g.MaxSize) +} + +func (g *GSO) afterLoad(context.Context) {} + +// +checklocksignore +func (g *GSO) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &g.Type) + stateSourceObject.Load(1, &g.NeedsCsum) + stateSourceObject.Load(2, &g.CsumOffset) + stateSourceObject.Load(3, &g.MSS) + stateSourceObject.Load(4, &g.L3HdrLen) + stateSourceObject.Load(5, &g.MaxSize) +} + +func (r *routeInfo) StateTypeName() string { + return "pkg/tcpip/stack.routeInfo" +} + +func (r *routeInfo) StateFields() []string { + return []string{ + "RemoteAddress", + "LocalAddress", + "LocalLinkAddress", + "NextHop", + "NetProto", + "Loop", + } +} + +func (r *routeInfo) beforeSave() {} + +// +checklocksignore +func (r *routeInfo) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.RemoteAddress) + stateSinkObject.Save(1, &r.LocalAddress) + stateSinkObject.Save(2, &r.LocalLinkAddress) + stateSinkObject.Save(3, &r.NextHop) + stateSinkObject.Save(4, &r.NetProto) + stateSinkObject.Save(5, &r.Loop) +} + +func (r *routeInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (r *routeInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.RemoteAddress) + stateSourceObject.Load(1, &r.LocalAddress) + stateSourceObject.Load(2, &r.LocalLinkAddress) + stateSourceObject.Load(3, &r.NextHop) + stateSourceObject.Load(4, &r.NetProto) + stateSourceObject.Load(5, &r.Loop) +} + +func (r *RouteInfo) StateTypeName() string { + return "pkg/tcpip/stack.RouteInfo" +} + +func (r *RouteInfo) StateFields() []string { + return []string{ + "routeInfo", + "RemoteLinkAddress", + } +} + +func (r *RouteInfo) beforeSave() {} + +// +checklocksignore +func (r *RouteInfo) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.routeInfo) + stateSinkObject.Save(1, &r.RemoteLinkAddress) +} + +func (r *RouteInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (r *RouteInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.routeInfo) + stateSourceObject.Load(1, &r.RemoteLinkAddress) +} + +func (t *transportProtocolState) StateTypeName() string { + return "pkg/tcpip/stack.transportProtocolState" +} + +func (t *transportProtocolState) StateFields() []string { + return []string{ + "proto", + } +} + +func (t *transportProtocolState) beforeSave() {} + +// +checklocksignore +func (t *transportProtocolState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.proto) +} + +func (t *transportProtocolState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *transportProtocolState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.proto) +} + +func (s *Stack) StateTypeName() string { + return "pkg/tcpip/stack.Stack" +} + +func (s *Stack) StateFields() []string { + return []string{ + "transportProtocols", + "networkProtocols", + "rawFactory", + "packetEndpointWriteSupported", + "demux", + "stats", + "loopbackNIC", + "defaultForwardingEnabled", + "cleanupEndpoints", + "PortManager", + "clock", + "handleLocal", + "restoredEndpoints", + "resumableEndpoints", + "icmpRateLimiter", + "seed", + "nudConfigs", + "nudDisp", + "sendBufferSize", + "receiveBufferSize", + "tcpInvalidRateLimit", + "tsOffsetSecret", + "saveRestoreEnabled", + } +} + +func (s *Stack) beforeSave() {} + +// +checklocksignore +func (s *Stack) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.transportProtocols) + stateSinkObject.Save(1, &s.networkProtocols) + stateSinkObject.Save(2, &s.rawFactory) + stateSinkObject.Save(3, &s.packetEndpointWriteSupported) + stateSinkObject.Save(4, &s.demux) + stateSinkObject.Save(5, &s.stats) + stateSinkObject.Save(6, &s.loopbackNIC) + stateSinkObject.Save(7, &s.defaultForwardingEnabled) + stateSinkObject.Save(8, &s.cleanupEndpoints) + stateSinkObject.Save(9, &s.PortManager) + stateSinkObject.Save(10, &s.clock) + stateSinkObject.Save(11, &s.handleLocal) + stateSinkObject.Save(12, &s.restoredEndpoints) + stateSinkObject.Save(13, &s.resumableEndpoints) + stateSinkObject.Save(14, &s.icmpRateLimiter) + stateSinkObject.Save(15, &s.seed) + stateSinkObject.Save(16, &s.nudConfigs) + stateSinkObject.Save(17, &s.nudDisp) + stateSinkObject.Save(18, &s.sendBufferSize) + stateSinkObject.Save(19, &s.receiveBufferSize) + stateSinkObject.Save(20, &s.tcpInvalidRateLimit) + stateSinkObject.Save(21, &s.tsOffsetSecret) + stateSinkObject.Save(22, &s.saveRestoreEnabled) +} + +// +checklocksignore +func (s *Stack) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.transportProtocols) + stateSourceObject.Load(1, &s.networkProtocols) + stateSourceObject.Load(2, &s.rawFactory) + stateSourceObject.Load(3, &s.packetEndpointWriteSupported) + stateSourceObject.Load(4, &s.demux) + stateSourceObject.Load(5, &s.stats) + stateSourceObject.Load(6, &s.loopbackNIC) + stateSourceObject.Load(7, &s.defaultForwardingEnabled) + stateSourceObject.Load(8, &s.cleanupEndpoints) + stateSourceObject.Load(9, &s.PortManager) + stateSourceObject.Load(10, &s.clock) + stateSourceObject.Load(11, &s.handleLocal) + stateSourceObject.Load(12, &s.restoredEndpoints) + stateSourceObject.Load(13, &s.resumableEndpoints) + stateSourceObject.Load(14, &s.icmpRateLimiter) + stateSourceObject.Load(15, &s.seed) + stateSourceObject.Load(16, &s.nudConfigs) + stateSourceObject.Load(17, &s.nudDisp) + stateSourceObject.Load(18, &s.sendBufferSize) + stateSourceObject.Load(19, &s.receiveBufferSize) + stateSourceObject.Load(20, &s.tcpInvalidRateLimit) + stateSourceObject.Load(21, &s.tsOffsetSecret) + stateSourceObject.Load(22, &s.saveRestoreEnabled) + stateSourceObject.AfterLoad(func() { s.afterLoad(ctx) }) +} + +func (t *TransportEndpointInfo) StateTypeName() string { + return "pkg/tcpip/stack.TransportEndpointInfo" +} + +func (t *TransportEndpointInfo) StateFields() []string { + return []string{ + "NetProto", + "TransProto", + "ID", + "BindNICID", + "BindAddr", + "RegisterNICID", + } +} + +func (t *TransportEndpointInfo) beforeSave() {} + +// +checklocksignore +func (t *TransportEndpointInfo) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.NetProto) + stateSinkObject.Save(1, &t.TransProto) + stateSinkObject.Save(2, &t.ID) + stateSinkObject.Save(3, &t.BindNICID) + stateSinkObject.Save(4, &t.BindAddr) + stateSinkObject.Save(5, &t.RegisterNICID) +} + +func (t *TransportEndpointInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TransportEndpointInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.NetProto) + stateSourceObject.Load(1, &t.TransProto) + stateSourceObject.Load(2, &t.ID) + stateSourceObject.Load(3, &t.BindNICID) + stateSourceObject.Load(4, &t.BindAddr) + stateSourceObject.Load(5, &t.RegisterNICID) +} + +func (p *protocolIDs) StateTypeName() string { + return "pkg/tcpip/stack.protocolIDs" +} + +func (p *protocolIDs) StateFields() []string { + return []string{ + "network", + "transport", + } +} + +func (p *protocolIDs) beforeSave() {} + +// +checklocksignore +func (p *protocolIDs) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.network) + stateSinkObject.Save(1, &p.transport) +} + +func (p *protocolIDs) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocolIDs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.network) + stateSourceObject.Load(1, &p.transport) +} + +func (eps *transportEndpoints) StateTypeName() string { + return "pkg/tcpip/stack.transportEndpoints" +} + +func (eps *transportEndpoints) StateFields() []string { + return []string{ + "endpoints", + "rawEndpoints", + } +} + +func (eps *transportEndpoints) beforeSave() {} + +// +checklocksignore +func (eps *transportEndpoints) StateSave(stateSinkObject state.Sink) { + eps.beforeSave() + stateSinkObject.Save(0, &eps.endpoints) + stateSinkObject.Save(1, &eps.rawEndpoints) +} + +func (eps *transportEndpoints) afterLoad(context.Context) {} + +// +checklocksignore +func (eps *transportEndpoints) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &eps.endpoints) + stateSourceObject.Load(1, &eps.rawEndpoints) +} + +func (epsByNIC *endpointsByNIC) StateTypeName() string { + return "pkg/tcpip/stack.endpointsByNIC" +} + +func (epsByNIC *endpointsByNIC) StateFields() []string { + return []string{ + "seed", + "endpoints", + } +} + +func (epsByNIC *endpointsByNIC) beforeSave() {} + +// +checklocksignore +func (epsByNIC *endpointsByNIC) StateSave(stateSinkObject state.Sink) { + epsByNIC.beforeSave() + stateSinkObject.Save(0, &epsByNIC.seed) + stateSinkObject.Save(1, &epsByNIC.endpoints) +} + +func (epsByNIC *endpointsByNIC) afterLoad(context.Context) {} + +// +checklocksignore +func (epsByNIC *endpointsByNIC) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &epsByNIC.seed) + stateSourceObject.Load(1, &epsByNIC.endpoints) +} + +func (d *transportDemuxer) StateTypeName() string { + return "pkg/tcpip/stack.transportDemuxer" +} + +func (d *transportDemuxer) StateFields() []string { + return []string{ + "stack", + "protocol", + "queuedProtocols", + } +} + +func (d *transportDemuxer) beforeSave() {} + +// +checklocksignore +func (d *transportDemuxer) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.stack) + stateSinkObject.Save(1, &d.protocol) + stateSinkObject.Save(2, &d.queuedProtocols) +} + +func (d *transportDemuxer) afterLoad(context.Context) {} + +// +checklocksignore +func (d *transportDemuxer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.stack) + stateSourceObject.Load(1, &d.protocol) + stateSourceObject.Load(2, &d.queuedProtocols) +} + +func (ep *multiPortEndpoint) StateTypeName() string { + return "pkg/tcpip/stack.multiPortEndpoint" +} + +func (ep *multiPortEndpoint) StateFields() []string { + return []string{ + "demux", + "netProto", + "transProto", + "flags", + "endpoints", + } +} + +func (ep *multiPortEndpoint) beforeSave() {} + +// +checklocksignore +func (ep *multiPortEndpoint) StateSave(stateSinkObject state.Sink) { + ep.beforeSave() + stateSinkObject.Save(0, &ep.demux) + stateSinkObject.Save(1, &ep.netProto) + stateSinkObject.Save(2, &ep.transProto) + stateSinkObject.Save(3, &ep.flags) + stateSinkObject.Save(4, &ep.endpoints) +} + +func (ep *multiPortEndpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (ep *multiPortEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ep.demux) + stateSourceObject.Load(1, &ep.netProto) + stateSourceObject.Load(2, &ep.transProto) + stateSourceObject.Load(3, &ep.flags) + stateSourceObject.Load(4, &ep.endpoints) +} + +func (l *tupleList) StateTypeName() string { + return "pkg/tcpip/stack.tupleList" +} + +func (l *tupleList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *tupleList) beforeSave() {} + +// +checklocksignore +func (l *tupleList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *tupleList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *tupleList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *tupleEntry) StateTypeName() string { + return "pkg/tcpip/stack.tupleEntry" +} + +func (e *tupleEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *tupleEntry) beforeSave() {} + +// +checklocksignore +func (e *tupleEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *tupleEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *tupleEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*addressStateRefs)(nil)) + state.Register((*AddressableEndpointState)(nil)) + state.Register((*AddressableEndpointStateOptions)(nil)) + state.Register((*bridgePort)(nil)) + state.Register((*BridgeEndpoint)(nil)) + state.Register((*tuple)(nil)) + state.Register((*tupleID)(nil)) + state.Register((*conn)(nil)) + state.Register((*ConnTrack)(nil)) + state.Register((*bucket)(nil)) + state.Register((*ICMPRateLimiter)(nil)) + state.Register((*AcceptTarget)(nil)) + state.Register((*DropTarget)(nil)) + state.Register((*RejectIPv4Target)(nil)) + state.Register((*RejectIPv6Target)(nil)) + state.Register((*ErrorTarget)(nil)) + state.Register((*UserChainTarget)(nil)) + state.Register((*ReturnTarget)(nil)) + state.Register((*DNATTarget)(nil)) + state.Register((*RedirectTarget)(nil)) + state.Register((*SNATTarget)(nil)) + state.Register((*MasqueradeTarget)(nil)) + state.Register((*IPTables)(nil)) + state.Register((*Table)(nil)) + state.Register((*Rule)(nil)) + state.Register((*IPHeaderFilter)(nil)) + state.Register((*dynamicCacheEntry)(nil)) + state.Register((*neighborCacheMu)(nil)) + state.Register((*neighborCache)(nil)) + state.Register((*NeighborEntry)(nil)) + state.Register((*timer)(nil)) + state.Register((*neighborEntryMu)(nil)) + state.Register((*neighborEntry)(nil)) + state.Register((*neighborEntryList)(nil)) + state.Register((*neighborEntryEntry)(nil)) + state.Register((*linkResolver)(nil)) + state.Register((*nic)(nil)) + state.Register((*packetEndpointList)(nil)) + state.Register((*delegatingQueueingDiscipline)(nil)) + state.Register((*sharedStats)(nil)) + state.Register((*multiCounterNICPacketStats)(nil)) + state.Register((*multiCounterNICNeighborStats)(nil)) + state.Register((*multiCounterNICStats)(nil)) + state.Register((*NUDConfigurations)(nil)) + state.Register((*nudStateMu)(nil)) + state.Register((*NUDState)(nil)) + state.Register((*PacketBuffer)(nil)) + state.Register((*headerInfo)(nil)) + state.Register((*PacketData)(nil)) + state.Register((*PacketBufferList)(nil)) + state.Register((*packetBufferRefs)(nil)) + state.Register((*pendingPacket)(nil)) + state.Register((*packetsPendingLinkResolution)(nil)) + state.Register((*TransportEndpointID)(nil)) + state.Register((*NetworkPacketInfo)(nil)) + state.Register((*PacketMMapOpts)(nil)) + state.Register((*AddressLifetimes)(nil)) + state.Register((*UnicastSourceAndMulticastDestination)(nil)) + state.Register((*DADConfigurations)(nil)) + state.Register((*GSOType)(nil)) + state.Register((*GSO)(nil)) + state.Register((*routeInfo)(nil)) + state.Register((*RouteInfo)(nil)) + state.Register((*transportProtocolState)(nil)) + state.Register((*Stack)(nil)) + state.Register((*TransportEndpointInfo)(nil)) + state.Register((*protocolIDs)(nil)) + state.Register((*transportEndpoints)(nil)) + state.Register((*endpointsByNIC)(nil)) + state.Register((*transportDemuxer)(nil)) + state.Register((*multiPortEndpoint)(nil)) + state.Register((*tupleList)(nil)) + state.Register((*tupleEntry)(nil)) +} diff --git a/pkg/tcpip/stack/stack_unsafe_state_autogen.go b/pkg/tcpip/stack/stack_unsafe_state_autogen.go new file mode 100644 index 0000000..758ab34 --- /dev/null +++ b/pkg/tcpip/stack/stack_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package stack diff --git a/pkg/tcpip/stack/state_conn_mutex.go b/pkg/tcpip/stack/state_conn_mutex.go new file mode 100644 index 0000000..62af231 --- /dev/null +++ b/pkg/tcpip/stack/state_conn_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type stateConnRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var stateConnlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type stateConnlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *stateConnRWMutex) Lock() { + locking.AddGLock(stateConnprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *stateConnRWMutex) NestedLock(i stateConnlockNameIndex) { + locking.AddGLock(stateConnprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *stateConnRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(stateConnprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *stateConnRWMutex) NestedUnlock(i stateConnlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(stateConnprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *stateConnRWMutex) RLock() { + locking.AddGLock(stateConnprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *stateConnRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(stateConnprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *stateConnRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *stateConnRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *stateConnRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var stateConnprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func stateConninitLockNames() {} + +func init() { + stateConninitLockNames() + stateConnprefixIndex = locking.NewMutexClass(reflect.TypeOf(stateConnRWMutex{}), stateConnlockNames) +} diff --git a/pkg/tcpip/stack/transport_demuxer.go b/pkg/tcpip/stack/transport_demuxer.go new file mode 100644 index 0000000..98f1f70 --- /dev/null +++ b/pkg/tcpip/stack/transport_demuxer.go @@ -0,0 +1,733 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/hash/jenkins" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" +) + +// +stateify savable +type protocolIDs struct { + network tcpip.NetworkProtocolNumber + transport tcpip.TransportProtocolNumber +} + +// transportEndpoints manages all endpoints of a given protocol. It has its own +// mutex so as to reduce interference between protocols. +// +// +stateify savable +type transportEndpoints struct { + mu transportEndpointsRWMutex `state:"nosave"` + // +checklocks:mu + endpoints map[TransportEndpointID]*endpointsByNIC + // rawEndpoints contains endpoints for raw sockets, which receive all + // traffic of a given protocol regardless of port. + // + // +checklocks:mu + rawEndpoints []RawTransportEndpoint +} + +// unregisterEndpoint unregisters the endpoint with the given id such that it +// won't receive any more packets. +func (eps *transportEndpoints) unregisterEndpoint(id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) { + eps.mu.Lock() + defer eps.mu.Unlock() + epsByNIC, ok := eps.endpoints[id] + if !ok { + return + } + if !epsByNIC.unregisterEndpoint(bindToDevice, ep, flags) { + return + } + delete(eps.endpoints, id) +} + +func (eps *transportEndpoints) transportEndpoints() []TransportEndpoint { + eps.mu.RLock() + defer eps.mu.RUnlock() + es := make([]TransportEndpoint, 0, len(eps.endpoints)) + for _, e := range eps.endpoints { + es = append(es, e.transportEndpoints()...) + } + return es +} + +// iterEndpointsLocked yields all endpointsByNIC in eps that match id, in +// descending order of match quality. If a call to yield returns false, +// iterEndpointsLocked stops iteration and returns immediately. +// +// +checklocksread:eps.mu +func (eps *transportEndpoints) iterEndpointsLocked(id TransportEndpointID, yield func(*endpointsByNIC) bool) { + // Try to find a match with the id as provided. + if ep, ok := eps.endpoints[id]; ok { + if !yield(ep) { + return + } + } + + // Try to find a match with the id minus the local address. + nid := id + + nid.LocalAddress = tcpip.Address{} + if ep, ok := eps.endpoints[nid]; ok { + if !yield(ep) { + return + } + } + + // Try to find a match with the id minus the remote part. + nid.LocalAddress = id.LocalAddress + nid.RemoteAddress = tcpip.Address{} + nid.RemotePort = 0 + if ep, ok := eps.endpoints[nid]; ok { + if !yield(ep) { + return + } + } + + // Try to find a match with only the local port. + nid.LocalAddress = tcpip.Address{} + if ep, ok := eps.endpoints[nid]; ok { + if !yield(ep) { + return + } + } +} + +// findAllEndpointsLocked returns all endpointsByNIC in eps that match id, in +// descending order of match quality. +// +// +checklocksread:eps.mu +func (eps *transportEndpoints) findAllEndpointsLocked(id TransportEndpointID) []*endpointsByNIC { + var matchedEPs []*endpointsByNIC + eps.iterEndpointsLocked(id, func(ep *endpointsByNIC) bool { + matchedEPs = append(matchedEPs, ep) + return true + }) + return matchedEPs +} + +// findEndpointLocked returns the endpoint that most closely matches the given id. +// +// +checklocksread:eps.mu +func (eps *transportEndpoints) findEndpointLocked(id TransportEndpointID) *endpointsByNIC { + var matchedEP *endpointsByNIC + eps.iterEndpointsLocked(id, func(ep *endpointsByNIC) bool { + matchedEP = ep + return false + }) + return matchedEP +} + +// +stateify savable +type endpointsByNIC struct { + // seed is a random secret for a jenkins hash. + seed uint32 + + mu endpointsByNICRWMutex `state:"nosave"` + // +checklocks:mu + endpoints map[tcpip.NICID]*multiPortEndpoint +} + +func (epsByNIC *endpointsByNIC) transportEndpoints() []TransportEndpoint { + epsByNIC.mu.RLock() + defer epsByNIC.mu.RUnlock() + var eps []TransportEndpoint + for _, ep := range epsByNIC.endpoints { + eps = append(eps, ep.transportEndpoints()...) + } + return eps +} + +// handlePacket is called by the stack when new packets arrive to this transport +// endpoint. It returns false if the packet could not be matched to any +// transport endpoint, true otherwise. +func (epsByNIC *endpointsByNIC) handlePacket(id TransportEndpointID, pkt *PacketBuffer) bool { + epsByNIC.mu.RLock() + + mpep, ok := epsByNIC.endpoints[pkt.NICID] + if !ok { + if mpep, ok = epsByNIC.endpoints[0]; !ok { + epsByNIC.mu.RUnlock() // Don't use defer for performance reasons. + return false + } + } + + // If this is a broadcast or multicast datagram, deliver the datagram to all + // endpoints bound to the right device. + if isInboundMulticastOrBroadcast(pkt, id.LocalAddress) { + mpep.handlePacketAll(id, pkt) + epsByNIC.mu.RUnlock() // Don't use defer for performance reasons. + return true + } + // multiPortEndpoints are guaranteed to have at least one element. + transEP := mpep.selectEndpoint(id, epsByNIC.seed) + if queuedProtocol, mustQueue := mpep.demux.queuedProtocols[protocolIDs{mpep.netProto, mpep.transProto}]; mustQueue { + queuedProtocol.QueuePacket(transEP, id, pkt) + epsByNIC.mu.RUnlock() + return true + } + epsByNIC.mu.RUnlock() + + transEP.HandlePacket(id, pkt) + return true +} + +// handleError delivers an error to the transport endpoint identified by id. +func (epsByNIC *endpointsByNIC) handleError(n *nic, id TransportEndpointID, transErr TransportError, pkt *PacketBuffer) { + epsByNIC.mu.RLock() + + mpep, ok := epsByNIC.endpoints[n.ID()] + if !ok { + mpep, ok = epsByNIC.endpoints[0] + } + if !ok { + epsByNIC.mu.RUnlock() + return + } + + // TODO(eyalsoha): Why don't we look at id to see if this packet needs to + // broadcast like we are doing with handlePacket above? + + // multiPortEndpoints are guaranteed to have at least one element. + transEP := mpep.selectEndpoint(id, epsByNIC.seed) + epsByNIC.mu.RUnlock() + + transEP.HandleError(transErr, pkt) +} + +// registerEndpoint returns true if it succeeds. It fails and returns +// false if ep already has an element with the same key. +func (epsByNIC *endpointsByNIC) registerEndpoint(d *transportDemuxer, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, t TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + epsByNIC.mu.Lock() + defer epsByNIC.mu.Unlock() + + multiPortEp, ok := epsByNIC.endpoints[bindToDevice] + if !ok { + multiPortEp = &multiPortEndpoint{ + demux: d, + netProto: netProto, + transProto: transProto, + } + } + + if err := multiPortEp.singleRegisterEndpoint(t, flags); err != nil { + return err + } + // Only add this newly created multiportEndpoint if the singleRegisterEndpoint + // succeeded. + if !ok { + epsByNIC.endpoints[bindToDevice] = multiPortEp + } + return nil +} + +func (epsByNIC *endpointsByNIC) checkEndpoint(flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + epsByNIC.mu.RLock() + defer epsByNIC.mu.RUnlock() + + multiPortEp, ok := epsByNIC.endpoints[bindToDevice] + if !ok { + return nil + } + + return multiPortEp.singleCheckEndpoint(flags) +} + +// unregisterEndpoint returns true if endpointsByNIC has to be unregistered. +func (epsByNIC *endpointsByNIC) unregisterEndpoint(bindToDevice tcpip.NICID, t TransportEndpoint, flags ports.Flags) bool { + epsByNIC.mu.Lock() + defer epsByNIC.mu.Unlock() + multiPortEp, ok := epsByNIC.endpoints[bindToDevice] + if !ok { + return false + } + if multiPortEp.unregisterEndpoint(t, flags) { + delete(epsByNIC.endpoints, bindToDevice) + } + return len(epsByNIC.endpoints) == 0 +} + +// transportDemuxer demultiplexes packets targeted at a transport endpoint +// (i.e., after they've been parsed by the network layer). It does two levels +// of demultiplexing: first based on the network and transport protocols, then +// based on endpoints IDs. It should only be instantiated via +// newTransportDemuxer. +// +// +stateify savable +type transportDemuxer struct { + stack *Stack + + // protocol is immutable. + protocol map[protocolIDs]*transportEndpoints + queuedProtocols map[protocolIDs]queuedTransportProtocol +} + +// queuedTransportProtocol if supported by a protocol implementation will cause +// the dispatcher to delivery packets to the QueuePacket method instead of +// calling HandlePacket directly on the endpoint. +type queuedTransportProtocol interface { + QueuePacket(ep TransportEndpoint, id TransportEndpointID, pkt *PacketBuffer) +} + +func newTransportDemuxer(stack *Stack) *transportDemuxer { + d := &transportDemuxer{ + stack: stack, + protocol: make(map[protocolIDs]*transportEndpoints), + queuedProtocols: make(map[protocolIDs]queuedTransportProtocol), + } + + // Add each network and transport pair to the demuxer. + for netProto := range stack.networkProtocols { + for proto := range stack.transportProtocols { + protoIDs := protocolIDs{netProto, proto} + d.protocol[protoIDs] = &transportEndpoints{ + endpoints: make(map[TransportEndpointID]*endpointsByNIC), + } + qTransProto, isQueued := (stack.transportProtocols[proto].proto).(queuedTransportProtocol) + if isQueued { + d.queuedProtocols[protoIDs] = qTransProto + } + } + } + + return d +} + +// registerEndpoint registers the given endpoint with the dispatcher such that +// packets that match the endpoint ID are delivered to it. +func (d *transportDemuxer) registerEndpoint(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + for i, n := range netProtos { + if err := d.singleRegisterEndpoint(n, protocol, id, ep, flags, bindToDevice); err != nil { + d.unregisterEndpoint(netProtos[:i], protocol, id, ep, flags, bindToDevice) + return err + } + } + + return nil +} + +// checkEndpoint checks if an endpoint can be registered with the dispatcher. +func (d *transportDemuxer) checkEndpoint(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + for _, n := range netProtos { + if err := d.singleCheckEndpoint(n, protocol, id, flags, bindToDevice); err != nil { + return err + } + } + + return nil +} + +// multiPortEndpoint is a container for TransportEndpoints which are bound to +// the same pair of address and port. endpointsArr always has at least one +// element. +// +// FIXME(gvisor.dev/issue/873): Restore this properly. Currently, we just save +// this to ensure that the underlying endpoints get saved/restored, but not not +// use the restored copy. +// +// +stateify savable +type multiPortEndpoint struct { + demux *transportDemuxer + netProto tcpip.NetworkProtocolNumber + transProto tcpip.TransportProtocolNumber + + flags ports.FlagCounter + + mu multiPortEndpointRWMutex `state:"nosave"` + // endpoints stores the transport endpoints in the order in which they + // were bound. This is required for UDP SO_REUSEADDR. + // + // +checklocks:mu + endpoints []TransportEndpoint +} + +func (ep *multiPortEndpoint) transportEndpoints() []TransportEndpoint { + ep.mu.RLock() + eps := append([]TransportEndpoint(nil), ep.endpoints...) + ep.mu.RUnlock() + return eps +} + +// reciprocalScale scales a value into range [0, n). +// +// This is similar to val % n, but faster. +// See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ +func reciprocalScale(val, n uint32) uint32 { + return uint32((uint64(val) * uint64(n)) >> 32) +} + +// selectEndpoint calculates a hash of destination and source addresses and +// ports then uses it to select a socket. In this case, all packets from one +// address will be sent to same endpoint. +func (ep *multiPortEndpoint) selectEndpoint(id TransportEndpointID, seed uint32) TransportEndpoint { + ep.mu.RLock() + defer ep.mu.RUnlock() + + if len(ep.endpoints) == 1 { + return ep.endpoints[0] + } + + if ep.flags.SharedFlags().ToFlags().Effective().MostRecent { + return ep.endpoints[len(ep.endpoints)-1] + } + + payload := []byte{ + byte(id.LocalPort), + byte(id.LocalPort >> 8), + byte(id.RemotePort), + byte(id.RemotePort >> 8), + } + + h := jenkins.Sum32(seed) + h.Write(payload) + h.Write(id.LocalAddress.AsSlice()) + h.Write(id.RemoteAddress.AsSlice()) + hash := h.Sum32() + + idx := reciprocalScale(hash, uint32(len(ep.endpoints))) + return ep.endpoints[idx] +} + +func (ep *multiPortEndpoint) handlePacketAll(id TransportEndpointID, pkt *PacketBuffer) { + ep.mu.RLock() + queuedProtocol, mustQueue := ep.demux.queuedProtocols[protocolIDs{ep.netProto, ep.transProto}] + // HandlePacket may modify pkt, so each endpoint needs + // its own copy except for the final one. + for _, endpoint := range ep.endpoints[:len(ep.endpoints)-1] { + clone := pkt.Clone() + if mustQueue { + queuedProtocol.QueuePacket(endpoint, id, clone) + } else { + endpoint.HandlePacket(id, clone) + } + clone.DecRef() + } + if endpoint := ep.endpoints[len(ep.endpoints)-1]; mustQueue { + queuedProtocol.QueuePacket(endpoint, id, pkt) + } else { + endpoint.HandlePacket(id, pkt) + } + ep.mu.RUnlock() // Don't use defer for performance reasons. +} + +// singleRegisterEndpoint tries to add an endpoint to the multiPortEndpoint +// list. The list might be empty already. +func (ep *multiPortEndpoint) singleRegisterEndpoint(t TransportEndpoint, flags ports.Flags) tcpip.Error { + ep.mu.Lock() + defer ep.mu.Unlock() + bits := flags.Bits() & ports.MultiBindFlagMask + + if len(ep.endpoints) != 0 { + // If it was previously bound, we need to check if we can bind again. + if ep.flags.TotalRefs() > 0 && bits&ep.flags.SharedFlags() == 0 { + return &tcpip.ErrPortInUse{} + } + } + + ep.endpoints = append(ep.endpoints, t) + ep.flags.AddRef(bits) + + return nil +} + +func (ep *multiPortEndpoint) singleCheckEndpoint(flags ports.Flags) tcpip.Error { + ep.mu.RLock() + defer ep.mu.RUnlock() + + bits := flags.Bits() & ports.MultiBindFlagMask + + if len(ep.endpoints) != 0 { + // If it was previously bound, we need to check if we can bind again. + if ep.flags.TotalRefs() > 0 && bits&ep.flags.SharedFlags() == 0 { + return &tcpip.ErrPortInUse{} + } + } + + return nil +} + +// unregisterEndpoint returns true if multiPortEndpoint has to be unregistered. +func (ep *multiPortEndpoint) unregisterEndpoint(t TransportEndpoint, flags ports.Flags) bool { + ep.mu.Lock() + defer ep.mu.Unlock() + + for i, endpoint := range ep.endpoints { + if endpoint == t { + copy(ep.endpoints[i:], ep.endpoints[i+1:]) + ep.endpoints[len(ep.endpoints)-1] = nil + ep.endpoints = ep.endpoints[:len(ep.endpoints)-1] + + ep.flags.DropRef(flags.Bits() & ports.MultiBindFlagMask) + break + } + } + return len(ep.endpoints) == 0 +} + +func (d *transportDemuxer) singleRegisterEndpoint(netProto tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + if id.RemotePort != 0 { + // SO_REUSEPORT only applies to bound/listening endpoints. + flags.LoadBalanced = false + } + + eps, ok := d.protocol[protocolIDs{netProto, protocol}] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + + eps.mu.Lock() + defer eps.mu.Unlock() + epsByNIC, ok := eps.endpoints[id] + if !ok { + epsByNIC = &endpointsByNIC{ + endpoints: make(map[tcpip.NICID]*multiPortEndpoint), + seed: d.stack.seed, + } + } + if err := epsByNIC.registerEndpoint(d, netProto, protocol, ep, flags, bindToDevice); err != nil { + return err + } + // Only add this newly created epsByNIC if registerEndpoint succeeded. + if !ok { + eps.endpoints[id] = epsByNIC + } + return nil +} + +func (d *transportDemuxer) singleCheckEndpoint(netProto tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, flags ports.Flags, bindToDevice tcpip.NICID) tcpip.Error { + if id.RemotePort != 0 { + // SO_REUSEPORT only applies to bound/listening endpoints. + flags.LoadBalanced = false + } + + eps, ok := d.protocol[protocolIDs{netProto, protocol}] + if !ok { + return &tcpip.ErrUnknownProtocol{} + } + + eps.mu.RLock() + defer eps.mu.RUnlock() + + epsByNIC, ok := eps.endpoints[id] + if !ok { + return nil + } + + return epsByNIC.checkEndpoint(flags, bindToDevice) +} + +// unregisterEndpoint unregisters the endpoint with the given id such that it +// won't receive any more packets. +func (d *transportDemuxer) unregisterEndpoint(netProtos []tcpip.NetworkProtocolNumber, protocol tcpip.TransportProtocolNumber, id TransportEndpointID, ep TransportEndpoint, flags ports.Flags, bindToDevice tcpip.NICID) { + if id.RemotePort != 0 { + // SO_REUSEPORT only applies to bound/listening endpoints. + flags.LoadBalanced = false + } + + for _, n := range netProtos { + if eps, ok := d.protocol[protocolIDs{n, protocol}]; ok { + eps.unregisterEndpoint(id, ep, flags, bindToDevice) + } + } +} + +// deliverPacket attempts to find one or more matching transport endpoints, and +// then, if matches are found, delivers the packet to them. Returns true if +// the packet no longer needs to be handled. +func (d *transportDemuxer) deliverPacket(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer, id TransportEndpointID) bool { + eps, ok := d.protocol[protocolIDs{pkt.NetworkProtocolNumber, protocol}] + if !ok { + return false + } + + // If the packet is a UDP broadcast or multicast, then find all matching + // transport endpoints. + if protocol == header.UDPProtocolNumber && isInboundMulticastOrBroadcast(pkt, id.LocalAddress) { + eps.mu.RLock() + destEPs := eps.findAllEndpointsLocked(id) + eps.mu.RUnlock() + // Fail if we didn't find at least one matching transport endpoint. + if len(destEPs) == 0 { + d.stack.stats.UDP.UnknownPortErrors.Increment() + return false + } + // handlePacket takes may modify pkt, so each endpoint needs its own + // copy except for the final one. + for _, ep := range destEPs[:len(destEPs)-1] { + clone := pkt.Clone() + ep.handlePacket(id, clone) + clone.DecRef() + } + destEPs[len(destEPs)-1].handlePacket(id, pkt) + return true + } + + // If the packet is a TCP packet with a unspecified source or non-unicast + // destination address, then do nothing further and instruct the caller to do + // the same. The network layer handles address validation for specified source + // addresses. + if protocol == header.TCPProtocolNumber && (!isSpecified(id.LocalAddress) || !isSpecified(id.RemoteAddress) || isInboundMulticastOrBroadcast(pkt, id.LocalAddress)) { + // TCP can only be used to communicate between a single source and a + // single destination; the addresses must be unicast.e + d.stack.stats.TCP.InvalidSegmentsReceived.Increment() + return true + } + + eps.mu.RLock() + ep := eps.findEndpointLocked(id) + eps.mu.RUnlock() + if ep == nil { + if protocol == header.UDPProtocolNumber { + d.stack.stats.UDP.UnknownPortErrors.Increment() + } + return false + } + return ep.handlePacket(id, pkt) +} + +// deliverRawPacket attempts to deliver the given packet and returns whether it +// was delivered successfully. +func (d *transportDemuxer) deliverRawPacket(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) bool { + eps, ok := d.protocol[protocolIDs{pkt.NetworkProtocolNumber, protocol}] + if !ok { + return false + } + + // As in net/ipv4/ip_input.c:ip_local_deliver, attempt to deliver via + // raw endpoint first. If there are multiple raw endpoints, they all + // receive the packet. + eps.mu.RLock() + // Copy the list of raw endpoints to avoid packet handling under lock. + var rawEPs []RawTransportEndpoint + if n := len(eps.rawEndpoints); n != 0 { + rawEPs = make([]RawTransportEndpoint, n) + if m := copy(rawEPs, eps.rawEndpoints); m != n { + panic(fmt.Sprintf("unexpected copy = %d, want %d", m, n)) + } + } + eps.mu.RUnlock() + for _, rawEP := range rawEPs { + // Each endpoint gets its own copy of the packet for the sake + // of save/restore. + clone := pkt.Clone() + rawEP.HandlePacket(clone) + clone.DecRef() + } + + return len(rawEPs) != 0 +} + +// deliverError attempts to deliver the given error to the appropriate transport +// endpoint. +// +// Returns true if the error was delivered. +func (d *transportDemuxer) deliverError(n *nic, net tcpip.NetworkProtocolNumber, trans tcpip.TransportProtocolNumber, transErr TransportError, pkt *PacketBuffer, id TransportEndpointID) bool { + eps, ok := d.protocol[protocolIDs{net, trans}] + if !ok { + return false + } + + eps.mu.RLock() + ep := eps.findEndpointLocked(id) + eps.mu.RUnlock() + if ep == nil { + return false + } + + ep.handleError(n, id, transErr, pkt) + return true +} + +// findTransportEndpoint find a single endpoint that most closely matches the provided id. +func (d *transportDemuxer) findTransportEndpoint(netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, id TransportEndpointID, nicID tcpip.NICID) TransportEndpoint { + eps, ok := d.protocol[protocolIDs{netProto, transProto}] + if !ok { + return nil + } + + eps.mu.RLock() + epsByNIC := eps.findEndpointLocked(id) + if epsByNIC == nil { + eps.mu.RUnlock() + return nil + } + + epsByNIC.mu.RLock() + eps.mu.RUnlock() + + mpep, ok := epsByNIC.endpoints[nicID] + if !ok { + if mpep, ok = epsByNIC.endpoints[0]; !ok { + epsByNIC.mu.RUnlock() // Don't use defer for performance reasons. + return nil + } + } + + ep := mpep.selectEndpoint(id, epsByNIC.seed) + epsByNIC.mu.RUnlock() + return ep +} + +// registerRawEndpoint registers the given endpoint with the dispatcher such +// that packets of the appropriate protocol are delivered to it. A single +// packet can be sent to one or more raw endpoints along with a non-raw +// endpoint. +func (d *transportDemuxer) registerRawEndpoint(netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, ep RawTransportEndpoint) tcpip.Error { + eps, ok := d.protocol[protocolIDs{netProto, transProto}] + if !ok { + return &tcpip.ErrNotSupported{} + } + + eps.mu.Lock() + eps.rawEndpoints = append(eps.rawEndpoints, ep) + eps.mu.Unlock() + + return nil +} + +// unregisterRawEndpoint unregisters the raw endpoint for the given transport +// protocol such that it won't receive any more packets. +func (d *transportDemuxer) unregisterRawEndpoint(netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, ep RawTransportEndpoint) { + eps, ok := d.protocol[protocolIDs{netProto, transProto}] + if !ok { + panic(fmt.Errorf("tried to unregister endpoint with unsupported network and transport protocol pair: %d, %d", netProto, transProto)) + } + + eps.mu.Lock() + for i, rawEP := range eps.rawEndpoints { + if rawEP == ep { + lastIdx := len(eps.rawEndpoints) - 1 + eps.rawEndpoints[i] = eps.rawEndpoints[lastIdx] + eps.rawEndpoints[lastIdx] = nil + eps.rawEndpoints = eps.rawEndpoints[:lastIdx] + break + } + } + eps.mu.Unlock() +} + +func isInboundMulticastOrBroadcast(pkt *PacketBuffer, localAddr tcpip.Address) bool { + return pkt.NetworkPacketInfo.LocalAddressBroadcast || header.IsV4MulticastAddress(localAddr) || header.IsV6MulticastAddress(localAddr) +} + +func isSpecified(addr tcpip.Address) bool { + return addr != header.IPv4Any && addr != header.IPv6Any +} diff --git a/pkg/tcpip/stack/transport_endpoints_mutex.go b/pkg/tcpip/stack/transport_endpoints_mutex.go new file mode 100644 index 0000000..6083a4a --- /dev/null +++ b/pkg/tcpip/stack/transport_endpoints_mutex.go @@ -0,0 +1,96 @@ +package stack + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type transportEndpointsRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var transportEndpointslockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type transportEndpointslockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *transportEndpointsRWMutex) Lock() { + locking.AddGLock(transportEndpointsprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *transportEndpointsRWMutex) NestedLock(i transportEndpointslockNameIndex) { + locking.AddGLock(transportEndpointsprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *transportEndpointsRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(transportEndpointsprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *transportEndpointsRWMutex) NestedUnlock(i transportEndpointslockNameIndex) { + m.mu.Unlock() + locking.DelGLock(transportEndpointsprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *transportEndpointsRWMutex) RLock() { + locking.AddGLock(transportEndpointsprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *transportEndpointsRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(transportEndpointsprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *transportEndpointsRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *transportEndpointsRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *transportEndpointsRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var transportEndpointsprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func transportEndpointsinitLockNames() {} + +func init() { + transportEndpointsinitLockNames() + transportEndpointsprefixIndex = locking.NewMutexClass(reflect.TypeOf(transportEndpointsRWMutex{}), transportEndpointslockNames) +} diff --git a/pkg/tcpip/stack/tuple_list.go b/pkg/tcpip/stack/tuple_list.go new file mode 100644 index 0000000..f7f9196 --- /dev/null +++ b/pkg/tcpip/stack/tuple_list.go @@ -0,0 +1,239 @@ +package stack + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type tupleElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (tupleElementMapper) linkerFor(elem *tuple) *tuple { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type tupleList struct { + head *tuple + tail *tuple +} + +// Reset resets list l to the empty state. +func (l *tupleList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *tupleList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *tupleList) Front() *tuple { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *tupleList) Back() *tuple { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *tupleList) Len() (count int) { + for e := l.Front(); e != nil; e = (tupleElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *tupleList) PushFront(e *tuple) { + linker := tupleElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + tupleElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *tupleList) PushFrontList(m *tupleList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + tupleElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + tupleElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *tupleList) PushBack(e *tuple) { + linker := tupleElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + tupleElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *tupleList) PushBackList(m *tupleList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + tupleElementMapper{}.linkerFor(l.tail).SetNext(m.head) + tupleElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *tupleList) InsertAfter(b, e *tuple) { + bLinker := tupleElementMapper{}.linkerFor(b) + eLinker := tupleElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + tupleElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *tupleList) InsertBefore(a, e *tuple) { + aLinker := tupleElementMapper{}.linkerFor(a) + eLinker := tupleElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + tupleElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *tupleList) Remove(e *tuple) { + linker := tupleElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + tupleElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + tupleElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type tupleEntry struct { + next *tuple + prev *tuple +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *tupleEntry) Next() *tuple { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *tupleEntry) Prev() *tuple { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *tupleEntry) SetNext(elem *tuple) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *tupleEntry) SetPrev(elem *tuple) { + e.prev = elem +} diff --git a/pkg/tcpip/stdclock.go b/pkg/tcpip/stdclock.go new file mode 100644 index 0000000..e80e7c4 --- /dev/null +++ b/pkg/tcpip/stdclock.go @@ -0,0 +1,114 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpip + +import ( + "fmt" + "time" +) + +// stdClock implements Clock with the time package. +// +// +stateify savable +type stdClock struct { + // baseTime holds the time when the clock was constructed. + // + // This value is used to calculate the monotonic time from the time package. + // As per https://golang.org/pkg/time/#hdr-Monotonic_Clocks, + // + // Operating systems provide both a “wall clock,” which is subject to + // changes for clock synchronization, and a “monotonic clock,” which is not. + // The general rule is that the wall clock is for telling time and the + // monotonic clock is for measuring time. Rather than split the API, in this + // package the Time returned by time.Now contains both a wall clock reading + // and a monotonic clock reading; later time-telling operations use the wall + // clock reading, but later time-measuring operations, specifically + // comparisons and subtractions, use the monotonic clock reading. + // + // ... + // + // If Times t and u both contain monotonic clock readings, the operations + // t.After(u), t.Before(u), t.Equal(u), and t.Sub(u) are carried out using + // the monotonic clock readings alone, ignoring the wall clock readings. If + // either t or u contains no monotonic clock reading, these operations fall + // back to using the wall clock readings. + // + // Given the above, we can safely conclude that time.Since(baseTime) will + // return monotonically increasing values if we use time.Now() to set baseTime + // at the time of clock construction. + // + // Note that time.Since(t) is shorthand for time.Now().Sub(t), as per + // https://golang.org/pkg/time/#Since. + baseTime time.Time `state:"nosave"` + + // monotonicOffset is the offset applied to the calculated monotonic time. + // + // monotonicOffset is assigned after restore so that the monotonic time + // will continue from where it "left off" before saving as part of S/R. + monotonicOffset MonotonicTime +} + +// NewStdClock returns an instance of a clock that uses the time package. +func NewStdClock() Clock { + return &stdClock{ + baseTime: time.Now(), + } +} + +var _ Clock = (*stdClock)(nil) + +// Now implements Clock.Now. +func (*stdClock) Now() time.Time { + return time.Now() +} + +// NowMonotonic implements Clock.NowMonotonic. +func (s *stdClock) NowMonotonic() MonotonicTime { + sinceBase := time.Since(s.baseTime) + if sinceBase < 0 { + panic(fmt.Sprintf("got negative duration = %s since base time = %s", sinceBase, s.baseTime)) + } + + return s.monotonicOffset.Add(sinceBase) +} + +// AfterFunc implements Clock.AfterFunc. +func (*stdClock) AfterFunc(d time.Duration, f func()) Timer { + return &stdTimer{ + t: time.AfterFunc(d, f), + } +} + +// +stateify savable +type stdTimer struct { + t *time.Timer +} + +var _ Timer = (*stdTimer)(nil) + +// Stop implements Timer.Stop. +func (st *stdTimer) Stop() bool { + return st.t.Stop() +} + +// Reset implements Timer.Reset. +func (st *stdTimer) Reset(d time.Duration) { + st.t.Reset(d) +} + +// NewStdTimer returns a Timer implemented with the time package. +func NewStdTimer(t *time.Timer) Timer { + return &stdTimer{t: t} +} diff --git a/pkg/tcpip/stdclock_state.go b/pkg/tcpip/stdclock_state.go new file mode 100644 index 0000000..530b46e --- /dev/null +++ b/pkg/tcpip/stdclock_state.go @@ -0,0 +1,30 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpip + +import ( + "context" + "time" +) + +// beforeSave is invoked by stateify. +func (s *stdClock) beforeSave() { + s.monotonicOffset = s.NowMonotonic() +} + +// afterLoad is invoked by stateify. +func (s *stdClock) afterLoad(context.Context) { + s.baseTime = time.Now() +} diff --git a/pkg/tcpip/tcpip.go b/pkg/tcpip/tcpip.go new file mode 100644 index 0000000..f1add0d --- /dev/null +++ b/pkg/tcpip/tcpip.go @@ -0,0 +1,2899 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tcpip provides the interfaces and related types that users of the +// tcpip stack will use in order to create endpoints used to send and receive +// data over the network stack. +// +// The starting point is the creation and configuration of a stack. A stack can +// be created by calling the New() function of the tcpip/stack/stack package; +// configuring a stack involves creating NICs (via calls to Stack.CreateNIC()), +// adding network addresses (via calls to Stack.AddProtocolAddress()), and +// setting a route table (via a call to Stack.SetRouteTable()). +// +// Once a stack is configured, endpoints can be created by calling +// Stack.NewEndpoint(). Such endpoints can be used to send/receive data, connect +// to peers, listen for connections, accept connections, etc., depending on the +// transport protocol selected. +package tcpip + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "math/bits" + "net" + "reflect" + "strconv" + "strings" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/rand" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// Using the header package here would cause an import cycle. +const ( + ipv4AddressSize = 4 + ipv4ProtocolNumber = 0x0800 + ipv6AddressSize = 16 + ipv6ProtocolNumber = 0x86dd +) + +const ( + // LinkAddressSize is the size of a MAC address. + LinkAddressSize = 6 +) + +// Known IP address. +var ( + IPv4Zero = []byte{0, 0, 0, 0} + IPv6Zero = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +) + +// Errors related to Subnet +var ( + errSubnetLengthMismatch = errors.New("subnet length of address and mask differ") + errSubnetAddressMasked = errors.New("subnet address has bits set outside the mask") +) + +// ErrSaveRejection indicates a failed save due to unsupported networking state. +// This type of errors is only used for save logic. +type ErrSaveRejection struct { + Err error +} + +// Error returns a sensible description of the save rejection error. +func (e *ErrSaveRejection) Error() string { + return "save rejected due to unsupported networking state: " + e.Err.Error() +} + +// MonotonicTime is a monotonic clock reading. +// +// +stateify savable +type MonotonicTime struct { + nanoseconds int64 +} + +// String implements Stringer. +func (mt MonotonicTime) String() string { + return strconv.FormatInt(mt.nanoseconds, 10) +} + +// MonotonicTimeInfinite returns the monotonic timestamp as far away in the +// future as possible. +func MonotonicTimeInfinite() MonotonicTime { + return MonotonicTime{nanoseconds: math.MaxInt64} +} + +// Before reports whether the monotonic clock reading mt is before u. +func (mt MonotonicTime) Before(u MonotonicTime) bool { + return mt.nanoseconds < u.nanoseconds +} + +// After reports whether the monotonic clock reading mt is after u. +func (mt MonotonicTime) After(u MonotonicTime) bool { + return mt.nanoseconds > u.nanoseconds +} + +// Add returns the monotonic clock reading mt+d. +func (mt MonotonicTime) Add(d time.Duration) MonotonicTime { + return MonotonicTime{ + nanoseconds: time.Unix(0, mt.nanoseconds).Add(d).Sub(time.Unix(0, 0)).Nanoseconds(), + } +} + +// Sub returns the duration mt-u. If the result exceeds the maximum (or minimum) +// value that can be stored in a Duration, the maximum (or minimum) duration +// will be returned. To compute t-d for a duration d, use t.Add(-d). +func (mt MonotonicTime) Sub(u MonotonicTime) time.Duration { + return time.Unix(0, mt.nanoseconds).Sub(time.Unix(0, u.nanoseconds)) +} + +// Milliseconds returns the time in milliseconds. +func (mt MonotonicTime) Milliseconds() int64 { + return mt.nanoseconds / 1e6 +} + +// A Clock provides the current time and schedules work for execution. +// +// Times returned by a Clock should always be used for application-visible +// time. Only monotonic times should be used for netstack internal timekeeping. +type Clock interface { + // Now returns the current local time. + Now() time.Time + + // NowMonotonic returns the current monotonic clock reading. + NowMonotonic() MonotonicTime + + // AfterFunc waits for the duration to elapse and then calls f in its own + // goroutine. It returns a Timer that can be used to cancel the call using + // its Stop method. + AfterFunc(d time.Duration, f func()) Timer +} + +// Timer represents a single event. A Timer must be created with +// Clock.AfterFunc. +type Timer interface { + // Stop prevents the Timer from firing. It returns true if the call stops the + // timer, false if the timer has already expired or been stopped. + // + // If Stop returns false, then the timer has already expired and the function + // f of Clock.AfterFunc(d, f) has been started in its own goroutine; Stop + // does not wait for f to complete before returning. If the caller needs to + // know whether f is completed, it must coordinate with f explicitly. + Stop() bool + + // Reset changes the timer to expire after duration d. + // + // Reset should be invoked only on stopped or expired timers. If the timer is + // known to have expired, Reset can be used directly. Otherwise, the caller + // must coordinate with the function f of Clock.AfterFunc(d, f). + Reset(d time.Duration) +} + +// Address is a byte slice cast as a string that represents the address of a +// network node. Or, in the case of unix endpoints, it may represent a path. +// +// +stateify savable +type Address struct { + addr [16]byte + length int +} + +// AddrFrom4 converts addr to an Address. +func AddrFrom4(addr [4]byte) Address { + ret := Address{ + length: 4, + } + // It's guaranteed that copy will return 4. + copy(ret.addr[:], addr[:]) + return ret +} + +// AddrFrom4Slice converts addr to an Address. It panics if len(addr) != 4. +func AddrFrom4Slice(addr []byte) Address { + if len(addr) != 4 { + panic(fmt.Sprintf("bad address length for address %v", addr)) + } + ret := Address{ + length: 4, + } + // It's guaranteed that copy will return 4. + copy(ret.addr[:], addr) + return ret +} + +// AddrFrom16 converts addr to an Address. +func AddrFrom16(addr [16]byte) Address { + ret := Address{ + length: 16, + } + // It's guaranteed that copy will return 16. + copy(ret.addr[:], addr[:]) + return ret +} + +// AddrFrom16Slice converts addr to an Address. It panics if len(addr) != 16. +func AddrFrom16Slice(addr []byte) Address { + if len(addr) != 16 { + panic(fmt.Sprintf("bad address length for address %v", addr)) + } + ret := Address{ + length: 16, + } + // It's guaranteed that copy will return 16. + copy(ret.addr[:], addr) + return ret +} + +// AddrFromSlice converts addr to an Address. It returns the Address zero value +// if len(addr) != 4 or 16. +func AddrFromSlice(addr []byte) Address { + switch len(addr) { + case ipv4AddressSize: + return AddrFrom4Slice(addr) + case ipv6AddressSize: + return AddrFrom16Slice(addr) + } + return Address{} +} + +// As4 returns a as a 4 byte array. It panics if the address length is not 4. +func (a Address) As4() [4]byte { + if a.Len() != 4 { + panic(fmt.Sprintf("bad address length for address %v", a.addr)) + } + return [4]byte(a.addr[:4]) +} + +// As16 returns a as a 16 byte array. It panics if the address length is not 16. +func (a Address) As16() [16]byte { + if a.Len() != 16 { + panic(fmt.Sprintf("bad address length for address %v", a.addr)) + } + return [16]byte(a.addr[:16]) +} + +// AsSlice returns a as a byte slice. Callers should be careful as it can +// return a window into existing memory. +// +// +checkescape +func (a *Address) AsSlice() []byte { + return a.addr[:a.length] +} + +// BitLen returns the length in bits of a. +func (a Address) BitLen() int { + return a.Len() * 8 +} + +// Len returns the length in bytes of a. +func (a Address) Len() int { + return a.length +} + +// WithPrefix returns the address with a prefix that represents a point subnet. +func (a Address) WithPrefix() AddressWithPrefix { + return AddressWithPrefix{ + Address: a, + PrefixLen: a.BitLen(), + } +} + +// Unspecified returns true if the address is unspecified. +func (a Address) Unspecified() bool { + for _, b := range a.addr { + if b != 0 { + return false + } + } + return true +} + +// Equal returns whether a and other are equal. It exists for use by the cmp +// library. +func (a Address) Equal(other Address) bool { + return a == other +} + +// MatchingPrefix returns the matching prefix length in bits. +// +// Panics if b and a have different lengths. +func (a Address) MatchingPrefix(b Address) uint8 { + const bitsInAByte = 8 + + if a.Len() != b.Len() { + panic(fmt.Sprintf("addresses %s and %s do not have the same length", a, b)) + } + + var prefix uint8 + for i := 0; i < a.length; i++ { + aByte := a.addr[i] + bByte := b.addr[i] + + if aByte == bByte { + prefix += bitsInAByte + continue + } + + // Count the remaining matching bits in the byte from MSbit to LSBbit. + mask := uint8(1) << (bitsInAByte - 1) + for { + if aByte&mask == bByte&mask { + prefix++ + mask >>= 1 + continue + } + + break + } + + break + } + + return prefix +} + +// AddressMask is a bitmask for an address. +// +// +stateify savable +type AddressMask struct { + mask [16]byte + length int +} + +// MaskFrom returns a Mask based on str. +// +// MaskFrom may allocate, and so should not be in hot paths. +func MaskFrom(str string) AddressMask { + mask := AddressMask{length: len(str)} + copy(mask.mask[:], str) + return mask +} + +// MaskFromBytes returns a Mask based on bs. +func MaskFromBytes(bs []byte) AddressMask { + mask := AddressMask{length: len(bs)} + copy(mask.mask[:], bs) + return mask +} + +// String implements Stringer. +func (m AddressMask) String() string { + return fmt.Sprintf("%x", m.mask) +} + +// AsSlice returns a as a byte slice. Callers should be careful as it can +// return a window into existing memory. +func (m *AddressMask) AsSlice() []byte { + return []byte(m.mask[:m.length]) +} + +// BitLen returns the length of the mask in bits. +func (m AddressMask) BitLen() int { + return m.length * 8 +} + +// Len returns the length of the mask in bytes. +func (m AddressMask) Len() int { + return m.length +} + +// Prefix returns the number of bits before the first host bit. +func (m AddressMask) Prefix() int { + p := 0 + for _, b := range m.mask[:m.length] { + p += bits.LeadingZeros8(^b) + } + return p +} + +// Equal returns whether m and other are equal. It exists for use by the cmp +// library. +func (m AddressMask) Equal(other AddressMask) bool { + return m == other +} + +// Subnet is a subnet defined by its address and mask. +// +// +stateify savable +type Subnet struct { + address Address + mask AddressMask +} + +// NewSubnet creates a new Subnet, checking that the address and mask are the same length. +func NewSubnet(a Address, m AddressMask) (Subnet, error) { + if a.Len() != m.Len() { + return Subnet{}, errSubnetLengthMismatch + } + for i := 0; i < a.Len(); i++ { + if a.addr[i]&^m.mask[i] != 0 { + return Subnet{}, errSubnetAddressMasked + } + } + return Subnet{a, m}, nil +} + +// String implements Stringer. +func (s Subnet) String() string { + return fmt.Sprintf("%s/%d", s.ID(), s.Prefix()) +} + +// Contains returns true iff the address is of the same length and matches the +// subnet address and mask. +func (s *Subnet) Contains(a Address) bool { + if a.Len() != s.address.Len() { + return false + } + for i := 0; i < a.Len(); i++ { + if a.addr[i]&s.mask.mask[i] != s.address.addr[i] { + return false + } + } + return true +} + +// ID returns the subnet ID. +func (s *Subnet) ID() Address { + return s.address +} + +// Bits returns the number of ones (network bits) and zeros (host bits) in the +// subnet mask. +func (s *Subnet) Bits() (ones int, zeros int) { + ones = s.mask.Prefix() + return ones, s.mask.BitLen() - ones +} + +// Prefix returns the number of bits before the first host bit. +func (s *Subnet) Prefix() int { + return s.mask.Prefix() +} + +// Mask returns the subnet mask. +func (s *Subnet) Mask() AddressMask { + return s.mask +} + +// Broadcast returns the subnet's broadcast address. +func (s *Subnet) Broadcast() Address { + addrCopy := s.address + for i := 0; i < addrCopy.Len(); i++ { + addrCopy.addr[i] |= ^s.mask.mask[i] + } + return addrCopy +} + +// IsBroadcast returns true if the address is considered a broadcast address. +func (s *Subnet) IsBroadcast(address Address) bool { + // Only IPv4 supports the notion of a broadcast address. + if address.Len() != ipv4AddressSize { + return false + } + + // Normally, we would just compare address with the subnet's broadcast + // address but there is an exception where a simple comparison is not + // correct. This exception is for /31 and /32 IPv4 subnets where all + // addresses are considered valid host addresses. + // + // For /31 subnets, the case is easy. RFC 3021 Section 2.1 states that + // both addresses in a /31 subnet "MUST be interpreted as host addresses." + // + // For /32, the case is a bit more vague. RFC 3021 makes no mention of /32 + // subnets. However, the same reasoning applies - if an exception is not + // made, then there do not exist any host addresses in a /32 subnet. RFC + // 4632 Section 3.1 also vaguely implies this interpretation by referring + // to addresses in /32 subnets as "host routes." + return s.Prefix() <= 30 && s.Broadcast() == address +} + +// Equal returns true if this Subnet is equal to the given Subnet. +func (s Subnet) Equal(o Subnet) bool { + // If this changes, update Route.Equal accordingly. + return s == o +} + +// NICID is a number that uniquely identifies a NIC. +type NICID int32 + +// ShutdownFlags represents flags that can be passed to the Shutdown() method +// of the Endpoint interface. +type ShutdownFlags int + +// Values of the flags that can be passed to the Shutdown() method. They can +// be OR'ed together. +const ( + ShutdownRead ShutdownFlags = 1 << iota + ShutdownWrite +) + +// PacketType is used to indicate the destination of the packet. +type PacketType uint8 + +const ( + // PacketHost indicates a packet addressed to the local host. + PacketHost PacketType = iota + + // PacketOtherHost indicates an outgoing packet addressed to + // another host caught by a NIC in promiscuous mode. + PacketOtherHost + + // PacketOutgoing for a packet originating from the local host + // that is looped back to a packet socket. + PacketOutgoing + + // PacketBroadcast indicates a link layer broadcast packet. + PacketBroadcast + + // PacketMulticast indicates a link layer multicast packet. + PacketMulticast +) + +// FullAddress represents a full transport node address, as required by the +// Connect() and Bind() methods. +// +// +stateify savable +type FullAddress struct { + // NIC is the ID of the NIC this address refers to. + // + // This may not be used by all endpoint types. + NIC NICID + + // Addr is the network address. + Addr Address + + // Port is the transport port. + // + // This may not be used by all endpoint types. + Port uint16 + + // LinkAddr is the link layer address. + LinkAddr LinkAddress +} + +// Payloader is an interface that provides data. +// +// This interface allows the endpoint to request the amount of data it needs +// based on internal buffers without exposing them. +type Payloader interface { + io.Reader + + // Len returns the number of bytes of the unread portion of the + // Reader. + Len() int +} + +var ( + _ Payloader = (*bytes.Buffer)(nil) + _ Payloader = (*bytes.Reader)(nil) +) + +var _ io.Writer = (*SliceWriter)(nil) + +// SliceWriter implements io.Writer for slices. +type SliceWriter []byte + +// Write implements io.Writer.Write. +func (s *SliceWriter) Write(b []byte) (int, error) { + n := copy(*s, b) + *s = (*s)[n:] + var err error + if n != len(b) { + err = io.ErrShortWrite + } + return n, err +} + +var _ io.Writer = (*LimitedWriter)(nil) + +// A LimitedWriter writes to W but limits the amount of data copied to just N +// bytes. Each call to Write updates N to reflect the new amount remaining. +type LimitedWriter struct { + W io.Writer + N int64 +} + +func (l *LimitedWriter) Write(p []byte) (int, error) { + pLen := int64(len(p)) + if pLen > l.N { + p = p[:l.N] + } + n, err := l.W.Write(p) + n64 := int64(n) + if err == nil && n64 != pLen { + err = io.ErrShortWrite + } + l.N -= n64 + return n, err +} + +// SendableControlMessages contains socket control messages that can be written. +// +// +stateify savable +type SendableControlMessages struct { + // HasTTL indicates whether TTL is valid/set. + HasTTL bool + + // TTL is the IPv4 Time To Live of the associated packet. + TTL uint8 + + // HasHopLimit indicates whether HopLimit is valid/set. + HasHopLimit bool + + // HopLimit is the IPv6 Hop Limit of the associated packet. + HopLimit uint8 + + // HasIPv6PacketInfo indicates whether IPv6PacketInfo is set. + HasIPv6PacketInfo bool + + // IPv6PacketInfo holds interface and address data on an incoming packet. + IPv6PacketInfo IPv6PacketInfo +} + +// ReceivableControlMessages contains socket control messages that can be +// received. +// +// +stateify savable +type ReceivableControlMessages struct { + // Timestamp is the time that the last packet used to create the read data + // was received. + Timestamp time.Time `state:".(int64)"` + + // HasInq indicates whether Inq is valid/set. + HasInq bool + + // Inq is the number of bytes ready to be received. + Inq int32 + + // HasTOS indicates whether TOS is valid/set. + HasTOS bool + + // TOS is the IPv4 type of service of the associated packet. + TOS uint8 + + // HasTTL indicates whether TTL is valid/set. + HasTTL bool + + // TTL is the IPv4 Time To Live of the associated packet. + TTL uint8 + + // HasHopLimit indicates whether HopLimit is valid/set. + HasHopLimit bool + + // HopLimit is the IPv6 Hop Limit of the associated packet. + HopLimit uint8 + + // HasTimestamp indicates whether Timestamp is valid/set. + HasTimestamp bool + + // HasTClass indicates whether TClass is valid/set. + HasTClass bool + + // TClass is the IPv6 traffic class of the associated packet. + TClass uint32 + + // HasIPPacketInfo indicates whether PacketInfo is set. + HasIPPacketInfo bool + + // PacketInfo holds interface and address data on an incoming packet. + PacketInfo IPPacketInfo + + // HasIPv6PacketInfo indicates whether IPv6PacketInfo is set. + HasIPv6PacketInfo bool + + // IPv6PacketInfo holds interface and address data on an incoming packet. + IPv6PacketInfo IPv6PacketInfo + + // HasOriginalDestinationAddress indicates whether OriginalDstAddress is + // set. + HasOriginalDstAddress bool + + // OriginalDestinationAddress holds the original destination address + // and port of the incoming packet. + OriginalDstAddress FullAddress + + // SockErr is the dequeued socket error on recvmsg(MSG_ERRQUEUE). + SockErr *SockError +} + +// PacketOwner is used to get UID and GID of the packet. +type PacketOwner interface { + // KUID returns KUID of the packet. + KUID() uint32 + + // KGID returns KGID of the packet. + KGID() uint32 +} + +// ReadOptions contains options for Endpoint.Read. +type ReadOptions struct { + // Peek indicates whether this read is a peek. + Peek bool + + // NeedRemoteAddr indicates whether to return the remote address, if + // supported. + NeedRemoteAddr bool + + // NeedLinkPacketInfo indicates whether to return the link-layer information, + // if supported. + NeedLinkPacketInfo bool +} + +// ReadResult represents result for a successful Endpoint.Read. +type ReadResult struct { + // Count is the number of bytes received and written to the buffer. + Count int + + // Total is the number of bytes of the received packet. This can be used to + // determine whether the read is truncated. + Total int + + // ControlMessages is the control messages received. + ControlMessages ReceivableControlMessages + + // RemoteAddr is the remote address if ReadOptions.NeedAddr is true. + RemoteAddr FullAddress + + // LinkPacketInfo is the link-layer information of the received packet if + // ReadOptions.NeedLinkPacketInfo is true. + LinkPacketInfo LinkPacketInfo +} + +// Endpoint is the interface implemented by transport protocols (e.g., tcp, udp) +// that exposes functionality like read, write, connect, etc. to users of the +// networking stack. +type Endpoint interface { + // Close puts the endpoint in a closed state and frees all resources + // associated with it. Close initiates the teardown process, the + // Endpoint may not be fully closed when Close returns. + Close() + + // Abort initiates an expedited endpoint teardown. As compared to + // Close, Abort prioritizes closing the Endpoint quickly over cleanly. + // Abort is best effort; implementing Abort with Close is acceptable. + Abort() + + // Read reads data from the endpoint and optionally writes to dst. + // + // This method does not block if there is no data pending; in this case, + // ErrWouldBlock is returned. + // + // If non-zero number of bytes are successfully read and written to dst, err + // must be nil. Otherwise, if dst failed to write anything, ErrBadBuffer + // should be returned. + Read(io.Writer, ReadOptions) (ReadResult, Error) + + // Write writes data to the endpoint's peer. This method does not block if + // the data cannot be written. + // + // Unlike io.Writer.Write, Endpoint.Write transfers ownership of any bytes + // successfully written to the Endpoint. That is, if a call to + // Write(SlicePayload{data}) returns (n, err), it may retain data[:n], and + // the caller should not use data[:n] after Write returns. + // + // Note that unlike io.Writer.Write, it is not an error for Write to + // perform a partial write (if n > 0, no error may be returned). Only + // stream (TCP) Endpoints may return partial writes, and even then only + // in the case where writing additional data would block. Other Endpoints + // will either write the entire message or return an error. + Write(Payloader, WriteOptions) (int64, Error) + + // Connect connects the endpoint to its peer. Specifying a NIC is + // optional. + // + // There are three classes of return values: + // nil -- the attempt to connect succeeded. + // ErrConnectStarted/ErrAlreadyConnecting -- the connect attempt started + // but hasn't completed yet. In this case, the caller must call Connect + // or GetSockOpt(ErrorOption) when the endpoint becomes writable to + // get the actual result. The first call to Connect after the socket has + // connected returns nil. Calling connect again results in ErrAlreadyConnected. + // Anything else -- the attempt to connect failed. + // + // If address.Addr is empty, this means that Endpoint has to be + // disconnected if this is supported, otherwise + // ErrAddressFamilyNotSupported must be returned. + Connect(address FullAddress) Error + + // Disconnect disconnects the endpoint from its peer. + Disconnect() Error + + // Shutdown closes the read and/or write end of the endpoint connection + // to its peer. + Shutdown(flags ShutdownFlags) Error + + // Listen puts the endpoint in "listen" mode, which allows it to accept + // new connections. + Listen(backlog int) Error + + // Accept returns a new endpoint if a peer has established a connection + // to an endpoint previously set to listen mode. This method does not + // block if no new connections are available. + // + // The returned Queue is the wait queue for the newly created endpoint. + // + // If peerAddr is not nil then it is populated with the peer address of the + // returned endpoint. + Accept(peerAddr *FullAddress) (Endpoint, *waiter.Queue, Error) + + // Bind binds the endpoint to a specific local address and port. + // Specifying a NIC is optional. + Bind(address FullAddress) Error + + // GetLocalAddress returns the address to which the endpoint is bound. + GetLocalAddress() (FullAddress, Error) + + // GetRemoteAddress returns the address to which the endpoint is + // connected. + GetRemoteAddress() (FullAddress, Error) + + // Readiness returns the current readiness of the endpoint. For example, + // if waiter.EventIn is set, the endpoint is immediately readable. + Readiness(mask waiter.EventMask) waiter.EventMask + + // SetSockOpt sets a socket option. + SetSockOpt(opt SettableSocketOption) Error + + // SetSockOptInt sets a socket option, for simple cases where a value + // has the int type. + SetSockOptInt(opt SockOptInt, v int) Error + + // GetSockOpt gets a socket option. + GetSockOpt(opt GettableSocketOption) Error + + // GetSockOptInt gets a socket option for simple cases where a return + // value has the int type. + GetSockOptInt(SockOptInt) (int, Error) + + // State returns a socket's lifecycle state. The returned value is + // protocol-specific and is primarily used for diagnostics. + State() uint32 + + // ModerateRecvBuf should be called everytime data is copied to the user + // space. This allows for dynamic tuning of recv buffer space for a + // given socket. + // + // NOTE: This method is a no-op for sockets other than TCP. + ModerateRecvBuf(copied int) + + // Info returns a copy to the transport endpoint info. + Info() EndpointInfo + + // Stats returns a reference to the endpoint stats. + Stats() EndpointStats + + // SetOwner sets the task owner to the endpoint owner. + SetOwner(owner PacketOwner) + + // LastError clears and returns the last error reported by the endpoint. + LastError() Error + + // SocketOptions returns the structure which contains all the socket + // level options. + SocketOptions() *SocketOptions +} + +// EndpointWithPreflight is the interface implemented by endpoints that need +// to expose the `Preflight` method for preparing the endpoint prior to +// calling `Write`. +type EndpointWithPreflight interface { + // Prepares the endpoint for writes using the provided WriteOptions, + // returning an error if the options were incompatible with the endpoint's + // current state. + Preflight(WriteOptions) Error +} + +// LinkPacketInfo holds Link layer information for a received packet. +// +// +stateify savable +type LinkPacketInfo struct { + // Protocol is the NetworkProtocolNumber for the packet. + Protocol NetworkProtocolNumber + + // PktType is used to indicate the destination of the packet. + PktType PacketType +} + +// EndpointInfo is the interface implemented by each endpoint info struct. +type EndpointInfo interface { + // IsEndpointInfo is an empty method to implement the tcpip.EndpointInfo + // marker interface. + IsEndpointInfo() +} + +// EndpointStats is the interface implemented by each endpoint stats struct. +type EndpointStats interface { + // IsEndpointStats is an empty method to implement the tcpip.EndpointStats + // marker interface. + IsEndpointStats() +} + +// WriteOptions contains options for Endpoint.Write. +type WriteOptions struct { + // If To is not nil, write to the given address instead of the endpoint's + // peer. + To *FullAddress + + // More has the same semantics as Linux's MSG_MORE. + More bool + + // EndOfRecord has the same semantics as Linux's MSG_EOR. + EndOfRecord bool + + // Atomic means that all data fetched from Payloader must be written to the + // endpoint. If Atomic is false, then data fetched from the Payloader may be + // discarded if available endpoint buffer space is insufficient. + Atomic bool + + // ControlMessages contains optional overrides used when writing a packet. + ControlMessages SendableControlMessages +} + +// SockOptInt represents socket options which values have the int type. +type SockOptInt int + +const ( + // KeepaliveCountOption is used by SetSockOptInt/GetSockOptInt to + // specify the number of un-ACKed TCP keepalives that will be sent + // before the connection is closed. + KeepaliveCountOption SockOptInt = iota + + // IPv4TOSOption is used by SetSockOptInt/GetSockOptInt to specify TOS + // for all subsequent outgoing IPv4 packets from the endpoint. + IPv4TOSOption + + // IPv6TrafficClassOption is used by SetSockOptInt/GetSockOptInt to + // specify TOS for all subsequent outgoing IPv6 packets from the + // endpoint. + IPv6TrafficClassOption + + // MaxSegOption is used by SetSockOptInt/GetSockOptInt to set/get the + // current Maximum Segment Size(MSS) value as specified using the + // TCP_MAXSEG option. + MaxSegOption + + // MTUDiscoverOption is used to set/get the path MTU discovery setting. + // + // NOTE: Setting this option to any other value than PMTUDiscoveryDont + // is not supported and will fail as such, and getting this option will + // always return PMTUDiscoveryDont. + MTUDiscoverOption + + // MulticastTTLOption is used by SetSockOptInt/GetSockOptInt to control + // the default TTL value for multicast messages. The default is 1. + MulticastTTLOption + + // ReceiveQueueSizeOption is used in GetSockOptInt to specify that the + // number of unread bytes in the input buffer should be returned. + ReceiveQueueSizeOption + + // SendQueueSizeOption is used in GetSockOptInt to specify that the + // number of unread bytes in the output buffer should be returned. + SendQueueSizeOption + + // IPv4TTLOption is used by SetSockOptInt/GetSockOptInt to control the default + // TTL value for unicast messages. + // + // The default is configured by DefaultTTLOption. A UseDefaultIPv4TTL value + // configures the endpoint to use the default. + IPv4TTLOption + + // IPv6HopLimitOption is used by SetSockOptInt/GetSockOptInt to control the + // default hop limit value for unicast messages. + // + // The default is configured by DefaultTTLOption. A UseDefaultIPv6HopLimit + // value configures the endpoint to use the default. + IPv6HopLimitOption + + // TCPSynCountOption is used by SetSockOptInt/GetSockOptInt to specify + // the number of SYN retransmits that TCP should send before aborting + // the attempt to connect. It cannot exceed 255. + // + // NOTE: This option is currently only stubbed out and is no-op. + TCPSynCountOption + + // TCPWindowClampOption is used by SetSockOptInt/GetSockOptInt to bound + // the size of the advertised window to this value. + // + // NOTE: This option is currently only stubed out and is a no-op + TCPWindowClampOption + + // IPv6Checksum is used to request the stack to populate and validate the IPv6 + // checksum for transport level headers. + IPv6Checksum + + // PacketMMapVersionOption is used to set the packet mmap version. + PacketMMapVersionOption + + // PacketMMapReserveOption is used to set the packet mmap reserved space + // between the aligned header and the payload. + PacketMMapReserveOption +) + +const ( + // UseDefaultIPv4TTL is the IPv4TTLOption value that configures an endpoint to + // use the default ttl currently configured by the IPv4 protocol (see + // DefaultTTLOption). + UseDefaultIPv4TTL = 0 + + // UseDefaultIPv6HopLimit is the IPv6HopLimitOption value that configures an + // endpoint to use the default hop limit currently configured by the IPv6 + // protocol (see DefaultTTLOption). + UseDefaultIPv6HopLimit = -1 +) + +// PMTUDStrategy is the kind of PMTUD to perform. +type PMTUDStrategy int + +const ( + // PMTUDiscoveryWant is a setting of the MTUDiscoverOption to use + // per-route settings. + PMTUDiscoveryWant PMTUDStrategy = iota + + // PMTUDiscoveryDont is a setting of the MTUDiscoverOption to disable + // path MTU discovery. + PMTUDiscoveryDont + + // PMTUDiscoveryDo is a setting of the MTUDiscoverOption to always do + // path MTU discovery. + PMTUDiscoveryDo + + // PMTUDiscoveryProbe is a setting of the MTUDiscoverOption to set DF + // but ignore path MTU. + PMTUDiscoveryProbe +) + +// GettableNetworkProtocolOption is a marker interface for network protocol +// options that may be queried. +type GettableNetworkProtocolOption interface { + isGettableNetworkProtocolOption() +} + +// SettableNetworkProtocolOption is a marker interface for network protocol +// options that may be set. +type SettableNetworkProtocolOption interface { + isSettableNetworkProtocolOption() +} + +// DefaultTTLOption is used by stack.(*Stack).NetworkProtocolOption to specify +// a default TTL. +type DefaultTTLOption uint8 + +func (*DefaultTTLOption) isGettableNetworkProtocolOption() {} + +func (*DefaultTTLOption) isSettableNetworkProtocolOption() {} + +// GettableTransportProtocolOption is a marker interface for transport protocol +// options that may be queried. +type GettableTransportProtocolOption interface { + isGettableTransportProtocolOption() +} + +// SettableTransportProtocolOption is a marker interface for transport protocol +// options that may be set. +type SettableTransportProtocolOption interface { + isSettableTransportProtocolOption() +} + +// TCPSACKEnabled the SACK option for TCP. +// +// See: https://tools.ietf.org/html/rfc2018. +type TCPSACKEnabled bool + +func (*TCPSACKEnabled) isGettableTransportProtocolOption() {} + +func (*TCPSACKEnabled) isSettableTransportProtocolOption() {} + +// TCPRecovery is the loss deteoction algorithm used by TCP. +type TCPRecovery int32 + +func (*TCPRecovery) isGettableTransportProtocolOption() {} + +func (*TCPRecovery) isSettableTransportProtocolOption() {} + +// TCPAlwaysUseSynCookies indicates unconditional usage of syncookies. +type TCPAlwaysUseSynCookies bool + +func (*TCPAlwaysUseSynCookies) isGettableTransportProtocolOption() {} + +func (*TCPAlwaysUseSynCookies) isSettableTransportProtocolOption() {} + +const ( + // TCPRACKLossDetection indicates RACK is used for loss detection and + // recovery. + TCPRACKLossDetection TCPRecovery = 1 << iota + + // TCPRACKStaticReoWnd indicates the reordering window should not be + // adjusted when DSACK is received. + TCPRACKStaticReoWnd + + // TCPRACKNoDupTh indicates RACK should not consider the classic three + // duplicate acknowledgements rule to mark the segments as lost. This + // is used when reordering is not detected. + TCPRACKNoDupTh +) + +// TCPDelayEnabled enables/disables Nagle's algorithm in TCP. +type TCPDelayEnabled bool + +func (*TCPDelayEnabled) isGettableTransportProtocolOption() {} + +func (*TCPDelayEnabled) isSettableTransportProtocolOption() {} + +// TCPSendBufferSizeRangeOption is the send buffer size range for TCP. +// +// +stateify savable +type TCPSendBufferSizeRangeOption struct { + Min int + Default int + Max int +} + +func (*TCPSendBufferSizeRangeOption) isGettableTransportProtocolOption() {} + +func (*TCPSendBufferSizeRangeOption) isSettableTransportProtocolOption() {} + +// TCPReceiveBufferSizeRangeOption is the receive buffer size range for TCP. +// +// +stateify savable +type TCPReceiveBufferSizeRangeOption struct { + Min int + Default int + Max int +} + +func (*TCPReceiveBufferSizeRangeOption) isGettableTransportProtocolOption() {} + +func (*TCPReceiveBufferSizeRangeOption) isSettableTransportProtocolOption() {} + +// TCPAvailableCongestionControlOption is the supported congestion control +// algorithms for TCP +type TCPAvailableCongestionControlOption string + +func (*TCPAvailableCongestionControlOption) isGettableTransportProtocolOption() {} + +func (*TCPAvailableCongestionControlOption) isSettableTransportProtocolOption() {} + +// TCPModerateReceiveBufferOption enables/disables receive buffer moderation +// for TCP. +type TCPModerateReceiveBufferOption bool + +func (*TCPModerateReceiveBufferOption) isGettableTransportProtocolOption() {} + +func (*TCPModerateReceiveBufferOption) isSettableTransportProtocolOption() {} + +// GettableSocketOption is a marker interface for socket options that may be +// queried. +type GettableSocketOption interface { + isGettableSocketOption() +} + +// SettableSocketOption is a marker interface for socket options that may be +// configured. +type SettableSocketOption interface { + isSettableSocketOption() +} + +// ICMPv6Filter specifies a filter for ICMPv6 types. +// +// +stateify savable +type ICMPv6Filter struct { + // DenyType indicates if an ICMP type should be blocked. + // + // The ICMPv6 type field is 8 bits so there are up to 256 different ICMPv6 + // types. + DenyType [8]uint32 +} + +// ShouldDeny returns true iff the ICMPv6 Type should be denied. +func (f *ICMPv6Filter) ShouldDeny(icmpType uint8) bool { + const bitsInUint32 = 32 + i := icmpType / bitsInUint32 + b := icmpType % bitsInUint32 + return f.DenyType[i]&(1<. +// +// +stateify savable +type TpacketStats struct { + Packets uint32 + Dropped uint32 +} + +func (*TpacketStats) isGettableSocketOption() {} + +// EndpointState represents the state of an endpoint. +type EndpointState uint8 + +// CongestionControlState indicates the current congestion control state for +// TCP sender. +type CongestionControlState int + +const ( + // Open indicates that the sender is receiving acks in order and + // no loss or dupACK's etc have been detected. + Open CongestionControlState = iota + // RTORecovery indicates that an RTO has occurred and the sender + // has entered an RTO based recovery phase. + RTORecovery + // FastRecovery indicates that the sender has entered FastRecovery + // based on receiving nDupAck's. This state is entered only when + // SACK is not in use. + FastRecovery + // SACKRecovery indicates that the sender has entered SACK based + // recovery. + SACKRecovery + // Disorder indicates the sender either received some SACK blocks + // or dupACK's. + Disorder +) + +// TCPInfoOption is used by GetSockOpt to expose TCP statistics. +type TCPInfoOption struct { + // RTT is the smoothed round trip time. + RTT time.Duration + + // RTTVar is the round trip time variation. + RTTVar time.Duration + + // RTO is the retransmission timeout for the endpoint. + RTO time.Duration + + // State is the current endpoint protocol state. + State EndpointState + + // CcState is the congestion control state. + CcState CongestionControlState + + // SndCwnd is the congestion window, in packets. + SndCwnd uint32 + + // SndSsthresh is the threshold between slow start and congestion + // avoidance. + SndSsthresh uint32 + + // ReorderSeen indicates if reordering is seen in the endpoint. + ReorderSeen bool +} + +func (*TCPInfoOption) isGettableSocketOption() {} + +// KeepaliveIdleOption is used by SetSockOpt/GetSockOpt to specify the time a +// connection must remain idle before the first TCP keepalive packet is sent. +// Once this time is reached, KeepaliveIntervalOption is used instead. +type KeepaliveIdleOption time.Duration + +func (*KeepaliveIdleOption) isGettableSocketOption() {} + +func (*KeepaliveIdleOption) isSettableSocketOption() {} + +// KeepaliveIntervalOption is used by SetSockOpt/GetSockOpt to specify the +// interval between sending TCP keepalive packets. +type KeepaliveIntervalOption time.Duration + +func (*KeepaliveIntervalOption) isGettableSocketOption() {} + +func (*KeepaliveIntervalOption) isSettableSocketOption() {} + +// TCPUserTimeoutOption is used by SetSockOpt/GetSockOpt to specify a user +// specified timeout for a given TCP connection. +// See: RFC5482 for details. +type TCPUserTimeoutOption time.Duration + +func (*TCPUserTimeoutOption) isGettableSocketOption() {} + +func (*TCPUserTimeoutOption) isSettableSocketOption() {} + +// CongestionControlOption is used by SetSockOpt/GetSockOpt to set/get +// the current congestion control algorithm. +type CongestionControlOption string + +func (*CongestionControlOption) isGettableSocketOption() {} + +func (*CongestionControlOption) isSettableSocketOption() {} + +func (*CongestionControlOption) isGettableTransportProtocolOption() {} + +func (*CongestionControlOption) isSettableTransportProtocolOption() {} + +// TCPLingerTimeoutOption is used by SetSockOpt/GetSockOpt to set/get the +// maximum duration for which a socket lingers in the TCP_FIN_WAIT_2 state +// before being marked closed. +type TCPLingerTimeoutOption time.Duration + +func (*TCPLingerTimeoutOption) isGettableSocketOption() {} + +func (*TCPLingerTimeoutOption) isSettableSocketOption() {} + +func (*TCPLingerTimeoutOption) isGettableTransportProtocolOption() {} + +func (*TCPLingerTimeoutOption) isSettableTransportProtocolOption() {} + +// TCPTimeWaitTimeoutOption is used by SetSockOpt/GetSockOpt to set/get the +// maximum duration for which a socket lingers in the TIME_WAIT state +// before being marked closed. +type TCPTimeWaitTimeoutOption time.Duration + +func (*TCPTimeWaitTimeoutOption) isGettableSocketOption() {} + +func (*TCPTimeWaitTimeoutOption) isSettableSocketOption() {} + +func (*TCPTimeWaitTimeoutOption) isGettableTransportProtocolOption() {} + +func (*TCPTimeWaitTimeoutOption) isSettableTransportProtocolOption() {} + +// TCPDeferAcceptOption is used by SetSockOpt/GetSockOpt to allow a +// accept to return a completed connection only when there is data to be +// read. This usually means the listening socket will drop the final ACK +// for a handshake till the specified timeout until a segment with data arrives. +type TCPDeferAcceptOption time.Duration + +func (*TCPDeferAcceptOption) isGettableSocketOption() {} + +func (*TCPDeferAcceptOption) isSettableSocketOption() {} + +// TCPMinRTOOption is use by SetSockOpt/GetSockOpt to allow overriding +// default MinRTO used by the Stack. +type TCPMinRTOOption time.Duration + +func (*TCPMinRTOOption) isGettableSocketOption() {} + +func (*TCPMinRTOOption) isSettableSocketOption() {} + +func (*TCPMinRTOOption) isGettableTransportProtocolOption() {} + +func (*TCPMinRTOOption) isSettableTransportProtocolOption() {} + +// TCPMaxRTOOption is use by SetSockOpt/GetSockOpt to allow overriding +// default MaxRTO used by the Stack. +type TCPMaxRTOOption time.Duration + +func (*TCPMaxRTOOption) isGettableSocketOption() {} + +func (*TCPMaxRTOOption) isSettableSocketOption() {} + +func (*TCPMaxRTOOption) isGettableTransportProtocolOption() {} + +func (*TCPMaxRTOOption) isSettableTransportProtocolOption() {} + +// TCPMaxRetriesOption is used by SetSockOpt/GetSockOpt to set/get the +// maximum number of retransmits after which we time out the connection. +type TCPMaxRetriesOption uint64 + +func (*TCPMaxRetriesOption) isGettableSocketOption() {} + +func (*TCPMaxRetriesOption) isSettableSocketOption() {} + +func (*TCPMaxRetriesOption) isGettableTransportProtocolOption() {} + +func (*TCPMaxRetriesOption) isSettableTransportProtocolOption() {} + +// TCPSynRetriesOption is used by SetSockOpt/GetSockOpt to specify stack-wide +// default for number of times SYN is retransmitted before aborting a connect. +type TCPSynRetriesOption uint8 + +func (*TCPSynRetriesOption) isGettableSocketOption() {} + +func (*TCPSynRetriesOption) isSettableSocketOption() {} + +func (*TCPSynRetriesOption) isGettableTransportProtocolOption() {} + +func (*TCPSynRetriesOption) isSettableTransportProtocolOption() {} + +// MulticastInterfaceOption is used by SetSockOpt/GetSockOpt to specify a +// default interface for multicast. +type MulticastInterfaceOption struct { + NIC NICID + InterfaceAddr Address +} + +func (*MulticastInterfaceOption) isGettableSocketOption() {} + +func (*MulticastInterfaceOption) isSettableSocketOption() {} + +// MembershipOption is used to identify a multicast membership on an interface. +type MembershipOption struct { + NIC NICID + InterfaceAddr Address + MulticastAddr Address +} + +// AddMembershipOption identifies a multicast group to join on some interface. +type AddMembershipOption MembershipOption + +func (*AddMembershipOption) isSettableSocketOption() {} + +// RemoveMembershipOption identifies a multicast group to leave on some +// interface. +type RemoveMembershipOption MembershipOption + +func (*RemoveMembershipOption) isSettableSocketOption() {} + +// SocketDetachFilterOption is used by SetSockOpt to detach a previously attached +// classic BPF filter on a given endpoint. +type SocketDetachFilterOption int + +func (*SocketDetachFilterOption) isSettableSocketOption() {} + +// OriginalDestinationOption is used to get the original destination address +// and port of a redirected packet. +type OriginalDestinationOption FullAddress + +func (*OriginalDestinationOption) isGettableSocketOption() {} + +// TCPTimeWaitReuseOption is used stack.(*Stack).TransportProtocolOption to +// specify if the stack can reuse the port bound by an endpoint in TIME-WAIT for +// new connections when it is safe from protocol viewpoint. +type TCPTimeWaitReuseOption uint8 + +func (*TCPTimeWaitReuseOption) isGettableSocketOption() {} + +func (*TCPTimeWaitReuseOption) isSettableSocketOption() {} + +func (*TCPTimeWaitReuseOption) isGettableTransportProtocolOption() {} + +func (*TCPTimeWaitReuseOption) isSettableTransportProtocolOption() {} + +const ( + // TCPTimeWaitReuseDisabled indicates reuse of port bound by endpoints in TIME-WAIT cannot + // be reused for new connections. + TCPTimeWaitReuseDisabled TCPTimeWaitReuseOption = iota + + // TCPTimeWaitReuseGlobal indicates reuse of port bound by endpoints in TIME-WAIT can + // be reused for new connections irrespective of the src/dest addresses. + TCPTimeWaitReuseGlobal + + // TCPTimeWaitReuseLoopbackOnly indicates reuse of port bound by endpoint in TIME-WAIT can + // only be reused if the connection was a connection over loopback. i.e. src/dest addresses + // are loopback addresses. + TCPTimeWaitReuseLoopbackOnly +) + +// LingerOption is used by SetSockOpt/GetSockOpt to set/get the +// duration for which a socket lingers before returning from Close. +// +// +marshal +// +stateify savable +type LingerOption struct { + Enabled bool + Timeout time.Duration +} + +// IPPacketInfo is the message structure for IP_PKTINFO. +// +// +stateify savable +type IPPacketInfo struct { + // NIC is the ID of the NIC to be used. + NIC NICID + + // LocalAddr is the local address. + LocalAddr Address + + // DestinationAddr is the destination address found in the IP header. + DestinationAddr Address +} + +// IPv6PacketInfo is the message structure for IPV6_PKTINFO. +// +// +stateify savable +type IPv6PacketInfo struct { + Addr Address + NIC NICID +} + +// SendBufferSizeOption is used by stack.(Stack*).Option/SetOption to +// get/set the default, min and max send buffer sizes. +// +// +stateify savable +type SendBufferSizeOption struct { + // Min is the minimum size for send buffer. + Min int + + // Default is the default size for send buffer. + Default int + + // Max is the maximum size for send buffer. + Max int +} + +// ReceiveBufferSizeOption is used by stack.(Stack*).Option/SetOption to +// get/set the default, min and max receive buffer sizes. +// +// +stateify savable +type ReceiveBufferSizeOption struct { + // Min is the minimum size for send buffer. + Min int + + // Default is the default size for send buffer. + Default int + + // Max is the maximum size for send buffer. + Max int +} + +// GetSendBufferLimits is used to get the send buffer size limits. +type GetSendBufferLimits func(StackHandler) SendBufferSizeOption + +// GetStackSendBufferLimits is used to get default, min and max send buffer size. +func GetStackSendBufferLimits(so StackHandler) SendBufferSizeOption { + var ss SendBufferSizeOption + if err := so.Option(&ss); err != nil { + panic(fmt.Sprintf("s.Option(%#v) = %s", ss, err)) + } + return ss +} + +// GetReceiveBufferLimits is used to get the send buffer size limits. +type GetReceiveBufferLimits func(StackHandler) ReceiveBufferSizeOption + +// GetStackReceiveBufferLimits is used to get default, min and max send buffer size. +func GetStackReceiveBufferLimits(so StackHandler) ReceiveBufferSizeOption { + var ss ReceiveBufferSizeOption + if err := so.Option(&ss); err != nil { + panic(fmt.Sprintf("s.Option(%#v) = %s", ss, err)) + } + return ss +} + +// Route is a row in the routing table. It specifies through which NIC (and +// gateway) sets of packets should be routed. A row is considered viable if the +// masked target address matches the destination address in the row. +// +// +stateify savable +type Route struct { + RouteEntry + + // Destination must contain the target address for this row to be viable. + Destination Subnet + + // Gateway is the gateway to be used if this row is viable. + Gateway Address + + // NIC is the id of the nic to be used if this row is viable. + NIC NICID + + // SourceHint indicates a preferred source address to use when NICs + // have multiple addresses. + SourceHint Address + + // MTU is the maximum transmission unit to use for this route. + // If MTU is 0, this field is ignored and the MTU of the NIC for which this route + // is configured is used for egress packets. + MTU uint32 +} + +// String implements the fmt.Stringer interface. +func (r Route) String() string { + var out strings.Builder + _, _ = fmt.Fprintf(&out, "%s", r.Destination) + if r.Gateway.length > 0 { + _, _ = fmt.Fprintf(&out, " via %s", r.Gateway) + } + _, _ = fmt.Fprintf(&out, " nic %d", r.NIC) + return out.String() +} + +// Equal returns true if the given Route is equal to this Route. +func (r Route) Equal(to Route) bool { + // NOTE: This relies on the fact that r.Destination == to.Destination + return r.Destination.Equal(to.Destination) && r.NIC == to.NIC +} + +// TransportProtocolNumber is the number of a transport protocol. +type TransportProtocolNumber uint32 + +// NetworkProtocolNumber is the EtherType of a network protocol in an Ethernet +// frame. +// +// See: https://www.iana.org/assignments/ieee-802-numbers/ieee-802-numbers.xhtml +type NetworkProtocolNumber uint32 + +// A StatCounter keeps track of a statistic. +// +// +stateify savable +type StatCounter struct { + count atomicbitops.Uint64 +} + +// Increment adds one to the counter. +func (s *StatCounter) Increment() { + s.IncrementBy(1) +} + +// Decrement minuses one to the counter. +func (s *StatCounter) Decrement() { + s.IncrementBy(^uint64(0)) +} + +// Value returns the current value of the counter. +func (s *StatCounter) Value() uint64 { + return s.count.Load() +} + +// IncrementBy increments the counter by v. +func (s *StatCounter) IncrementBy(v uint64) { + s.count.Add(v) +} + +func (s *StatCounter) String() string { + return strconv.FormatUint(s.Value(), 10) +} + +// A MultiCounterStat keeps track of two counters at once. +// +// +stateify savable +type MultiCounterStat struct { + a *StatCounter + b *StatCounter +} + +// Init sets both internal counters to point to a and b. +func (m *MultiCounterStat) Init(a, b *StatCounter) { + m.a = a + m.b = b +} + +// Increment adds one to the counters. +func (m *MultiCounterStat) Increment() { + m.a.Increment() + m.b.Increment() +} + +// IncrementBy increments the counters by v. +func (m *MultiCounterStat) IncrementBy(v uint64) { + m.a.IncrementBy(v) + m.b.IncrementBy(v) +} + +// ICMPv4PacketStats enumerates counts for all ICMPv4 packet types. +// +// +stateify savable +type ICMPv4PacketStats struct { + // LINT.IfChange(ICMPv4PacketStats) + + // EchoRequest is the number of ICMPv4 echo packets counted. + EchoRequest *StatCounter + + // EchoReply is the number of ICMPv4 echo reply packets counted. + EchoReply *StatCounter + + // DstUnreachable is the number of ICMPv4 destination unreachable packets + // counted. + DstUnreachable *StatCounter + + // SrcQuench is the number of ICMPv4 source quench packets counted. + SrcQuench *StatCounter + + // Redirect is the number of ICMPv4 redirect packets counted. + Redirect *StatCounter + + // TimeExceeded is the number of ICMPv4 time exceeded packets counted. + TimeExceeded *StatCounter + + // ParamProblem is the number of ICMPv4 parameter problem packets counted. + ParamProblem *StatCounter + + // Timestamp is the number of ICMPv4 timestamp packets counted. + Timestamp *StatCounter + + // TimestampReply is the number of ICMPv4 timestamp reply packets counted. + TimestampReply *StatCounter + + // InfoRequest is the number of ICMPv4 information request packets counted. + InfoRequest *StatCounter + + // InfoReply is the number of ICMPv4 information reply packets counted. + InfoReply *StatCounter + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterICMPv4PacketStats) +} + +// ICMPv4SentPacketStats collects outbound ICMPv4-specific stats. +// +// +stateify savable +type ICMPv4SentPacketStats struct { + // LINT.IfChange(ICMPv4SentPacketStats) + + ICMPv4PacketStats + + // Dropped is the number of ICMPv4 packets dropped due to link layer errors. + Dropped *StatCounter + + // RateLimited is the number of ICMPv4 packets dropped due to rate limit being + // exceeded. + RateLimited *StatCounter + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterICMPv4SentPacketStats) +} + +// ICMPv4ReceivedPacketStats collects inbound ICMPv4-specific stats. +// +// +stateify savable +type ICMPv4ReceivedPacketStats struct { + // LINT.IfChange(ICMPv4ReceivedPacketStats) + + ICMPv4PacketStats + + // Invalid is the number of invalid ICMPv4 packets received. + Invalid *StatCounter + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterICMPv4ReceivedPacketStats) +} + +// ICMPv4Stats collects ICMPv4-specific stats. +// +// +stateify savable +type ICMPv4Stats struct { + // LINT.IfChange(ICMPv4Stats) + + // PacketsSent contains statistics about sent packets. + PacketsSent ICMPv4SentPacketStats + + // PacketsReceived contains statistics about received packets. + PacketsReceived ICMPv4ReceivedPacketStats + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterICMPv4Stats) +} + +// ICMPv6PacketStats enumerates counts for all ICMPv6 packet types. +// +// +stateify savable +type ICMPv6PacketStats struct { + // LINT.IfChange(ICMPv6PacketStats) + + // EchoRequest is the number of ICMPv6 echo request packets counted. + EchoRequest *StatCounter + + // EchoReply is the number of ICMPv6 echo reply packets counted. + EchoReply *StatCounter + + // DstUnreachable is the number of ICMPv6 destination unreachable packets + // counted. + DstUnreachable *StatCounter + + // PacketTooBig is the number of ICMPv6 packet too big packets counted. + PacketTooBig *StatCounter + + // TimeExceeded is the number of ICMPv6 time exceeded packets counted. + TimeExceeded *StatCounter + + // ParamProblem is the number of ICMPv6 parameter problem packets counted. + ParamProblem *StatCounter + + // RouterSolicit is the number of ICMPv6 router solicit packets counted. + RouterSolicit *StatCounter + + // RouterAdvert is the number of ICMPv6 router advert packets counted. + RouterAdvert *StatCounter + + // NeighborSolicit is the number of ICMPv6 neighbor solicit packets counted. + NeighborSolicit *StatCounter + + // NeighborAdvert is the number of ICMPv6 neighbor advert packets counted. + NeighborAdvert *StatCounter + + // RedirectMsg is the number of ICMPv6 redirect message packets counted. + RedirectMsg *StatCounter + + // MulticastListenerQuery is the number of Multicast Listener Query messages + // counted. + MulticastListenerQuery *StatCounter + + // MulticastListenerReport is the number of Multicast Listener Report messages + // counted. + MulticastListenerReport *StatCounter + + // MulticastListenerReportV2 is the number of Multicast Listener Report + // messages counted. + MulticastListenerReportV2 *StatCounter + + // MulticastListenerDone is the number of Multicast Listener Done messages + // counted. + MulticastListenerDone *StatCounter + + // LINT.ThenChange(network/ipv6/stats.go:multiCounterICMPv6PacketStats) +} + +// ICMPv6SentPacketStats collects outbound ICMPv6-specific stats. +// +// +stateify savable +type ICMPv6SentPacketStats struct { + // LINT.IfChange(ICMPv6SentPacketStats) + + ICMPv6PacketStats + + // Dropped is the number of ICMPv6 packets dropped due to link layer errors. + Dropped *StatCounter + + // RateLimited is the number of ICMPv6 packets dropped due to rate limit being + // exceeded. + RateLimited *StatCounter + + // LINT.ThenChange(network/ipv6/stats.go:multiCounterICMPv6SentPacketStats) +} + +// ICMPv6ReceivedPacketStats collects inbound ICMPv6-specific stats. +// +// +stateify savable +type ICMPv6ReceivedPacketStats struct { + // LINT.IfChange(ICMPv6ReceivedPacketStats) + + ICMPv6PacketStats + + // Unrecognized is the number of ICMPv6 packets received that the transport + // layer does not know how to parse. + Unrecognized *StatCounter + + // Invalid is the number of invalid ICMPv6 packets received. + Invalid *StatCounter + + // RouterOnlyPacketsDroppedByHost is the number of ICMPv6 packets dropped due + // to being router-specific packets. + RouterOnlyPacketsDroppedByHost *StatCounter + + // LINT.ThenChange(network/ipv6/stats.go:multiCounterICMPv6ReceivedPacketStats) +} + +// ICMPv6Stats collects ICMPv6-specific stats. +// +// +stateify savable +type ICMPv6Stats struct { + // LINT.IfChange(ICMPv6Stats) + + // PacketsSent contains statistics about sent packets. + PacketsSent ICMPv6SentPacketStats + + // PacketsReceived contains statistics about received packets. + PacketsReceived ICMPv6ReceivedPacketStats + + // LINT.ThenChange(network/ipv6/stats.go:multiCounterICMPv6Stats) +} + +// ICMPStats collects ICMP-specific stats (both v4 and v6). +// +// +stateify savable +type ICMPStats struct { + // V4 contains the ICMPv4-specifics stats. + V4 ICMPv4Stats + + // V6 contains the ICMPv4-specifics stats. + V6 ICMPv6Stats +} + +// IGMPPacketStats enumerates counts for all IGMP packet types. +// +// +stateify savable +type IGMPPacketStats struct { + // LINT.IfChange(IGMPPacketStats) + + // MembershipQuery is the number of Membership Query messages counted. + MembershipQuery *StatCounter + + // V1MembershipReport is the number of Version 1 Membership Report messages + // counted. + V1MembershipReport *StatCounter + + // V2MembershipReport is the number of Version 2 Membership Report messages + // counted. + V2MembershipReport *StatCounter + + // V3MembershipReport is the number of Version 3 Membership Report messages + // counted. + V3MembershipReport *StatCounter + + // LeaveGroup is the number of Leave Group messages counted. + LeaveGroup *StatCounter + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterIGMPPacketStats) +} + +// IGMPSentPacketStats collects outbound IGMP-specific stats. +// +// +stateify savable +type IGMPSentPacketStats struct { + // LINT.IfChange(IGMPSentPacketStats) + + IGMPPacketStats + + // Dropped is the number of IGMP packets dropped. + Dropped *StatCounter + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterIGMPSentPacketStats) +} + +// IGMPReceivedPacketStats collects inbound IGMP-specific stats. +// +// +stateify savable +type IGMPReceivedPacketStats struct { + // LINT.IfChange(IGMPReceivedPacketStats) + + IGMPPacketStats + + // Invalid is the number of invalid IGMP packets received. + Invalid *StatCounter + + // ChecksumErrors is the number of IGMP packets dropped due to bad checksums. + ChecksumErrors *StatCounter + + // Unrecognized is the number of unrecognized messages counted, these are + // silently ignored for forward-compatibility. + Unrecognized *StatCounter + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterIGMPReceivedPacketStats) +} + +// IGMPStats collects IGMP-specific stats. +// +// +stateify savable +type IGMPStats struct { + // LINT.IfChange(IGMPStats) + + // PacketsSent contains statistics about sent packets. + PacketsSent IGMPSentPacketStats + + // PacketsReceived contains statistics about received packets. + PacketsReceived IGMPReceivedPacketStats + + // LINT.ThenChange(network/ipv4/stats.go:multiCounterIGMPStats) +} + +// IPForwardingStats collects stats related to IP forwarding (both v4 and v6). +// +// +stateify savable +type IPForwardingStats struct { + // LINT.IfChange(IPForwardingStats) + + // Unrouteable is the number of IP packets received which were dropped + // because a route to their destination could not be constructed. + Unrouteable *StatCounter + + // ExhaustedTTL is the number of IP packets received which were dropped + // because their TTL was exhausted. + ExhaustedTTL *StatCounter + + // InitializingSource is the number of IP packets which were dropped + // because they contained a source address that may only be used on the local + // network as part of initialization work. + InitializingSource *StatCounter + + // LinkLocalSource is the number of IP packets which were dropped + // because they contained a link-local source address. + LinkLocalSource *StatCounter + + // LinkLocalDestination is the number of IP packets which were dropped + // because they contained a link-local destination address. + LinkLocalDestination *StatCounter + + // PacketTooBig is the number of IP packets which were dropped because they + // were too big for the outgoing MTU. + PacketTooBig *StatCounter + + // HostUnreachable is the number of IP packets received which could not be + // successfully forwarded due to an unresolvable next hop. + HostUnreachable *StatCounter + + // ExtensionHeaderProblem is the number of IP packets which were dropped + // because of a problem encountered when processing an IPv6 extension + // header. + ExtensionHeaderProblem *StatCounter + + // UnexpectedMulticastInputInterface is the number of multicast packets that + // were received on an interface that did not match the corresponding route's + // expected input interface. + UnexpectedMulticastInputInterface *StatCounter + + // UnknownOutputEndpoint is the number of packets that could not be forwarded + // because the output endpoint could not be found. + UnknownOutputEndpoint *StatCounter + + // NoMulticastPendingQueueBufferSpace is the number of multicast packets that + // were dropped due to insufficient buffer space in the pending packet queue. + NoMulticastPendingQueueBufferSpace *StatCounter + + // OutgoingDeviceNoBufferSpace is the number of packets that were dropped due + // to insufficient space in the outgoing device. + OutgoingDeviceNoBufferSpace *StatCounter + + // Errors is the number of IP packets received which could not be + // successfully forwarded. + Errors *StatCounter + + // OutgoingDeviceClosedForSend is the number of packets that were dropped due + // to the outgoing device being closed for send. + OutgoingDeviceClosedForSend *StatCounter + + // LINT.ThenChange(network/internal/ip/stats.go:MultiCounterIPForwardingStats) +} + +// IPStats collects IP-specific stats (both v4 and v6). +// +// +stateify savable +type IPStats struct { + // LINT.IfChange(IPStats) + + // PacketsReceived is the number of IP packets received from the link layer. + PacketsReceived *StatCounter + + // ValidPacketsReceived is the number of valid IP packets that reached the IP + // layer. + ValidPacketsReceived *StatCounter + + // DisabledPacketsReceived is the number of IP packets received from the link + // layer when the IP layer is disabled. + DisabledPacketsReceived *StatCounter + + // InvalidDestinationAddressesReceived is the number of IP packets received + // with an unknown or invalid destination address. + InvalidDestinationAddressesReceived *StatCounter + + // InvalidSourceAddressesReceived is the number of IP packets received with a + // source address that should never have been received on the wire. + InvalidSourceAddressesReceived *StatCounter + + // PacketsDelivered is the number of incoming IP packets that are successfully + // delivered to the transport layer. + PacketsDelivered *StatCounter + + // PacketsSent is the number of IP packets sent via WritePacket. + PacketsSent *StatCounter + + // OutgoingPacketErrors is the number of IP packets which failed to write to a + // link-layer endpoint. + OutgoingPacketErrors *StatCounter + + // MalformedPacketsReceived is the number of IP Packets that were dropped due + // to the IP packet header failing validation checks. + MalformedPacketsReceived *StatCounter + + // MalformedFragmentsReceived is the number of IP Fragments that were dropped + // due to the fragment failing validation checks. + MalformedFragmentsReceived *StatCounter + + // IPTablesPreroutingDropped is the number of IP packets dropped in the + // Prerouting chain. + IPTablesPreroutingDropped *StatCounter + + // IPTablesInputDropped is the number of IP packets dropped in the Input + // chain. + IPTablesInputDropped *StatCounter + + // IPTablesForwardDropped is the number of IP packets dropped in the Forward + // chain. + IPTablesForwardDropped *StatCounter + + // IPTablesOutputDropped is the number of IP packets dropped in the Output + // chain. + IPTablesOutputDropped *StatCounter + + // IPTablesPostroutingDropped is the number of IP packets dropped in the + // Postrouting chain. + IPTablesPostroutingDropped *StatCounter + + // TODO(https://gvisor.dev/issues/5529): Move the IPv4-only option stats out + // of IPStats. + // OptionTimestampReceived is the number of Timestamp options seen. + OptionTimestampReceived *StatCounter + + // OptionRecordRouteReceived is the number of Record Route options seen. + OptionRecordRouteReceived *StatCounter + + // OptionRouterAlertReceived is the number of Router Alert options seen. + OptionRouterAlertReceived *StatCounter + + // OptionUnknownReceived is the number of unknown IP options seen. + OptionUnknownReceived *StatCounter + + // Forwarding collects stats related to IP forwarding. + Forwarding IPForwardingStats + + // LINT.ThenChange(network/internal/ip/stats.go:MultiCounterIPStats) +} + +// ARPStats collects ARP-specific stats. +// +// +stateify savable +type ARPStats struct { + // LINT.IfChange(ARPStats) + + // PacketsReceived is the number of ARP packets received from the link layer. + PacketsReceived *StatCounter + + // DisabledPacketsReceived is the number of ARP packets received from the link + // layer when the ARP layer is disabled. + DisabledPacketsReceived *StatCounter + + // MalformedPacketsReceived is the number of ARP packets that were dropped due + // to being malformed. + MalformedPacketsReceived *StatCounter + + // RequestsReceived is the number of ARP requests received. + RequestsReceived *StatCounter + + // RequestsReceivedUnknownTargetAddress is the number of ARP requests that + // were targeted to an interface different from the one it was received on. + RequestsReceivedUnknownTargetAddress *StatCounter + + // OutgoingRequestInterfaceHasNoLocalAddressErrors is the number of failures + // to send an ARP request because the interface has no network address + // assigned to it. + OutgoingRequestInterfaceHasNoLocalAddressErrors *StatCounter + + // OutgoingRequestBadLocalAddressErrors is the number of failures to send an + // ARP request with a bad local address. + OutgoingRequestBadLocalAddressErrors *StatCounter + + // OutgoingRequestsDropped is the number of ARP requests which failed to write + // to a link-layer endpoint. + OutgoingRequestsDropped *StatCounter + + // OutgoingRequestSent is the number of ARP requests successfully written to a + // link-layer endpoint. + OutgoingRequestsSent *StatCounter + + // RepliesReceived is the number of ARP replies received. + RepliesReceived *StatCounter + + // OutgoingRepliesDropped is the number of ARP replies which failed to write + // to a link-layer endpoint. + OutgoingRepliesDropped *StatCounter + + // OutgoingRepliesSent is the number of ARP replies successfully written to a + // link-layer endpoint. + OutgoingRepliesSent *StatCounter + + // LINT.ThenChange(network/arp/stats.go:multiCounterARPStats) +} + +// TCPStats collects TCP-specific stats. +// +// +stateify savable +type TCPStats struct { + // ActiveConnectionOpenings is the number of connections opened + // successfully via Connect. + ActiveConnectionOpenings *StatCounter + + // PassiveConnectionOpenings is the number of connections opened + // successfully via Listen. + PassiveConnectionOpenings *StatCounter + + // CurrentEstablished is the number of TCP connections for which the + // current state is ESTABLISHED. + CurrentEstablished *StatCounter + + // CurrentConnected is the number of TCP connections that + // are in connected state. + CurrentConnected *StatCounter + + // EstablishedResets is the number of times TCP connections have made + // a direct transition to the CLOSED state from either the + // ESTABLISHED state or the CLOSE-WAIT state. + EstablishedResets *StatCounter + + // EstablishedClosed is the number of times established TCP connections + // made a transition to CLOSED state. + EstablishedClosed *StatCounter + + // EstablishedTimedout is the number of times an established connection + // was reset because of keep-alive time out. + EstablishedTimedout *StatCounter + + // ListenOverflowSynDrop is the number of times the listen queue overflowed + // and a SYN was dropped. + ListenOverflowSynDrop *StatCounter + + // ListenOverflowAckDrop is the number of times the final ACK + // in the handshake was dropped due to overflow. + ListenOverflowAckDrop *StatCounter + + // ListenOverflowCookieSent is the number of times a SYN cookie was sent. + ListenOverflowSynCookieSent *StatCounter + + // ListenOverflowSynCookieRcvd is the number of times a valid SYN + // cookie was received. + ListenOverflowSynCookieRcvd *StatCounter + + // ListenOverflowInvalidSynCookieRcvd is the number of times an invalid SYN cookie + // was received. + ListenOverflowInvalidSynCookieRcvd *StatCounter + + // FailedConnectionAttempts is the number of calls to Connect or Listen + // (active and passive openings, respectively) that end in an error. + FailedConnectionAttempts *StatCounter + + // ValidSegmentsReceived is the number of TCP segments received that + // the transport layer successfully parsed. + ValidSegmentsReceived *StatCounter + + // InvalidSegmentsReceived is the number of TCP segments received that + // the transport layer could not parse. + InvalidSegmentsReceived *StatCounter + + // SegmentsSent is the number of TCP segments sent. + SegmentsSent *StatCounter + + // SegmentSendErrors is the number of TCP segments failed to be sent. + SegmentSendErrors *StatCounter + + // ResetsSent is the number of TCP resets sent. + ResetsSent *StatCounter + + // ResetsReceived is the number of TCP resets received. + ResetsReceived *StatCounter + + // Retransmits is the number of TCP segments retransmitted. + Retransmits *StatCounter + + // FastRecovery is the number of times Fast Recovery was used to + // recover from packet loss. + FastRecovery *StatCounter + + // SACKRecovery is the number of times SACK Recovery was used to + // recover from packet loss. + SACKRecovery *StatCounter + + // TLPRecovery is the number of times recovery was accomplished by the tail + // loss probe. + TLPRecovery *StatCounter + + // SlowStartRetransmits is the number of segments retransmitted in slow + // start. + SlowStartRetransmits *StatCounter + + // FastRetransmit is the number of segments retransmitted in fast + // recovery. + FastRetransmit *StatCounter + + // Timeouts is the number of times the RTO expired. + Timeouts *StatCounter + + // ChecksumErrors is the number of segments dropped due to bad checksums. + ChecksumErrors *StatCounter + + // FailedPortReservations is the number of times TCP failed to reserve + // a port. + FailedPortReservations *StatCounter + + // SegmentsAckedWithDSACK is the number of segments acknowledged with + // DSACK. + SegmentsAckedWithDSACK *StatCounter + + // SpuriousRecovery is the number of times the connection entered loss + // recovery spuriously. + SpuriousRecovery *StatCounter + + // SpuriousRTORecovery is the number of spurious RTOs. + SpuriousRTORecovery *StatCounter + + // ForwardMaxInFlightDrop is the number of connection requests that are + // dropped due to exceeding the maximum number of in-flight connection + // requests. + ForwardMaxInFlightDrop *StatCounter +} + +// UDPStats collects UDP-specific stats. +// +// +stateify savable +type UDPStats struct { + // PacketsReceived is the number of UDP datagrams received via + // HandlePacket. + PacketsReceived *StatCounter + + // UnknownPortErrors is the number of incoming UDP datagrams dropped + // because they did not have a known destination port. + UnknownPortErrors *StatCounter + + // ReceiveBufferErrors is the number of incoming UDP datagrams dropped + // due to the receiving buffer being in an invalid state. + ReceiveBufferErrors *StatCounter + + // MalformedPacketsReceived is the number of incoming UDP datagrams + // dropped due to the UDP header being in a malformed state. + MalformedPacketsReceived *StatCounter + + // PacketsSent is the number of UDP datagrams sent via sendUDP. + PacketsSent *StatCounter + + // PacketSendErrors is the number of datagrams failed to be sent. + PacketSendErrors *StatCounter + + // ChecksumErrors is the number of datagrams dropped due to bad checksums. + ChecksumErrors *StatCounter +} + +// NICNeighborStats holds metrics for the neighbor table. +// +// +stateify savable +type NICNeighborStats struct { + // LINT.IfChange(NICNeighborStats) + + // UnreachableEntryLookups counts the number of lookups performed on an + // entry in Unreachable state. + UnreachableEntryLookups *StatCounter + + // DroppedConfirmationForNoninitiatedNeighbor counts the number of neighbor + // responses that were dropped because they didn't match an entry in the + // cache. + DroppedConfirmationForNoninitiatedNeighbor *StatCounter + + // DroppedInvalidLinkAddressConfirmations counts the number of neighbor + // responses that were ignored because they had an invalid source link-layer + // address. + DroppedInvalidLinkAddressConfirmations *StatCounter + + // LINT.ThenChange(stack/nic_stats.go:multiCounterNICNeighborStats) +} + +// NICPacketStats holds basic packet statistics. +// +// +stateify savable +type NICPacketStats struct { + // LINT.IfChange(NICPacketStats) + + // Packets is the number of packets counted. + Packets *StatCounter + + // Bytes is the number of bytes counted. + Bytes *StatCounter + + // LINT.ThenChange(stack/nic_stats.go:multiCounterNICPacketStats) +} + +// IntegralStatCounterMap holds a map associating integral keys with +// StatCounters. +// +// +stateify savable +type IntegralStatCounterMap struct { + mu sync.RWMutex `state:"nosave"` + // +checklocks:mu + counterMap map[uint64]*StatCounter +} + +// Keys returns all keys present in the map. +func (m *IntegralStatCounterMap) Keys() []uint64 { + m.mu.RLock() + defer m.mu.RUnlock() + var keys []uint64 + for k := range m.counterMap { + keys = append(keys, k) + } + return keys +} + +// Get returns the counter mapped by the provided key. +func (m *IntegralStatCounterMap) Get(key uint64) (*StatCounter, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + counter, ok := m.counterMap[key] + return counter, ok +} + +// Init initializes the map. +func (m *IntegralStatCounterMap) Init() { + m.mu.Lock() + defer m.mu.Unlock() + m.counterMap = make(map[uint64]*StatCounter) +} + +// Increment increments the counter associated with the provided key. +func (m *IntegralStatCounterMap) Increment(key uint64) { + m.mu.RLock() + counter, ok := m.counterMap[key] + m.mu.RUnlock() + + if !ok { + m.mu.Lock() + counter, ok = m.counterMap[key] + if !ok { + counter = new(StatCounter) + m.counterMap[key] = counter + } + m.mu.Unlock() + } + counter.Increment() +} + +// A MultiIntegralStatCounterMap keeps track of two integral counter maps at +// once. +// +// +stateify savable +type MultiIntegralStatCounterMap struct { + a *IntegralStatCounterMap + b *IntegralStatCounterMap +} + +// Init sets the internal integral counter maps to point to a and b. +func (m *MultiIntegralStatCounterMap) Init(a, b *IntegralStatCounterMap) { + m.a = a + m.b = b +} + +// Increment increments the counter in each map corresponding to the +// provided key. +func (m *MultiIntegralStatCounterMap) Increment(key uint64) { + m.a.Increment(key) + m.b.Increment(key) +} + +// NICStats holds NIC statistics. +// +// +stateify savable +type NICStats struct { + // LINT.IfChange(NICStats) + + // UnknownL3ProtocolRcvdPacketCounts records the number of packets received + // for each unknown or unsupported network protocol number. + UnknownL3ProtocolRcvdPacketCounts *IntegralStatCounterMap + + // UnknownL4ProtocolRcvdPacketCounts records the number of packets received + // for each unknown or unsupported transport protocol number. + UnknownL4ProtocolRcvdPacketCounts *IntegralStatCounterMap + + // MalformedL4RcvdPackets is the number of packets received by a NIC that + // could not be delivered to a transport endpoint because the L4 header could + // not be parsed. + MalformedL4RcvdPackets *StatCounter + + // Tx contains statistics about transmitted packets. + Tx NICPacketStats + + // TxPacketsDroppedNoBufferSpace is the number of packets dropepd due to the + // NIC not having enough buffer space to send the packet. + // + // Packets may be dropped with a no buffer space error when the device TX + // queue is full. + TxPacketsDroppedNoBufferSpace *StatCounter + + // Rx contains statistics about received packets. + Rx NICPacketStats + + // DisabledRx contains statistics about received packets on disabled NICs. + DisabledRx NICPacketStats + + // Neighbor contains statistics about neighbor entries. + Neighbor NICNeighborStats + + // LINT.ThenChange(stack/nic_stats.go:multiCounterNICStats) +} + +// FillIn returns a copy of s with nil fields initialized to new StatCounters. +func (s NICStats) FillIn() NICStats { + InitStatCounters(reflect.ValueOf(&s).Elem()) + return s +} + +// Stats holds statistics about the networking stack. +// +// +stateify savable +type Stats struct { + // TODO(https://gvisor.dev/issues/5986): Make the DroppedPackets stat less + // ambiguous. + + // DroppedPackets is the number of packets dropped at the transport layer. + DroppedPackets *StatCounter + + // NICs is an aggregation of every NIC's statistics. These should not be + // incremented using this field, but using the relevant NIC multicounters. + NICs NICStats + + // ICMP is an aggregation of every NetworkEndpoint's ICMP statistics (both v4 + // and v6). These should not be incremented using this field, but using the + // relevant NetworkEndpoint ICMP multicounters. + ICMP ICMPStats + + // IGMP is an aggregation of every NetworkEndpoint's IGMP statistics. These + // should not be incremented using this field, but using the relevant + // NetworkEndpoint IGMP multicounters. + IGMP IGMPStats + + // IP is an aggregation of every NetworkEndpoint's IP statistics. These should + // not be incremented using this field, but using the relevant NetworkEndpoint + // IP multicounters. + IP IPStats + + // ARP is an aggregation of every NetworkEndpoint's ARP statistics. These + // should not be incremented using this field, but using the relevant + // NetworkEndpoint ARP multicounters. + ARP ARPStats + + // TCP holds TCP-specific stats. + TCP TCPStats + + // UDP holds UDP-specific stats. + UDP UDPStats +} + +// ReceiveErrors collects packet receive errors within transport endpoint. +// +// +stateify savable +type ReceiveErrors struct { + // ReceiveBufferOverflow is the number of received packets dropped + // due to the receive buffer being full. + ReceiveBufferOverflow StatCounter + + // MalformedPacketsReceived is the number of incoming packets + // dropped due to the packet header being in a malformed state. + MalformedPacketsReceived StatCounter + + // ClosedReceiver is the number of received packets dropped because + // of receiving endpoint state being closed. + ClosedReceiver StatCounter + + // ChecksumErrors is the number of packets dropped due to bad checksums. + ChecksumErrors StatCounter +} + +// SendErrors collects packet send errors within the transport layer for an +// endpoint. +// +// +stateify savable +type SendErrors struct { + // SendToNetworkFailed is the number of packets failed to be written to + // the network endpoint. + SendToNetworkFailed StatCounter + + // NoRoute is the number of times we failed to resolve IP route. + NoRoute StatCounter +} + +// ReadErrors collects segment read errors from an endpoint read call. +// +// +stateify savable +type ReadErrors struct { + // ReadClosed is the number of received packet drops because the endpoint + // was shutdown for read. + ReadClosed StatCounter + + // InvalidEndpointState is the number of times we found the endpoint state + // to be unexpected. + InvalidEndpointState StatCounter + + // NotConnected is the number of times we tried to read but found that the + // endpoint was not connected. + NotConnected StatCounter +} + +// WriteErrors collects packet write errors from an endpoint write call. +// +// +stateify savable +type WriteErrors struct { + // WriteClosed is the number of packet drops because the endpoint + // was shutdown for write. + WriteClosed StatCounter + + // InvalidEndpointState is the number of times we found the endpoint state + // to be unexpected. + InvalidEndpointState StatCounter + + // InvalidArgs is the number of times invalid input arguments were + // provided for endpoint Write call. + InvalidArgs StatCounter +} + +// TransportEndpointStats collects statistics about the endpoint. +// +// +stateify savable +type TransportEndpointStats struct { + // PacketsReceived is the number of successful packet receives. + PacketsReceived StatCounter + + // PacketsSent is the number of successful packet sends. + PacketsSent StatCounter + + // ReceiveErrors collects packet receive errors within transport layer. + ReceiveErrors ReceiveErrors + + // ReadErrors collects packet read errors from an endpoint read call. + ReadErrors ReadErrors + + // SendErrors collects packet send errors within the transport layer. + SendErrors SendErrors + + // WriteErrors collects packet write errors from an endpoint write call. + WriteErrors WriteErrors +} + +// IsEndpointStats is an empty method to implement the tcpip.EndpointStats +// marker interface. +func (*TransportEndpointStats) IsEndpointStats() {} + +// InitStatCounters initializes v's fields with nil StatCounter fields to new +// StatCounters. +func InitStatCounters(v reflect.Value) { + for i := 0; i < v.NumField(); i++ { + v := v.Field(i) + if s, ok := v.Addr().Interface().(**StatCounter); ok { + if *s == nil { + *s = new(StatCounter) + } + } else if s, ok := v.Addr().Interface().(**IntegralStatCounterMap); ok { + if *s == nil { + *s = new(IntegralStatCounterMap) + (*s).Init() + } + } else { + InitStatCounters(v) + } + } +} + +// FillIn returns a copy of s with nil fields initialized to new StatCounters. +func (s Stats) FillIn() Stats { + InitStatCounters(reflect.ValueOf(&s).Elem()) + return s +} + +// Clone clones a copy of the TransportEndpointStats into dst by atomically +// reading each field. +func (src *TransportEndpointStats) Clone(dst *TransportEndpointStats) { + clone(reflect.ValueOf(dst).Elem(), reflect.ValueOf(src).Elem()) +} + +func clone(dst reflect.Value, src reflect.Value) { + for i := 0; i < dst.NumField(); i++ { + d := dst.Field(i) + s := src.Field(i) + if c, ok := s.Addr().Interface().(*StatCounter); ok { + d.Addr().Interface().(*StatCounter).IncrementBy(c.Value()) + } else { + clone(d, s) + } + } +} + +// String implements the fmt.Stringer interface. +func (a Address) String() string { + switch l := a.Len(); l { + case 4: + return fmt.Sprintf("%d.%d.%d.%d", int(a.addr[0]), int(a.addr[1]), int(a.addr[2]), int(a.addr[3])) + case 16: + // Find the longest subsequence of hexadecimal zeros. + start, end := -1, -1 + for i := 0; i < a.Len(); i += 2 { + j := i + for j < a.Len() && a.addr[j] == 0 && a.addr[j+1] == 0 { + j += 2 + } + if j > i+2 && j-i > end-start { + start, end = i, j + } + } + + var b strings.Builder + for i := 0; i < a.Len(); i += 2 { + if i == start { + b.WriteString("::") + i = end + if end >= a.Len() { + break + } + } else if i > 0 { + b.WriteByte(':') + } + v := uint16(a.addr[i+0])<<8 | uint16(a.addr[i+1]) + if v == 0 { + b.WriteByte('0') + } else { + const digits = "0123456789abcdef" + for i := uint(3); i < 4; i-- { + if v := v >> (i * 4); v != 0 { + b.WriteByte(digits[v&0xf]) + } + } + } + } + return b.String() + default: + return fmt.Sprintf("%x", a.addr[:l]) + } +} + +// To4 converts the IPv4 address to a 4-byte representation. +// If the address is not an IPv4 address, To4 returns the empty Address. +func (a Address) To4() Address { + const ( + ipv4len = 4 + ipv6len = 16 + ) + if a.Len() == ipv4len { + return a + } + if a.Len() == ipv6len && + isZeros(a.addr[:10]) && + a.addr[10] == 0xff && + a.addr[11] == 0xff { + return AddrFrom4Slice(a.addr[12:16]) + } + return Address{} +} + +// isZeros reports whether addr is all zeros. +func isZeros(addr []byte) bool { + for _, b := range addr { + if b != 0 { + return false + } + } + return true +} + +// LinkAddress is a byte slice cast as a string that represents a link address. +// It is typically a 6-byte MAC address. +type LinkAddress string + +// String implements the fmt.Stringer interface. +func (a LinkAddress) String() string { + switch len(a) { + case 6: + return fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", a[0], a[1], a[2], a[3], a[4], a[5]) + default: + return fmt.Sprintf("%x", []byte(a)) + } +} + +// ParseMACAddress parses an IEEE 802 address. +// +// It must be in the format aa:bb:cc:dd:ee:ff or aa-bb-cc-dd-ee-ff. +func ParseMACAddress(s string) (LinkAddress, error) { + parts := strings.FieldsFunc(s, func(c rune) bool { + return c == ':' || c == '-' + }) + if len(parts) != LinkAddressSize { + return "", fmt.Errorf("inconsistent parts: %s", s) + } + addr := make([]byte, 0, len(parts)) + for _, part := range parts { + u, err := strconv.ParseUint(part, 16, 8) + if err != nil { + return "", fmt.Errorf("invalid hex digits: %s", s) + } + addr = append(addr, byte(u)) + } + return LinkAddress(addr), nil +} + +// GetRandMacAddr returns a mac address that can be used for local virtual devices. +func GetRandMacAddr() LinkAddress { + mac := make(net.HardwareAddr, LinkAddressSize) + rand.Read(mac) // Fill with random data. + mac[0] &^= 0x1 // Clear multicast bit. + mac[0] |= 0x2 // Set local assignment bit (IEEE802). + return LinkAddress(mac) +} + +// AddressWithPrefix is an address with its subnet prefix length. +// +// +stateify savable +type AddressWithPrefix struct { + // Address is a network address. + Address Address + + // PrefixLen is the subnet prefix length. + PrefixLen int +} + +// String implements the fmt.Stringer interface. +func (a AddressWithPrefix) String() string { + return fmt.Sprintf("%s/%d", a.Address, a.PrefixLen) +} + +// Subnet converts the address and prefix into a Subnet value and returns it. +func (a AddressWithPrefix) Subnet() Subnet { + addrLen := a.Address.length + if a.PrefixLen <= 0 { + return Subnet{ + address: Address{length: addrLen}, + mask: AddressMask{length: addrLen}, + } + } + if a.PrefixLen >= addrLen*8 { + sub := Subnet{ + address: a.Address, + mask: AddressMask{length: addrLen}, + } + for i := 0; i < addrLen; i++ { + sub.mask.mask[i] = 0xff + } + return sub + } + + sa := Address{length: addrLen} + sm := AddressMask{length: addrLen} + n := uint(a.PrefixLen) + for i := 0; i < addrLen; i++ { + if n >= 8 { + sa.addr[i] = a.Address.addr[i] + sm.mask[i] = 0xff + n -= 8 + continue + } + sm.mask[i] = ^byte(0xff >> n) + sa.addr[i] = a.Address.addr[i] & sm.mask[i] + n = 0 + } + + // For extra caution, call NewSubnet rather than directly creating the Subnet + // value. If that fails it indicates a serious bug in this code, so panic is + // in order. + s, err := NewSubnet(sa, sm) + if err != nil { + panic("invalid subnet: " + err.Error()) + } + return s +} + +// ProtocolAddress is an address and the network protocol it is associated +// with. +// +// +stateify savable +type ProtocolAddress struct { + // Protocol is the protocol of the address. + Protocol NetworkProtocolNumber + + // AddressWithPrefix is a network address with its subnet prefix length. + AddressWithPrefix AddressWithPrefix +} + +var ( + // danglingEndpointsMu protects access to danglingEndpoints. + danglingEndpointsMu sync.Mutex + + // danglingEndpoints tracks all dangling endpoints no longer owned by the app. + danglingEndpoints = make(map[Endpoint]struct{}) +) + +// GetDanglingEndpoints returns all dangling endpoints. +func GetDanglingEndpoints() []Endpoint { + danglingEndpointsMu.Lock() + es := make([]Endpoint, 0, len(danglingEndpoints)) + for e := range danglingEndpoints { + es = append(es, e) + } + danglingEndpointsMu.Unlock() + return es +} + +// ReleaseDanglingEndpoints clears out all all reference counted objects held by +// dangling endpoints. +func ReleaseDanglingEndpoints() { + // Get the dangling endpoints first to avoid locking around Release(), which + // can cause a lock inversion with endpoint.mu and danglingEndpointsMu. + // Calling Release on a dangling endpoint that has been deleted is a noop. + eps := GetDanglingEndpoints() + for _, ep := range eps { + ep.Abort() + } +} + +// AddDanglingEndpoint adds a dangling endpoint. +func AddDanglingEndpoint(e Endpoint) { + danglingEndpointsMu.Lock() + danglingEndpoints[e] = struct{}{} + danglingEndpointsMu.Unlock() +} + +// DeleteDanglingEndpoint removes a dangling endpoint. +func DeleteDanglingEndpoint(e Endpoint) { + danglingEndpointsMu.Lock() + delete(danglingEndpoints, e) + danglingEndpointsMu.Unlock() +} + +// AsyncLoading is the global barrier for asynchronous endpoint loading +// activities. +var AsyncLoading sync.WaitGroup diff --git a/pkg/tcpip/tcpip_linux_state_autogen.go b/pkg/tcpip/tcpip_linux_state_autogen.go new file mode 100644 index 0000000..cbd75fa --- /dev/null +++ b/pkg/tcpip/tcpip_linux_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build linux +// +build linux + +package tcpip diff --git a/pkg/tcpip/tcpip_state.go b/pkg/tcpip/tcpip_state.go new file mode 100644 index 0000000..0603ff0 --- /dev/null +++ b/pkg/tcpip/tcpip_state.go @@ -0,0 +1,28 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpip + +import ( + "context" + "time" +) + +func (c *ReceivableControlMessages) saveTimestamp() int64 { + return c.Timestamp.UnixNano() +} + +func (c *ReceivableControlMessages) loadTimestamp(_ context.Context, nsec int64) { + c.Timestamp = time.Unix(0, nsec) +} diff --git a/pkg/tcpip/tcpip_state_autogen.go b/pkg/tcpip/tcpip_state_autogen.go new file mode 100644 index 0000000..db7dcd7 --- /dev/null +++ b/pkg/tcpip/tcpip_state_autogen.go @@ -0,0 +1,3386 @@ +// automatically generated by stateify. + +package tcpip + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *ErrAborted) StateTypeName() string { + return "pkg/tcpip.ErrAborted" +} + +func (e *ErrAborted) StateFields() []string { + return []string{} +} + +func (e *ErrAborted) beforeSave() {} + +// +checklocksignore +func (e *ErrAborted) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrAborted) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrAborted) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrAddressFamilyNotSupported) StateTypeName() string { + return "pkg/tcpip.ErrAddressFamilyNotSupported" +} + +func (e *ErrAddressFamilyNotSupported) StateFields() []string { + return []string{} +} + +func (e *ErrAddressFamilyNotSupported) beforeSave() {} + +// +checklocksignore +func (e *ErrAddressFamilyNotSupported) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrAddressFamilyNotSupported) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrAddressFamilyNotSupported) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrAlreadyBound) StateTypeName() string { + return "pkg/tcpip.ErrAlreadyBound" +} + +func (e *ErrAlreadyBound) StateFields() []string { + return []string{} +} + +func (e *ErrAlreadyBound) beforeSave() {} + +// +checklocksignore +func (e *ErrAlreadyBound) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrAlreadyBound) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrAlreadyBound) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrAlreadyConnected) StateTypeName() string { + return "pkg/tcpip.ErrAlreadyConnected" +} + +func (e *ErrAlreadyConnected) StateFields() []string { + return []string{} +} + +func (e *ErrAlreadyConnected) beforeSave() {} + +// +checklocksignore +func (e *ErrAlreadyConnected) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrAlreadyConnected) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrAlreadyConnected) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrAlreadyConnecting) StateTypeName() string { + return "pkg/tcpip.ErrAlreadyConnecting" +} + +func (e *ErrAlreadyConnecting) StateFields() []string { + return []string{} +} + +func (e *ErrAlreadyConnecting) beforeSave() {} + +// +checklocksignore +func (e *ErrAlreadyConnecting) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrAlreadyConnecting) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrAlreadyConnecting) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrBadAddress) StateTypeName() string { + return "pkg/tcpip.ErrBadAddress" +} + +func (e *ErrBadAddress) StateFields() []string { + return []string{} +} + +func (e *ErrBadAddress) beforeSave() {} + +// +checklocksignore +func (e *ErrBadAddress) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrBadAddress) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrBadAddress) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrBadBuffer) StateTypeName() string { + return "pkg/tcpip.ErrBadBuffer" +} + +func (e *ErrBadBuffer) StateFields() []string { + return []string{} +} + +func (e *ErrBadBuffer) beforeSave() {} + +// +checklocksignore +func (e *ErrBadBuffer) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrBadBuffer) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrBadBuffer) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrBadLocalAddress) StateTypeName() string { + return "pkg/tcpip.ErrBadLocalAddress" +} + +func (e *ErrBadLocalAddress) StateFields() []string { + return []string{} +} + +func (e *ErrBadLocalAddress) beforeSave() {} + +// +checklocksignore +func (e *ErrBadLocalAddress) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrBadLocalAddress) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrBadLocalAddress) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrBroadcastDisabled) StateTypeName() string { + return "pkg/tcpip.ErrBroadcastDisabled" +} + +func (e *ErrBroadcastDisabled) StateFields() []string { + return []string{} +} + +func (e *ErrBroadcastDisabled) beforeSave() {} + +// +checklocksignore +func (e *ErrBroadcastDisabled) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrBroadcastDisabled) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrBroadcastDisabled) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrClosedForReceive) StateTypeName() string { + return "pkg/tcpip.ErrClosedForReceive" +} + +func (e *ErrClosedForReceive) StateFields() []string { + return []string{} +} + +func (e *ErrClosedForReceive) beforeSave() {} + +// +checklocksignore +func (e *ErrClosedForReceive) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrClosedForReceive) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrClosedForReceive) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrClosedForSend) StateTypeName() string { + return "pkg/tcpip.ErrClosedForSend" +} + +func (e *ErrClosedForSend) StateFields() []string { + return []string{} +} + +func (e *ErrClosedForSend) beforeSave() {} + +// +checklocksignore +func (e *ErrClosedForSend) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrClosedForSend) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrClosedForSend) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrConnectStarted) StateTypeName() string { + return "pkg/tcpip.ErrConnectStarted" +} + +func (e *ErrConnectStarted) StateFields() []string { + return []string{} +} + +func (e *ErrConnectStarted) beforeSave() {} + +// +checklocksignore +func (e *ErrConnectStarted) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrConnectStarted) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrConnectStarted) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrConnectionAborted) StateTypeName() string { + return "pkg/tcpip.ErrConnectionAborted" +} + +func (e *ErrConnectionAborted) StateFields() []string { + return []string{} +} + +func (e *ErrConnectionAborted) beforeSave() {} + +// +checklocksignore +func (e *ErrConnectionAborted) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrConnectionAborted) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrConnectionAborted) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrConnectionRefused) StateTypeName() string { + return "pkg/tcpip.ErrConnectionRefused" +} + +func (e *ErrConnectionRefused) StateFields() []string { + return []string{} +} + +func (e *ErrConnectionRefused) beforeSave() {} + +// +checklocksignore +func (e *ErrConnectionRefused) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrConnectionRefused) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrConnectionRefused) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrConnectionReset) StateTypeName() string { + return "pkg/tcpip.ErrConnectionReset" +} + +func (e *ErrConnectionReset) StateFields() []string { + return []string{} +} + +func (e *ErrConnectionReset) beforeSave() {} + +// +checklocksignore +func (e *ErrConnectionReset) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrConnectionReset) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrConnectionReset) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrDestinationRequired) StateTypeName() string { + return "pkg/tcpip.ErrDestinationRequired" +} + +func (e *ErrDestinationRequired) StateFields() []string { + return []string{} +} + +func (e *ErrDestinationRequired) beforeSave() {} + +// +checklocksignore +func (e *ErrDestinationRequired) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrDestinationRequired) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrDestinationRequired) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrDuplicateAddress) StateTypeName() string { + return "pkg/tcpip.ErrDuplicateAddress" +} + +func (e *ErrDuplicateAddress) StateFields() []string { + return []string{} +} + +func (e *ErrDuplicateAddress) beforeSave() {} + +// +checklocksignore +func (e *ErrDuplicateAddress) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrDuplicateAddress) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrDuplicateAddress) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrDuplicateNICID) StateTypeName() string { + return "pkg/tcpip.ErrDuplicateNICID" +} + +func (e *ErrDuplicateNICID) StateFields() []string { + return []string{} +} + +func (e *ErrDuplicateNICID) beforeSave() {} + +// +checklocksignore +func (e *ErrDuplicateNICID) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrDuplicateNICID) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrDuplicateNICID) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrInvalidNICID) StateTypeName() string { + return "pkg/tcpip.ErrInvalidNICID" +} + +func (e *ErrInvalidNICID) StateFields() []string { + return []string{} +} + +func (e *ErrInvalidNICID) beforeSave() {} + +// +checklocksignore +func (e *ErrInvalidNICID) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrInvalidNICID) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrInvalidNICID) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrInvalidEndpointState) StateTypeName() string { + return "pkg/tcpip.ErrInvalidEndpointState" +} + +func (e *ErrInvalidEndpointState) StateFields() []string { + return []string{} +} + +func (e *ErrInvalidEndpointState) beforeSave() {} + +// +checklocksignore +func (e *ErrInvalidEndpointState) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrInvalidEndpointState) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrInvalidEndpointState) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrInvalidOptionValue) StateTypeName() string { + return "pkg/tcpip.ErrInvalidOptionValue" +} + +func (e *ErrInvalidOptionValue) StateFields() []string { + return []string{} +} + +func (e *ErrInvalidOptionValue) beforeSave() {} + +// +checklocksignore +func (e *ErrInvalidOptionValue) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrInvalidOptionValue) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrInvalidOptionValue) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrInvalidPortRange) StateTypeName() string { + return "pkg/tcpip.ErrInvalidPortRange" +} + +func (e *ErrInvalidPortRange) StateFields() []string { + return []string{} +} + +func (e *ErrInvalidPortRange) beforeSave() {} + +// +checklocksignore +func (e *ErrInvalidPortRange) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrInvalidPortRange) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrInvalidPortRange) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrMalformedHeader) StateTypeName() string { + return "pkg/tcpip.ErrMalformedHeader" +} + +func (e *ErrMalformedHeader) StateFields() []string { + return []string{} +} + +func (e *ErrMalformedHeader) beforeSave() {} + +// +checklocksignore +func (e *ErrMalformedHeader) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrMalformedHeader) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrMalformedHeader) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrMessageTooLong) StateTypeName() string { + return "pkg/tcpip.ErrMessageTooLong" +} + +func (e *ErrMessageTooLong) StateFields() []string { + return []string{} +} + +func (e *ErrMessageTooLong) beforeSave() {} + +// +checklocksignore +func (e *ErrMessageTooLong) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrMessageTooLong) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrMessageTooLong) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNetworkUnreachable) StateTypeName() string { + return "pkg/tcpip.ErrNetworkUnreachable" +} + +func (e *ErrNetworkUnreachable) StateFields() []string { + return []string{} +} + +func (e *ErrNetworkUnreachable) beforeSave() {} + +// +checklocksignore +func (e *ErrNetworkUnreachable) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNetworkUnreachable) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNetworkUnreachable) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNoBufferSpace) StateTypeName() string { + return "pkg/tcpip.ErrNoBufferSpace" +} + +func (e *ErrNoBufferSpace) StateFields() []string { + return []string{} +} + +func (e *ErrNoBufferSpace) beforeSave() {} + +// +checklocksignore +func (e *ErrNoBufferSpace) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNoBufferSpace) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNoBufferSpace) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNoPortAvailable) StateTypeName() string { + return "pkg/tcpip.ErrNoPortAvailable" +} + +func (e *ErrNoPortAvailable) StateFields() []string { + return []string{} +} + +func (e *ErrNoPortAvailable) beforeSave() {} + +// +checklocksignore +func (e *ErrNoPortAvailable) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNoPortAvailable) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNoPortAvailable) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrHostUnreachable) StateTypeName() string { + return "pkg/tcpip.ErrHostUnreachable" +} + +func (e *ErrHostUnreachable) StateFields() []string { + return []string{} +} + +func (e *ErrHostUnreachable) beforeSave() {} + +// +checklocksignore +func (e *ErrHostUnreachable) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrHostUnreachable) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrHostUnreachable) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrHostDown) StateTypeName() string { + return "pkg/tcpip.ErrHostDown" +} + +func (e *ErrHostDown) StateFields() []string { + return []string{} +} + +func (e *ErrHostDown) beforeSave() {} + +// +checklocksignore +func (e *ErrHostDown) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrHostDown) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrHostDown) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNoNet) StateTypeName() string { + return "pkg/tcpip.ErrNoNet" +} + +func (e *ErrNoNet) StateFields() []string { + return []string{} +} + +func (e *ErrNoNet) beforeSave() {} + +// +checklocksignore +func (e *ErrNoNet) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNoNet) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNoNet) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNoSuchFile) StateTypeName() string { + return "pkg/tcpip.ErrNoSuchFile" +} + +func (e *ErrNoSuchFile) StateFields() []string { + return []string{} +} + +func (e *ErrNoSuchFile) beforeSave() {} + +// +checklocksignore +func (e *ErrNoSuchFile) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNoSuchFile) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNoSuchFile) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNotConnected) StateTypeName() string { + return "pkg/tcpip.ErrNotConnected" +} + +func (e *ErrNotConnected) StateFields() []string { + return []string{} +} + +func (e *ErrNotConnected) beforeSave() {} + +// +checklocksignore +func (e *ErrNotConnected) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNotConnected) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNotConnected) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNotPermitted) StateTypeName() string { + return "pkg/tcpip.ErrNotPermitted" +} + +func (e *ErrNotPermitted) StateFields() []string { + return []string{} +} + +func (e *ErrNotPermitted) beforeSave() {} + +// +checklocksignore +func (e *ErrNotPermitted) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNotPermitted) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNotPermitted) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrNotSupported) StateTypeName() string { + return "pkg/tcpip.ErrNotSupported" +} + +func (e *ErrNotSupported) StateFields() []string { + return []string{} +} + +func (e *ErrNotSupported) beforeSave() {} + +// +checklocksignore +func (e *ErrNotSupported) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrNotSupported) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrNotSupported) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrPortInUse) StateTypeName() string { + return "pkg/tcpip.ErrPortInUse" +} + +func (e *ErrPortInUse) StateFields() []string { + return []string{} +} + +func (e *ErrPortInUse) beforeSave() {} + +// +checklocksignore +func (e *ErrPortInUse) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrPortInUse) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrPortInUse) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrQueueSizeNotSupported) StateTypeName() string { + return "pkg/tcpip.ErrQueueSizeNotSupported" +} + +func (e *ErrQueueSizeNotSupported) StateFields() []string { + return []string{} +} + +func (e *ErrQueueSizeNotSupported) beforeSave() {} + +// +checklocksignore +func (e *ErrQueueSizeNotSupported) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrQueueSizeNotSupported) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrQueueSizeNotSupported) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrTimeout) StateTypeName() string { + return "pkg/tcpip.ErrTimeout" +} + +func (e *ErrTimeout) StateFields() []string { + return []string{} +} + +func (e *ErrTimeout) beforeSave() {} + +// +checklocksignore +func (e *ErrTimeout) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrTimeout) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrTimeout) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrUnknownDevice) StateTypeName() string { + return "pkg/tcpip.ErrUnknownDevice" +} + +func (e *ErrUnknownDevice) StateFields() []string { + return []string{} +} + +func (e *ErrUnknownDevice) beforeSave() {} + +// +checklocksignore +func (e *ErrUnknownDevice) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrUnknownDevice) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrUnknownDevice) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrUnknownNICID) StateTypeName() string { + return "pkg/tcpip.ErrUnknownNICID" +} + +func (e *ErrUnknownNICID) StateFields() []string { + return []string{} +} + +func (e *ErrUnknownNICID) beforeSave() {} + +// +checklocksignore +func (e *ErrUnknownNICID) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrUnknownNICID) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrUnknownNICID) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrUnknownProtocol) StateTypeName() string { + return "pkg/tcpip.ErrUnknownProtocol" +} + +func (e *ErrUnknownProtocol) StateFields() []string { + return []string{} +} + +func (e *ErrUnknownProtocol) beforeSave() {} + +// +checklocksignore +func (e *ErrUnknownProtocol) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrUnknownProtocol) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrUnknownProtocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrUnknownProtocolOption) StateTypeName() string { + return "pkg/tcpip.ErrUnknownProtocolOption" +} + +func (e *ErrUnknownProtocolOption) StateFields() []string { + return []string{} +} + +func (e *ErrUnknownProtocolOption) beforeSave() {} + +// +checklocksignore +func (e *ErrUnknownProtocolOption) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrUnknownProtocolOption) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrUnknownProtocolOption) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrWouldBlock) StateTypeName() string { + return "pkg/tcpip.ErrWouldBlock" +} + +func (e *ErrWouldBlock) StateFields() []string { + return []string{} +} + +func (e *ErrWouldBlock) beforeSave() {} + +// +checklocksignore +func (e *ErrWouldBlock) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrWouldBlock) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrWouldBlock) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrMissingRequiredFields) StateTypeName() string { + return "pkg/tcpip.ErrMissingRequiredFields" +} + +func (e *ErrMissingRequiredFields) StateFields() []string { + return []string{} +} + +func (e *ErrMissingRequiredFields) beforeSave() {} + +// +checklocksignore +func (e *ErrMissingRequiredFields) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrMissingRequiredFields) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrMissingRequiredFields) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrMulticastInputCannotBeOutput) StateTypeName() string { + return "pkg/tcpip.ErrMulticastInputCannotBeOutput" +} + +func (e *ErrMulticastInputCannotBeOutput) StateFields() []string { + return []string{} +} + +func (e *ErrMulticastInputCannotBeOutput) beforeSave() {} + +// +checklocksignore +func (e *ErrMulticastInputCannotBeOutput) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrMulticastInputCannotBeOutput) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrMulticastInputCannotBeOutput) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (e *ErrEndpointBusy) StateTypeName() string { + return "pkg/tcpip.ErrEndpointBusy" +} + +func (e *ErrEndpointBusy) StateFields() []string { + return []string{} +} + +func (e *ErrEndpointBusy) beforeSave() {} + +// +checklocksignore +func (e *ErrEndpointBusy) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *ErrEndpointBusy) afterLoad(context.Context) {} + +// +checklocksignore +func (e *ErrEndpointBusy) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (l *RouteList) StateTypeName() string { + return "pkg/tcpip.RouteList" +} + +func (l *RouteList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *RouteList) beforeSave() {} + +// +checklocksignore +func (l *RouteList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *RouteList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *RouteList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *RouteEntry) StateTypeName() string { + return "pkg/tcpip.RouteEntry" +} + +func (e *RouteEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *RouteEntry) beforeSave() {} + +// +checklocksignore +func (e *RouteEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *RouteEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *RouteEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func (l *sockErrorList) StateTypeName() string { + return "pkg/tcpip.sockErrorList" +} + +func (l *sockErrorList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *sockErrorList) beforeSave() {} + +// +checklocksignore +func (l *sockErrorList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *sockErrorList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *sockErrorList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *sockErrorEntry) StateTypeName() string { + return "pkg/tcpip.sockErrorEntry" +} + +func (e *sockErrorEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *sockErrorEntry) beforeSave() {} + +// +checklocksignore +func (e *sockErrorEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *sockErrorEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *sockErrorEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func (so *SocketOptions) StateTypeName() string { + return "pkg/tcpip.SocketOptions" +} + +func (so *SocketOptions) StateFields() []string { + return []string{ + "handler", + "broadcastEnabled", + "passCredEnabled", + "noChecksumEnabled", + "reuseAddressEnabled", + "reusePortEnabled", + "keepAliveEnabled", + "multicastLoopEnabled", + "receiveTOSEnabled", + "receiveTTLEnabled", + "receiveHopLimitEnabled", + "receiveTClassEnabled", + "receivePacketInfoEnabled", + "receiveIPv6PacketInfoEnabled", + "hdrIncludedEnabled", + "v6OnlyEnabled", + "quickAckEnabled", + "delayOptionEnabled", + "corkOptionEnabled", + "receiveOriginalDstAddress", + "ipv4RecvErrEnabled", + "ipv6RecvErrEnabled", + "errQueue", + "bindToDevice", + "sendBufferSize", + "receiveBufferSize", + "linger", + "rcvlowat", + "experimentOptionValue", + } +} + +func (so *SocketOptions) beforeSave() {} + +// +checklocksignore +func (so *SocketOptions) StateSave(stateSinkObject state.Sink) { + so.beforeSave() + stateSinkObject.Save(0, &so.handler) + stateSinkObject.Save(1, &so.broadcastEnabled) + stateSinkObject.Save(2, &so.passCredEnabled) + stateSinkObject.Save(3, &so.noChecksumEnabled) + stateSinkObject.Save(4, &so.reuseAddressEnabled) + stateSinkObject.Save(5, &so.reusePortEnabled) + stateSinkObject.Save(6, &so.keepAliveEnabled) + stateSinkObject.Save(7, &so.multicastLoopEnabled) + stateSinkObject.Save(8, &so.receiveTOSEnabled) + stateSinkObject.Save(9, &so.receiveTTLEnabled) + stateSinkObject.Save(10, &so.receiveHopLimitEnabled) + stateSinkObject.Save(11, &so.receiveTClassEnabled) + stateSinkObject.Save(12, &so.receivePacketInfoEnabled) + stateSinkObject.Save(13, &so.receiveIPv6PacketInfoEnabled) + stateSinkObject.Save(14, &so.hdrIncludedEnabled) + stateSinkObject.Save(15, &so.v6OnlyEnabled) + stateSinkObject.Save(16, &so.quickAckEnabled) + stateSinkObject.Save(17, &so.delayOptionEnabled) + stateSinkObject.Save(18, &so.corkOptionEnabled) + stateSinkObject.Save(19, &so.receiveOriginalDstAddress) + stateSinkObject.Save(20, &so.ipv4RecvErrEnabled) + stateSinkObject.Save(21, &so.ipv6RecvErrEnabled) + stateSinkObject.Save(22, &so.errQueue) + stateSinkObject.Save(23, &so.bindToDevice) + stateSinkObject.Save(24, &so.sendBufferSize) + stateSinkObject.Save(25, &so.receiveBufferSize) + stateSinkObject.Save(26, &so.linger) + stateSinkObject.Save(27, &so.rcvlowat) + stateSinkObject.Save(28, &so.experimentOptionValue) +} + +func (so *SocketOptions) afterLoad(context.Context) {} + +// +checklocksignore +func (so *SocketOptions) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &so.handler) + stateSourceObject.Load(1, &so.broadcastEnabled) + stateSourceObject.Load(2, &so.passCredEnabled) + stateSourceObject.Load(3, &so.noChecksumEnabled) + stateSourceObject.Load(4, &so.reuseAddressEnabled) + stateSourceObject.Load(5, &so.reusePortEnabled) + stateSourceObject.Load(6, &so.keepAliveEnabled) + stateSourceObject.Load(7, &so.multicastLoopEnabled) + stateSourceObject.Load(8, &so.receiveTOSEnabled) + stateSourceObject.Load(9, &so.receiveTTLEnabled) + stateSourceObject.Load(10, &so.receiveHopLimitEnabled) + stateSourceObject.Load(11, &so.receiveTClassEnabled) + stateSourceObject.Load(12, &so.receivePacketInfoEnabled) + stateSourceObject.Load(13, &so.receiveIPv6PacketInfoEnabled) + stateSourceObject.Load(14, &so.hdrIncludedEnabled) + stateSourceObject.Load(15, &so.v6OnlyEnabled) + stateSourceObject.Load(16, &so.quickAckEnabled) + stateSourceObject.Load(17, &so.delayOptionEnabled) + stateSourceObject.Load(18, &so.corkOptionEnabled) + stateSourceObject.Load(19, &so.receiveOriginalDstAddress) + stateSourceObject.Load(20, &so.ipv4RecvErrEnabled) + stateSourceObject.Load(21, &so.ipv6RecvErrEnabled) + stateSourceObject.Load(22, &so.errQueue) + stateSourceObject.Load(23, &so.bindToDevice) + stateSourceObject.Load(24, &so.sendBufferSize) + stateSourceObject.Load(25, &so.receiveBufferSize) + stateSourceObject.Load(26, &so.linger) + stateSourceObject.Load(27, &so.rcvlowat) + stateSourceObject.Load(28, &so.experimentOptionValue) +} + +func (l *LocalSockError) StateTypeName() string { + return "pkg/tcpip.LocalSockError" +} + +func (l *LocalSockError) StateFields() []string { + return []string{ + "info", + } +} + +func (l *LocalSockError) beforeSave() {} + +// +checklocksignore +func (l *LocalSockError) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.info) +} + +func (l *LocalSockError) afterLoad(context.Context) {} + +// +checklocksignore +func (l *LocalSockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.info) +} + +func (s *SockError) StateTypeName() string { + return "pkg/tcpip.SockError" +} + +func (s *SockError) StateFields() []string { + return []string{ + "sockErrorEntry", + "Err", + "Cause", + "Payload", + "Dst", + "Offender", + "NetProto", + } +} + +func (s *SockError) beforeSave() {} + +// +checklocksignore +func (s *SockError) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.sockErrorEntry) + stateSinkObject.Save(1, &s.Err) + stateSinkObject.Save(2, &s.Cause) + stateSinkObject.Save(3, &s.Payload) + stateSinkObject.Save(4, &s.Dst) + stateSinkObject.Save(5, &s.Offender) + stateSinkObject.Save(6, &s.NetProto) +} + +func (s *SockError) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SockError) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.sockErrorEntry) + stateSourceObject.Load(1, &s.Err) + stateSourceObject.Load(2, &s.Cause) + stateSourceObject.Load(3, &s.Payload) + stateSourceObject.Load(4, &s.Dst) + stateSourceObject.Load(5, &s.Offender) + stateSourceObject.Load(6, &s.NetProto) +} + +func (s *stdClock) StateTypeName() string { + return "pkg/tcpip.stdClock" +} + +func (s *stdClock) StateFields() []string { + return []string{ + "monotonicOffset", + } +} + +// +checklocksignore +func (s *stdClock) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.monotonicOffset) +} + +// +checklocksignore +func (s *stdClock) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.monotonicOffset) + stateSourceObject.AfterLoad(func() { s.afterLoad(ctx) }) +} + +func (st *stdTimer) StateTypeName() string { + return "pkg/tcpip.stdTimer" +} + +func (st *stdTimer) StateFields() []string { + return []string{ + "t", + } +} + +func (st *stdTimer) beforeSave() {} + +// +checklocksignore +func (st *stdTimer) StateSave(stateSinkObject state.Sink) { + st.beforeSave() + stateSinkObject.Save(0, &st.t) +} + +func (st *stdTimer) afterLoad(context.Context) {} + +// +checklocksignore +func (st *stdTimer) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &st.t) +} + +func (mt *MonotonicTime) StateTypeName() string { + return "pkg/tcpip.MonotonicTime" +} + +func (mt *MonotonicTime) StateFields() []string { + return []string{ + "nanoseconds", + } +} + +func (mt *MonotonicTime) beforeSave() {} + +// +checklocksignore +func (mt *MonotonicTime) StateSave(stateSinkObject state.Sink) { + mt.beforeSave() + stateSinkObject.Save(0, &mt.nanoseconds) +} + +func (mt *MonotonicTime) afterLoad(context.Context) {} + +// +checklocksignore +func (mt *MonotonicTime) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &mt.nanoseconds) +} + +func (a *Address) StateTypeName() string { + return "pkg/tcpip.Address" +} + +func (a *Address) StateFields() []string { + return []string{ + "addr", + "length", + } +} + +func (a *Address) beforeSave() {} + +// +checklocksignore +func (a *Address) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.addr) + stateSinkObject.Save(1, &a.length) +} + +func (a *Address) afterLoad(context.Context) {} + +// +checklocksignore +func (a *Address) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.addr) + stateSourceObject.Load(1, &a.length) +} + +func (m *AddressMask) StateTypeName() string { + return "pkg/tcpip.AddressMask" +} + +func (m *AddressMask) StateFields() []string { + return []string{ + "mask", + "length", + } +} + +func (m *AddressMask) beforeSave() {} + +// +checklocksignore +func (m *AddressMask) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.mask) + stateSinkObject.Save(1, &m.length) +} + +func (m *AddressMask) afterLoad(context.Context) {} + +// +checklocksignore +func (m *AddressMask) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.mask) + stateSourceObject.Load(1, &m.length) +} + +func (s *Subnet) StateTypeName() string { + return "pkg/tcpip.Subnet" +} + +func (s *Subnet) StateFields() []string { + return []string{ + "address", + "mask", + } +} + +func (s *Subnet) beforeSave() {} + +// +checklocksignore +func (s *Subnet) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.address) + stateSinkObject.Save(1, &s.mask) +} + +func (s *Subnet) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Subnet) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.address) + stateSourceObject.Load(1, &s.mask) +} + +func (f *FullAddress) StateTypeName() string { + return "pkg/tcpip.FullAddress" +} + +func (f *FullAddress) StateFields() []string { + return []string{ + "NIC", + "Addr", + "Port", + "LinkAddr", + } +} + +func (f *FullAddress) beforeSave() {} + +// +checklocksignore +func (f *FullAddress) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.NIC) + stateSinkObject.Save(1, &f.Addr) + stateSinkObject.Save(2, &f.Port) + stateSinkObject.Save(3, &f.LinkAddr) +} + +func (f *FullAddress) afterLoad(context.Context) {} + +// +checklocksignore +func (f *FullAddress) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.NIC) + stateSourceObject.Load(1, &f.Addr) + stateSourceObject.Load(2, &f.Port) + stateSourceObject.Load(3, &f.LinkAddr) +} + +func (s *SendableControlMessages) StateTypeName() string { + return "pkg/tcpip.SendableControlMessages" +} + +func (s *SendableControlMessages) StateFields() []string { + return []string{ + "HasTTL", + "TTL", + "HasHopLimit", + "HopLimit", + "HasIPv6PacketInfo", + "IPv6PacketInfo", + } +} + +func (s *SendableControlMessages) beforeSave() {} + +// +checklocksignore +func (s *SendableControlMessages) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.HasTTL) + stateSinkObject.Save(1, &s.TTL) + stateSinkObject.Save(2, &s.HasHopLimit) + stateSinkObject.Save(3, &s.HopLimit) + stateSinkObject.Save(4, &s.HasIPv6PacketInfo) + stateSinkObject.Save(5, &s.IPv6PacketInfo) +} + +func (s *SendableControlMessages) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SendableControlMessages) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.HasTTL) + stateSourceObject.Load(1, &s.TTL) + stateSourceObject.Load(2, &s.HasHopLimit) + stateSourceObject.Load(3, &s.HopLimit) + stateSourceObject.Load(4, &s.HasIPv6PacketInfo) + stateSourceObject.Load(5, &s.IPv6PacketInfo) +} + +func (c *ReceivableControlMessages) StateTypeName() string { + return "pkg/tcpip.ReceivableControlMessages" +} + +func (c *ReceivableControlMessages) StateFields() []string { + return []string{ + "Timestamp", + "HasInq", + "Inq", + "HasTOS", + "TOS", + "HasTTL", + "TTL", + "HasHopLimit", + "HopLimit", + "HasTimestamp", + "HasTClass", + "TClass", + "HasIPPacketInfo", + "PacketInfo", + "HasIPv6PacketInfo", + "IPv6PacketInfo", + "HasOriginalDstAddress", + "OriginalDstAddress", + "SockErr", + } +} + +func (c *ReceivableControlMessages) beforeSave() {} + +// +checklocksignore +func (c *ReceivableControlMessages) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + var TimestampValue int64 + TimestampValue = c.saveTimestamp() + stateSinkObject.SaveValue(0, TimestampValue) + stateSinkObject.Save(1, &c.HasInq) + stateSinkObject.Save(2, &c.Inq) + stateSinkObject.Save(3, &c.HasTOS) + stateSinkObject.Save(4, &c.TOS) + stateSinkObject.Save(5, &c.HasTTL) + stateSinkObject.Save(6, &c.TTL) + stateSinkObject.Save(7, &c.HasHopLimit) + stateSinkObject.Save(8, &c.HopLimit) + stateSinkObject.Save(9, &c.HasTimestamp) + stateSinkObject.Save(10, &c.HasTClass) + stateSinkObject.Save(11, &c.TClass) + stateSinkObject.Save(12, &c.HasIPPacketInfo) + stateSinkObject.Save(13, &c.PacketInfo) + stateSinkObject.Save(14, &c.HasIPv6PacketInfo) + stateSinkObject.Save(15, &c.IPv6PacketInfo) + stateSinkObject.Save(16, &c.HasOriginalDstAddress) + stateSinkObject.Save(17, &c.OriginalDstAddress) + stateSinkObject.Save(18, &c.SockErr) +} + +func (c *ReceivableControlMessages) afterLoad(context.Context) {} + +// +checklocksignore +func (c *ReceivableControlMessages) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(1, &c.HasInq) + stateSourceObject.Load(2, &c.Inq) + stateSourceObject.Load(3, &c.HasTOS) + stateSourceObject.Load(4, &c.TOS) + stateSourceObject.Load(5, &c.HasTTL) + stateSourceObject.Load(6, &c.TTL) + stateSourceObject.Load(7, &c.HasHopLimit) + stateSourceObject.Load(8, &c.HopLimit) + stateSourceObject.Load(9, &c.HasTimestamp) + stateSourceObject.Load(10, &c.HasTClass) + stateSourceObject.Load(11, &c.TClass) + stateSourceObject.Load(12, &c.HasIPPacketInfo) + stateSourceObject.Load(13, &c.PacketInfo) + stateSourceObject.Load(14, &c.HasIPv6PacketInfo) + stateSourceObject.Load(15, &c.IPv6PacketInfo) + stateSourceObject.Load(16, &c.HasOriginalDstAddress) + stateSourceObject.Load(17, &c.OriginalDstAddress) + stateSourceObject.Load(18, &c.SockErr) + stateSourceObject.LoadValue(0, new(int64), func(y any) { c.loadTimestamp(ctx, y.(int64)) }) +} + +func (l *LinkPacketInfo) StateTypeName() string { + return "pkg/tcpip.LinkPacketInfo" +} + +func (l *LinkPacketInfo) StateFields() []string { + return []string{ + "Protocol", + "PktType", + } +} + +func (l *LinkPacketInfo) beforeSave() {} + +// +checklocksignore +func (l *LinkPacketInfo) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.Protocol) + stateSinkObject.Save(1, &l.PktType) +} + +func (l *LinkPacketInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (l *LinkPacketInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.Protocol) + stateSourceObject.Load(1, &l.PktType) +} + +func (t *TCPSendBufferSizeRangeOption) StateTypeName() string { + return "pkg/tcpip.TCPSendBufferSizeRangeOption" +} + +func (t *TCPSendBufferSizeRangeOption) StateFields() []string { + return []string{ + "Min", + "Default", + "Max", + } +} + +func (t *TCPSendBufferSizeRangeOption) beforeSave() {} + +// +checklocksignore +func (t *TCPSendBufferSizeRangeOption) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.Min) + stateSinkObject.Save(1, &t.Default) + stateSinkObject.Save(2, &t.Max) +} + +func (t *TCPSendBufferSizeRangeOption) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPSendBufferSizeRangeOption) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.Min) + stateSourceObject.Load(1, &t.Default) + stateSourceObject.Load(2, &t.Max) +} + +func (t *TCPReceiveBufferSizeRangeOption) StateTypeName() string { + return "pkg/tcpip.TCPReceiveBufferSizeRangeOption" +} + +func (t *TCPReceiveBufferSizeRangeOption) StateFields() []string { + return []string{ + "Min", + "Default", + "Max", + } +} + +func (t *TCPReceiveBufferSizeRangeOption) beforeSave() {} + +// +checklocksignore +func (t *TCPReceiveBufferSizeRangeOption) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.Min) + stateSinkObject.Save(1, &t.Default) + stateSinkObject.Save(2, &t.Max) +} + +func (t *TCPReceiveBufferSizeRangeOption) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPReceiveBufferSizeRangeOption) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.Min) + stateSourceObject.Load(1, &t.Default) + stateSourceObject.Load(2, &t.Max) +} + +func (f *ICMPv6Filter) StateTypeName() string { + return "pkg/tcpip.ICMPv6Filter" +} + +func (f *ICMPv6Filter) StateFields() []string { + return []string{ + "DenyType", + } +} + +func (f *ICMPv6Filter) beforeSave() {} + +// +checklocksignore +func (f *ICMPv6Filter) StateSave(stateSinkObject state.Sink) { + f.beforeSave() + stateSinkObject.Save(0, &f.DenyType) +} + +func (f *ICMPv6Filter) afterLoad(context.Context) {} + +// +checklocksignore +func (f *ICMPv6Filter) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &f.DenyType) +} + +func (t *TpacketReq) StateTypeName() string { + return "pkg/tcpip.TpacketReq" +} + +func (t *TpacketReq) StateFields() []string { + return []string{ + "TpBlockSize", + "TpBlockNr", + "TpFrameSize", + "TpFrameNr", + } +} + +func (t *TpacketReq) beforeSave() {} + +// +checklocksignore +func (t *TpacketReq) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.TpBlockSize) + stateSinkObject.Save(1, &t.TpBlockNr) + stateSinkObject.Save(2, &t.TpFrameSize) + stateSinkObject.Save(3, &t.TpFrameNr) +} + +func (t *TpacketReq) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TpacketReq) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.TpBlockSize) + stateSourceObject.Load(1, &t.TpBlockNr) + stateSourceObject.Load(2, &t.TpFrameSize) + stateSourceObject.Load(3, &t.TpFrameNr) +} + +func (t *TpacketStats) StateTypeName() string { + return "pkg/tcpip.TpacketStats" +} + +func (t *TpacketStats) StateFields() []string { + return []string{ + "Packets", + "Dropped", + } +} + +func (t *TpacketStats) beforeSave() {} + +// +checklocksignore +func (t *TpacketStats) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.Packets) + stateSinkObject.Save(1, &t.Dropped) +} + +func (t *TpacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TpacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.Packets) + stateSourceObject.Load(1, &t.Dropped) +} + +func (l *LingerOption) StateTypeName() string { + return "pkg/tcpip.LingerOption" +} + +func (l *LingerOption) StateFields() []string { + return []string{ + "Enabled", + "Timeout", + } +} + +func (l *LingerOption) beforeSave() {} + +// +checklocksignore +func (l *LingerOption) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.Enabled) + stateSinkObject.Save(1, &l.Timeout) +} + +func (l *LingerOption) afterLoad(context.Context) {} + +// +checklocksignore +func (l *LingerOption) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.Enabled) + stateSourceObject.Load(1, &l.Timeout) +} + +func (i *IPPacketInfo) StateTypeName() string { + return "pkg/tcpip.IPPacketInfo" +} + +func (i *IPPacketInfo) StateFields() []string { + return []string{ + "NIC", + "LocalAddr", + "DestinationAddr", + } +} + +func (i *IPPacketInfo) beforeSave() {} + +// +checklocksignore +func (i *IPPacketInfo) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.NIC) + stateSinkObject.Save(1, &i.LocalAddr) + stateSinkObject.Save(2, &i.DestinationAddr) +} + +func (i *IPPacketInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IPPacketInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.NIC) + stateSourceObject.Load(1, &i.LocalAddr) + stateSourceObject.Load(2, &i.DestinationAddr) +} + +func (i *IPv6PacketInfo) StateTypeName() string { + return "pkg/tcpip.IPv6PacketInfo" +} + +func (i *IPv6PacketInfo) StateFields() []string { + return []string{ + "Addr", + "NIC", + } +} + +func (i *IPv6PacketInfo) beforeSave() {} + +// +checklocksignore +func (i *IPv6PacketInfo) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Addr) + stateSinkObject.Save(1, &i.NIC) +} + +func (i *IPv6PacketInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IPv6PacketInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Addr) + stateSourceObject.Load(1, &i.NIC) +} + +func (s *SendBufferSizeOption) StateTypeName() string { + return "pkg/tcpip.SendBufferSizeOption" +} + +func (s *SendBufferSizeOption) StateFields() []string { + return []string{ + "Min", + "Default", + "Max", + } +} + +func (s *SendBufferSizeOption) beforeSave() {} + +// +checklocksignore +func (s *SendBufferSizeOption) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.Min) + stateSinkObject.Save(1, &s.Default) + stateSinkObject.Save(2, &s.Max) +} + +func (s *SendBufferSizeOption) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SendBufferSizeOption) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.Min) + stateSourceObject.Load(1, &s.Default) + stateSourceObject.Load(2, &s.Max) +} + +func (r *ReceiveBufferSizeOption) StateTypeName() string { + return "pkg/tcpip.ReceiveBufferSizeOption" +} + +func (r *ReceiveBufferSizeOption) StateFields() []string { + return []string{ + "Min", + "Default", + "Max", + } +} + +func (r *ReceiveBufferSizeOption) beforeSave() {} + +// +checklocksignore +func (r *ReceiveBufferSizeOption) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.Min) + stateSinkObject.Save(1, &r.Default) + stateSinkObject.Save(2, &r.Max) +} + +func (r *ReceiveBufferSizeOption) afterLoad(context.Context) {} + +// +checklocksignore +func (r *ReceiveBufferSizeOption) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.Min) + stateSourceObject.Load(1, &r.Default) + stateSourceObject.Load(2, &r.Max) +} + +func (r *Route) StateTypeName() string { + return "pkg/tcpip.Route" +} + +func (r *Route) StateFields() []string { + return []string{ + "RouteEntry", + "Destination", + "Gateway", + "NIC", + "SourceHint", + "MTU", + } +} + +func (r *Route) beforeSave() {} + +// +checklocksignore +func (r *Route) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.RouteEntry) + stateSinkObject.Save(1, &r.Destination) + stateSinkObject.Save(2, &r.Gateway) + stateSinkObject.Save(3, &r.NIC) + stateSinkObject.Save(4, &r.SourceHint) + stateSinkObject.Save(5, &r.MTU) +} + +func (r *Route) afterLoad(context.Context) {} + +// +checklocksignore +func (r *Route) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.RouteEntry) + stateSourceObject.Load(1, &r.Destination) + stateSourceObject.Load(2, &r.Gateway) + stateSourceObject.Load(3, &r.NIC) + stateSourceObject.Load(4, &r.SourceHint) + stateSourceObject.Load(5, &r.MTU) +} + +func (s *StatCounter) StateTypeName() string { + return "pkg/tcpip.StatCounter" +} + +func (s *StatCounter) StateFields() []string { + return []string{ + "count", + } +} + +func (s *StatCounter) beforeSave() {} + +// +checklocksignore +func (s *StatCounter) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.count) +} + +func (s *StatCounter) afterLoad(context.Context) {} + +// +checklocksignore +func (s *StatCounter) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.count) +} + +func (m *MultiCounterStat) StateTypeName() string { + return "pkg/tcpip.MultiCounterStat" +} + +func (m *MultiCounterStat) StateFields() []string { + return []string{ + "a", + "b", + } +} + +func (m *MultiCounterStat) beforeSave() {} + +// +checklocksignore +func (m *MultiCounterStat) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.a) + stateSinkObject.Save(1, &m.b) +} + +func (m *MultiCounterStat) afterLoad(context.Context) {} + +// +checklocksignore +func (m *MultiCounterStat) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.a) + stateSourceObject.Load(1, &m.b) +} + +func (i *ICMPv4PacketStats) StateTypeName() string { + return "pkg/tcpip.ICMPv4PacketStats" +} + +func (i *ICMPv4PacketStats) StateFields() []string { + return []string{ + "EchoRequest", + "EchoReply", + "DstUnreachable", + "SrcQuench", + "Redirect", + "TimeExceeded", + "ParamProblem", + "Timestamp", + "TimestampReply", + "InfoRequest", + "InfoReply", + } +} + +func (i *ICMPv4PacketStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv4PacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.EchoRequest) + stateSinkObject.Save(1, &i.EchoReply) + stateSinkObject.Save(2, &i.DstUnreachable) + stateSinkObject.Save(3, &i.SrcQuench) + stateSinkObject.Save(4, &i.Redirect) + stateSinkObject.Save(5, &i.TimeExceeded) + stateSinkObject.Save(6, &i.ParamProblem) + stateSinkObject.Save(7, &i.Timestamp) + stateSinkObject.Save(8, &i.TimestampReply) + stateSinkObject.Save(9, &i.InfoRequest) + stateSinkObject.Save(10, &i.InfoReply) +} + +func (i *ICMPv4PacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv4PacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.EchoRequest) + stateSourceObject.Load(1, &i.EchoReply) + stateSourceObject.Load(2, &i.DstUnreachable) + stateSourceObject.Load(3, &i.SrcQuench) + stateSourceObject.Load(4, &i.Redirect) + stateSourceObject.Load(5, &i.TimeExceeded) + stateSourceObject.Load(6, &i.ParamProblem) + stateSourceObject.Load(7, &i.Timestamp) + stateSourceObject.Load(8, &i.TimestampReply) + stateSourceObject.Load(9, &i.InfoRequest) + stateSourceObject.Load(10, &i.InfoReply) +} + +func (i *ICMPv4SentPacketStats) StateTypeName() string { + return "pkg/tcpip.ICMPv4SentPacketStats" +} + +func (i *ICMPv4SentPacketStats) StateFields() []string { + return []string{ + "ICMPv4PacketStats", + "Dropped", + "RateLimited", + } +} + +func (i *ICMPv4SentPacketStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv4SentPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.ICMPv4PacketStats) + stateSinkObject.Save(1, &i.Dropped) + stateSinkObject.Save(2, &i.RateLimited) +} + +func (i *ICMPv4SentPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv4SentPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.ICMPv4PacketStats) + stateSourceObject.Load(1, &i.Dropped) + stateSourceObject.Load(2, &i.RateLimited) +} + +func (i *ICMPv4ReceivedPacketStats) StateTypeName() string { + return "pkg/tcpip.ICMPv4ReceivedPacketStats" +} + +func (i *ICMPv4ReceivedPacketStats) StateFields() []string { + return []string{ + "ICMPv4PacketStats", + "Invalid", + } +} + +func (i *ICMPv4ReceivedPacketStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv4ReceivedPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.ICMPv4PacketStats) + stateSinkObject.Save(1, &i.Invalid) +} + +func (i *ICMPv4ReceivedPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv4ReceivedPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.ICMPv4PacketStats) + stateSourceObject.Load(1, &i.Invalid) +} + +func (i *ICMPv4Stats) StateTypeName() string { + return "pkg/tcpip.ICMPv4Stats" +} + +func (i *ICMPv4Stats) StateFields() []string { + return []string{ + "PacketsSent", + "PacketsReceived", + } +} + +func (i *ICMPv4Stats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv4Stats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.PacketsSent) + stateSinkObject.Save(1, &i.PacketsReceived) +} + +func (i *ICMPv4Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv4Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.PacketsSent) + stateSourceObject.Load(1, &i.PacketsReceived) +} + +func (i *ICMPv6PacketStats) StateTypeName() string { + return "pkg/tcpip.ICMPv6PacketStats" +} + +func (i *ICMPv6PacketStats) StateFields() []string { + return []string{ + "EchoRequest", + "EchoReply", + "DstUnreachable", + "PacketTooBig", + "TimeExceeded", + "ParamProblem", + "RouterSolicit", + "RouterAdvert", + "NeighborSolicit", + "NeighborAdvert", + "RedirectMsg", + "MulticastListenerQuery", + "MulticastListenerReport", + "MulticastListenerReportV2", + "MulticastListenerDone", + } +} + +func (i *ICMPv6PacketStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv6PacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.EchoRequest) + stateSinkObject.Save(1, &i.EchoReply) + stateSinkObject.Save(2, &i.DstUnreachable) + stateSinkObject.Save(3, &i.PacketTooBig) + stateSinkObject.Save(4, &i.TimeExceeded) + stateSinkObject.Save(5, &i.ParamProblem) + stateSinkObject.Save(6, &i.RouterSolicit) + stateSinkObject.Save(7, &i.RouterAdvert) + stateSinkObject.Save(8, &i.NeighborSolicit) + stateSinkObject.Save(9, &i.NeighborAdvert) + stateSinkObject.Save(10, &i.RedirectMsg) + stateSinkObject.Save(11, &i.MulticastListenerQuery) + stateSinkObject.Save(12, &i.MulticastListenerReport) + stateSinkObject.Save(13, &i.MulticastListenerReportV2) + stateSinkObject.Save(14, &i.MulticastListenerDone) +} + +func (i *ICMPv6PacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv6PacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.EchoRequest) + stateSourceObject.Load(1, &i.EchoReply) + stateSourceObject.Load(2, &i.DstUnreachable) + stateSourceObject.Load(3, &i.PacketTooBig) + stateSourceObject.Load(4, &i.TimeExceeded) + stateSourceObject.Load(5, &i.ParamProblem) + stateSourceObject.Load(6, &i.RouterSolicit) + stateSourceObject.Load(7, &i.RouterAdvert) + stateSourceObject.Load(8, &i.NeighborSolicit) + stateSourceObject.Load(9, &i.NeighborAdvert) + stateSourceObject.Load(10, &i.RedirectMsg) + stateSourceObject.Load(11, &i.MulticastListenerQuery) + stateSourceObject.Load(12, &i.MulticastListenerReport) + stateSourceObject.Load(13, &i.MulticastListenerReportV2) + stateSourceObject.Load(14, &i.MulticastListenerDone) +} + +func (i *ICMPv6SentPacketStats) StateTypeName() string { + return "pkg/tcpip.ICMPv6SentPacketStats" +} + +func (i *ICMPv6SentPacketStats) StateFields() []string { + return []string{ + "ICMPv6PacketStats", + "Dropped", + "RateLimited", + } +} + +func (i *ICMPv6SentPacketStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv6SentPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.ICMPv6PacketStats) + stateSinkObject.Save(1, &i.Dropped) + stateSinkObject.Save(2, &i.RateLimited) +} + +func (i *ICMPv6SentPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv6SentPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.ICMPv6PacketStats) + stateSourceObject.Load(1, &i.Dropped) + stateSourceObject.Load(2, &i.RateLimited) +} + +func (i *ICMPv6ReceivedPacketStats) StateTypeName() string { + return "pkg/tcpip.ICMPv6ReceivedPacketStats" +} + +func (i *ICMPv6ReceivedPacketStats) StateFields() []string { + return []string{ + "ICMPv6PacketStats", + "Unrecognized", + "Invalid", + "RouterOnlyPacketsDroppedByHost", + } +} + +func (i *ICMPv6ReceivedPacketStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv6ReceivedPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.ICMPv6PacketStats) + stateSinkObject.Save(1, &i.Unrecognized) + stateSinkObject.Save(2, &i.Invalid) + stateSinkObject.Save(3, &i.RouterOnlyPacketsDroppedByHost) +} + +func (i *ICMPv6ReceivedPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv6ReceivedPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.ICMPv6PacketStats) + stateSourceObject.Load(1, &i.Unrecognized) + stateSourceObject.Load(2, &i.Invalid) + stateSourceObject.Load(3, &i.RouterOnlyPacketsDroppedByHost) +} + +func (i *ICMPv6Stats) StateTypeName() string { + return "pkg/tcpip.ICMPv6Stats" +} + +func (i *ICMPv6Stats) StateFields() []string { + return []string{ + "PacketsSent", + "PacketsReceived", + } +} + +func (i *ICMPv6Stats) beforeSave() {} + +// +checklocksignore +func (i *ICMPv6Stats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.PacketsSent) + stateSinkObject.Save(1, &i.PacketsReceived) +} + +func (i *ICMPv6Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPv6Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.PacketsSent) + stateSourceObject.Load(1, &i.PacketsReceived) +} + +func (i *ICMPStats) StateTypeName() string { + return "pkg/tcpip.ICMPStats" +} + +func (i *ICMPStats) StateFields() []string { + return []string{ + "V4", + "V6", + } +} + +func (i *ICMPStats) beforeSave() {} + +// +checklocksignore +func (i *ICMPStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.V4) + stateSinkObject.Save(1, &i.V6) +} + +func (i *ICMPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *ICMPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.V4) + stateSourceObject.Load(1, &i.V6) +} + +func (i *IGMPPacketStats) StateTypeName() string { + return "pkg/tcpip.IGMPPacketStats" +} + +func (i *IGMPPacketStats) StateFields() []string { + return []string{ + "MembershipQuery", + "V1MembershipReport", + "V2MembershipReport", + "V3MembershipReport", + "LeaveGroup", + } +} + +func (i *IGMPPacketStats) beforeSave() {} + +// +checklocksignore +func (i *IGMPPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.MembershipQuery) + stateSinkObject.Save(1, &i.V1MembershipReport) + stateSinkObject.Save(2, &i.V2MembershipReport) + stateSinkObject.Save(3, &i.V3MembershipReport) + stateSinkObject.Save(4, &i.LeaveGroup) +} + +func (i *IGMPPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IGMPPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.MembershipQuery) + stateSourceObject.Load(1, &i.V1MembershipReport) + stateSourceObject.Load(2, &i.V2MembershipReport) + stateSourceObject.Load(3, &i.V3MembershipReport) + stateSourceObject.Load(4, &i.LeaveGroup) +} + +func (i *IGMPSentPacketStats) StateTypeName() string { + return "pkg/tcpip.IGMPSentPacketStats" +} + +func (i *IGMPSentPacketStats) StateFields() []string { + return []string{ + "IGMPPacketStats", + "Dropped", + } +} + +func (i *IGMPSentPacketStats) beforeSave() {} + +// +checklocksignore +func (i *IGMPSentPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.IGMPPacketStats) + stateSinkObject.Save(1, &i.Dropped) +} + +func (i *IGMPSentPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IGMPSentPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.IGMPPacketStats) + stateSourceObject.Load(1, &i.Dropped) +} + +func (i *IGMPReceivedPacketStats) StateTypeName() string { + return "pkg/tcpip.IGMPReceivedPacketStats" +} + +func (i *IGMPReceivedPacketStats) StateFields() []string { + return []string{ + "IGMPPacketStats", + "Invalid", + "ChecksumErrors", + "Unrecognized", + } +} + +func (i *IGMPReceivedPacketStats) beforeSave() {} + +// +checklocksignore +func (i *IGMPReceivedPacketStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.IGMPPacketStats) + stateSinkObject.Save(1, &i.Invalid) + stateSinkObject.Save(2, &i.ChecksumErrors) + stateSinkObject.Save(3, &i.Unrecognized) +} + +func (i *IGMPReceivedPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IGMPReceivedPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.IGMPPacketStats) + stateSourceObject.Load(1, &i.Invalid) + stateSourceObject.Load(2, &i.ChecksumErrors) + stateSourceObject.Load(3, &i.Unrecognized) +} + +func (i *IGMPStats) StateTypeName() string { + return "pkg/tcpip.IGMPStats" +} + +func (i *IGMPStats) StateFields() []string { + return []string{ + "PacketsSent", + "PacketsReceived", + } +} + +func (i *IGMPStats) beforeSave() {} + +// +checklocksignore +func (i *IGMPStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.PacketsSent) + stateSinkObject.Save(1, &i.PacketsReceived) +} + +func (i *IGMPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IGMPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.PacketsSent) + stateSourceObject.Load(1, &i.PacketsReceived) +} + +func (i *IPForwardingStats) StateTypeName() string { + return "pkg/tcpip.IPForwardingStats" +} + +func (i *IPForwardingStats) StateFields() []string { + return []string{ + "Unrouteable", + "ExhaustedTTL", + "InitializingSource", + "LinkLocalSource", + "LinkLocalDestination", + "PacketTooBig", + "HostUnreachable", + "ExtensionHeaderProblem", + "UnexpectedMulticastInputInterface", + "UnknownOutputEndpoint", + "NoMulticastPendingQueueBufferSpace", + "OutgoingDeviceNoBufferSpace", + "Errors", + "OutgoingDeviceClosedForSend", + } +} + +func (i *IPForwardingStats) beforeSave() {} + +// +checklocksignore +func (i *IPForwardingStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.Unrouteable) + stateSinkObject.Save(1, &i.ExhaustedTTL) + stateSinkObject.Save(2, &i.InitializingSource) + stateSinkObject.Save(3, &i.LinkLocalSource) + stateSinkObject.Save(4, &i.LinkLocalDestination) + stateSinkObject.Save(5, &i.PacketTooBig) + stateSinkObject.Save(6, &i.HostUnreachable) + stateSinkObject.Save(7, &i.ExtensionHeaderProblem) + stateSinkObject.Save(8, &i.UnexpectedMulticastInputInterface) + stateSinkObject.Save(9, &i.UnknownOutputEndpoint) + stateSinkObject.Save(10, &i.NoMulticastPendingQueueBufferSpace) + stateSinkObject.Save(11, &i.OutgoingDeviceNoBufferSpace) + stateSinkObject.Save(12, &i.Errors) + stateSinkObject.Save(13, &i.OutgoingDeviceClosedForSend) +} + +func (i *IPForwardingStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IPForwardingStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.Unrouteable) + stateSourceObject.Load(1, &i.ExhaustedTTL) + stateSourceObject.Load(2, &i.InitializingSource) + stateSourceObject.Load(3, &i.LinkLocalSource) + stateSourceObject.Load(4, &i.LinkLocalDestination) + stateSourceObject.Load(5, &i.PacketTooBig) + stateSourceObject.Load(6, &i.HostUnreachable) + stateSourceObject.Load(7, &i.ExtensionHeaderProblem) + stateSourceObject.Load(8, &i.UnexpectedMulticastInputInterface) + stateSourceObject.Load(9, &i.UnknownOutputEndpoint) + stateSourceObject.Load(10, &i.NoMulticastPendingQueueBufferSpace) + stateSourceObject.Load(11, &i.OutgoingDeviceNoBufferSpace) + stateSourceObject.Load(12, &i.Errors) + stateSourceObject.Load(13, &i.OutgoingDeviceClosedForSend) +} + +func (i *IPStats) StateTypeName() string { + return "pkg/tcpip.IPStats" +} + +func (i *IPStats) StateFields() []string { + return []string{ + "PacketsReceived", + "ValidPacketsReceived", + "DisabledPacketsReceived", + "InvalidDestinationAddressesReceived", + "InvalidSourceAddressesReceived", + "PacketsDelivered", + "PacketsSent", + "OutgoingPacketErrors", + "MalformedPacketsReceived", + "MalformedFragmentsReceived", + "IPTablesPreroutingDropped", + "IPTablesInputDropped", + "IPTablesForwardDropped", + "IPTablesOutputDropped", + "IPTablesPostroutingDropped", + "OptionTimestampReceived", + "OptionRecordRouteReceived", + "OptionRouterAlertReceived", + "OptionUnknownReceived", + "Forwarding", + } +} + +func (i *IPStats) beforeSave() {} + +// +checklocksignore +func (i *IPStats) StateSave(stateSinkObject state.Sink) { + i.beforeSave() + stateSinkObject.Save(0, &i.PacketsReceived) + stateSinkObject.Save(1, &i.ValidPacketsReceived) + stateSinkObject.Save(2, &i.DisabledPacketsReceived) + stateSinkObject.Save(3, &i.InvalidDestinationAddressesReceived) + stateSinkObject.Save(4, &i.InvalidSourceAddressesReceived) + stateSinkObject.Save(5, &i.PacketsDelivered) + stateSinkObject.Save(6, &i.PacketsSent) + stateSinkObject.Save(7, &i.OutgoingPacketErrors) + stateSinkObject.Save(8, &i.MalformedPacketsReceived) + stateSinkObject.Save(9, &i.MalformedFragmentsReceived) + stateSinkObject.Save(10, &i.IPTablesPreroutingDropped) + stateSinkObject.Save(11, &i.IPTablesInputDropped) + stateSinkObject.Save(12, &i.IPTablesForwardDropped) + stateSinkObject.Save(13, &i.IPTablesOutputDropped) + stateSinkObject.Save(14, &i.IPTablesPostroutingDropped) + stateSinkObject.Save(15, &i.OptionTimestampReceived) + stateSinkObject.Save(16, &i.OptionRecordRouteReceived) + stateSinkObject.Save(17, &i.OptionRouterAlertReceived) + stateSinkObject.Save(18, &i.OptionUnknownReceived) + stateSinkObject.Save(19, &i.Forwarding) +} + +func (i *IPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (i *IPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &i.PacketsReceived) + stateSourceObject.Load(1, &i.ValidPacketsReceived) + stateSourceObject.Load(2, &i.DisabledPacketsReceived) + stateSourceObject.Load(3, &i.InvalidDestinationAddressesReceived) + stateSourceObject.Load(4, &i.InvalidSourceAddressesReceived) + stateSourceObject.Load(5, &i.PacketsDelivered) + stateSourceObject.Load(6, &i.PacketsSent) + stateSourceObject.Load(7, &i.OutgoingPacketErrors) + stateSourceObject.Load(8, &i.MalformedPacketsReceived) + stateSourceObject.Load(9, &i.MalformedFragmentsReceived) + stateSourceObject.Load(10, &i.IPTablesPreroutingDropped) + stateSourceObject.Load(11, &i.IPTablesInputDropped) + stateSourceObject.Load(12, &i.IPTablesForwardDropped) + stateSourceObject.Load(13, &i.IPTablesOutputDropped) + stateSourceObject.Load(14, &i.IPTablesPostroutingDropped) + stateSourceObject.Load(15, &i.OptionTimestampReceived) + stateSourceObject.Load(16, &i.OptionRecordRouteReceived) + stateSourceObject.Load(17, &i.OptionRouterAlertReceived) + stateSourceObject.Load(18, &i.OptionUnknownReceived) + stateSourceObject.Load(19, &i.Forwarding) +} + +func (a *ARPStats) StateTypeName() string { + return "pkg/tcpip.ARPStats" +} + +func (a *ARPStats) StateFields() []string { + return []string{ + "PacketsReceived", + "DisabledPacketsReceived", + "MalformedPacketsReceived", + "RequestsReceived", + "RequestsReceivedUnknownTargetAddress", + "OutgoingRequestInterfaceHasNoLocalAddressErrors", + "OutgoingRequestBadLocalAddressErrors", + "OutgoingRequestsDropped", + "OutgoingRequestsSent", + "RepliesReceived", + "OutgoingRepliesDropped", + "OutgoingRepliesSent", + } +} + +func (a *ARPStats) beforeSave() {} + +// +checklocksignore +func (a *ARPStats) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.PacketsReceived) + stateSinkObject.Save(1, &a.DisabledPacketsReceived) + stateSinkObject.Save(2, &a.MalformedPacketsReceived) + stateSinkObject.Save(3, &a.RequestsReceived) + stateSinkObject.Save(4, &a.RequestsReceivedUnknownTargetAddress) + stateSinkObject.Save(5, &a.OutgoingRequestInterfaceHasNoLocalAddressErrors) + stateSinkObject.Save(6, &a.OutgoingRequestBadLocalAddressErrors) + stateSinkObject.Save(7, &a.OutgoingRequestsDropped) + stateSinkObject.Save(8, &a.OutgoingRequestsSent) + stateSinkObject.Save(9, &a.RepliesReceived) + stateSinkObject.Save(10, &a.OutgoingRepliesDropped) + stateSinkObject.Save(11, &a.OutgoingRepliesSent) +} + +func (a *ARPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (a *ARPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.PacketsReceived) + stateSourceObject.Load(1, &a.DisabledPacketsReceived) + stateSourceObject.Load(2, &a.MalformedPacketsReceived) + stateSourceObject.Load(3, &a.RequestsReceived) + stateSourceObject.Load(4, &a.RequestsReceivedUnknownTargetAddress) + stateSourceObject.Load(5, &a.OutgoingRequestInterfaceHasNoLocalAddressErrors) + stateSourceObject.Load(6, &a.OutgoingRequestBadLocalAddressErrors) + stateSourceObject.Load(7, &a.OutgoingRequestsDropped) + stateSourceObject.Load(8, &a.OutgoingRequestsSent) + stateSourceObject.Load(9, &a.RepliesReceived) + stateSourceObject.Load(10, &a.OutgoingRepliesDropped) + stateSourceObject.Load(11, &a.OutgoingRepliesSent) +} + +func (t *TCPStats) StateTypeName() string { + return "pkg/tcpip.TCPStats" +} + +func (t *TCPStats) StateFields() []string { + return []string{ + "ActiveConnectionOpenings", + "PassiveConnectionOpenings", + "CurrentEstablished", + "CurrentConnected", + "EstablishedResets", + "EstablishedClosed", + "EstablishedTimedout", + "ListenOverflowSynDrop", + "ListenOverflowAckDrop", + "ListenOverflowSynCookieSent", + "ListenOverflowSynCookieRcvd", + "ListenOverflowInvalidSynCookieRcvd", + "FailedConnectionAttempts", + "ValidSegmentsReceived", + "InvalidSegmentsReceived", + "SegmentsSent", + "SegmentSendErrors", + "ResetsSent", + "ResetsReceived", + "Retransmits", + "FastRecovery", + "SACKRecovery", + "TLPRecovery", + "SlowStartRetransmits", + "FastRetransmit", + "Timeouts", + "ChecksumErrors", + "FailedPortReservations", + "SegmentsAckedWithDSACK", + "SpuriousRecovery", + "SpuriousRTORecovery", + "ForwardMaxInFlightDrop", + } +} + +func (t *TCPStats) beforeSave() {} + +// +checklocksignore +func (t *TCPStats) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.ActiveConnectionOpenings) + stateSinkObject.Save(1, &t.PassiveConnectionOpenings) + stateSinkObject.Save(2, &t.CurrentEstablished) + stateSinkObject.Save(3, &t.CurrentConnected) + stateSinkObject.Save(4, &t.EstablishedResets) + stateSinkObject.Save(5, &t.EstablishedClosed) + stateSinkObject.Save(6, &t.EstablishedTimedout) + stateSinkObject.Save(7, &t.ListenOverflowSynDrop) + stateSinkObject.Save(8, &t.ListenOverflowAckDrop) + stateSinkObject.Save(9, &t.ListenOverflowSynCookieSent) + stateSinkObject.Save(10, &t.ListenOverflowSynCookieRcvd) + stateSinkObject.Save(11, &t.ListenOverflowInvalidSynCookieRcvd) + stateSinkObject.Save(12, &t.FailedConnectionAttempts) + stateSinkObject.Save(13, &t.ValidSegmentsReceived) + stateSinkObject.Save(14, &t.InvalidSegmentsReceived) + stateSinkObject.Save(15, &t.SegmentsSent) + stateSinkObject.Save(16, &t.SegmentSendErrors) + stateSinkObject.Save(17, &t.ResetsSent) + stateSinkObject.Save(18, &t.ResetsReceived) + stateSinkObject.Save(19, &t.Retransmits) + stateSinkObject.Save(20, &t.FastRecovery) + stateSinkObject.Save(21, &t.SACKRecovery) + stateSinkObject.Save(22, &t.TLPRecovery) + stateSinkObject.Save(23, &t.SlowStartRetransmits) + stateSinkObject.Save(24, &t.FastRetransmit) + stateSinkObject.Save(25, &t.Timeouts) + stateSinkObject.Save(26, &t.ChecksumErrors) + stateSinkObject.Save(27, &t.FailedPortReservations) + stateSinkObject.Save(28, &t.SegmentsAckedWithDSACK) + stateSinkObject.Save(29, &t.SpuriousRecovery) + stateSinkObject.Save(30, &t.SpuriousRTORecovery) + stateSinkObject.Save(31, &t.ForwardMaxInFlightDrop) +} + +func (t *TCPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.ActiveConnectionOpenings) + stateSourceObject.Load(1, &t.PassiveConnectionOpenings) + stateSourceObject.Load(2, &t.CurrentEstablished) + stateSourceObject.Load(3, &t.CurrentConnected) + stateSourceObject.Load(4, &t.EstablishedResets) + stateSourceObject.Load(5, &t.EstablishedClosed) + stateSourceObject.Load(6, &t.EstablishedTimedout) + stateSourceObject.Load(7, &t.ListenOverflowSynDrop) + stateSourceObject.Load(8, &t.ListenOverflowAckDrop) + stateSourceObject.Load(9, &t.ListenOverflowSynCookieSent) + stateSourceObject.Load(10, &t.ListenOverflowSynCookieRcvd) + stateSourceObject.Load(11, &t.ListenOverflowInvalidSynCookieRcvd) + stateSourceObject.Load(12, &t.FailedConnectionAttempts) + stateSourceObject.Load(13, &t.ValidSegmentsReceived) + stateSourceObject.Load(14, &t.InvalidSegmentsReceived) + stateSourceObject.Load(15, &t.SegmentsSent) + stateSourceObject.Load(16, &t.SegmentSendErrors) + stateSourceObject.Load(17, &t.ResetsSent) + stateSourceObject.Load(18, &t.ResetsReceived) + stateSourceObject.Load(19, &t.Retransmits) + stateSourceObject.Load(20, &t.FastRecovery) + stateSourceObject.Load(21, &t.SACKRecovery) + stateSourceObject.Load(22, &t.TLPRecovery) + stateSourceObject.Load(23, &t.SlowStartRetransmits) + stateSourceObject.Load(24, &t.FastRetransmit) + stateSourceObject.Load(25, &t.Timeouts) + stateSourceObject.Load(26, &t.ChecksumErrors) + stateSourceObject.Load(27, &t.FailedPortReservations) + stateSourceObject.Load(28, &t.SegmentsAckedWithDSACK) + stateSourceObject.Load(29, &t.SpuriousRecovery) + stateSourceObject.Load(30, &t.SpuriousRTORecovery) + stateSourceObject.Load(31, &t.ForwardMaxInFlightDrop) +} + +func (u *UDPStats) StateTypeName() string { + return "pkg/tcpip.UDPStats" +} + +func (u *UDPStats) StateFields() []string { + return []string{ + "PacketsReceived", + "UnknownPortErrors", + "ReceiveBufferErrors", + "MalformedPacketsReceived", + "PacketsSent", + "PacketSendErrors", + "ChecksumErrors", + } +} + +func (u *UDPStats) beforeSave() {} + +// +checklocksignore +func (u *UDPStats) StateSave(stateSinkObject state.Sink) { + u.beforeSave() + stateSinkObject.Save(0, &u.PacketsReceived) + stateSinkObject.Save(1, &u.UnknownPortErrors) + stateSinkObject.Save(2, &u.ReceiveBufferErrors) + stateSinkObject.Save(3, &u.MalformedPacketsReceived) + stateSinkObject.Save(4, &u.PacketsSent) + stateSinkObject.Save(5, &u.PacketSendErrors) + stateSinkObject.Save(6, &u.ChecksumErrors) +} + +func (u *UDPStats) afterLoad(context.Context) {} + +// +checklocksignore +func (u *UDPStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &u.PacketsReceived) + stateSourceObject.Load(1, &u.UnknownPortErrors) + stateSourceObject.Load(2, &u.ReceiveBufferErrors) + stateSourceObject.Load(3, &u.MalformedPacketsReceived) + stateSourceObject.Load(4, &u.PacketsSent) + stateSourceObject.Load(5, &u.PacketSendErrors) + stateSourceObject.Load(6, &u.ChecksumErrors) +} + +func (n *NICNeighborStats) StateTypeName() string { + return "pkg/tcpip.NICNeighborStats" +} + +func (n *NICNeighborStats) StateFields() []string { + return []string{ + "UnreachableEntryLookups", + "DroppedConfirmationForNoninitiatedNeighbor", + "DroppedInvalidLinkAddressConfirmations", + } +} + +func (n *NICNeighborStats) beforeSave() {} + +// +checklocksignore +func (n *NICNeighborStats) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.UnreachableEntryLookups) + stateSinkObject.Save(1, &n.DroppedConfirmationForNoninitiatedNeighbor) + stateSinkObject.Save(2, &n.DroppedInvalidLinkAddressConfirmations) +} + +func (n *NICNeighborStats) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NICNeighborStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.UnreachableEntryLookups) + stateSourceObject.Load(1, &n.DroppedConfirmationForNoninitiatedNeighbor) + stateSourceObject.Load(2, &n.DroppedInvalidLinkAddressConfirmations) +} + +func (n *NICPacketStats) StateTypeName() string { + return "pkg/tcpip.NICPacketStats" +} + +func (n *NICPacketStats) StateFields() []string { + return []string{ + "Packets", + "Bytes", + } +} + +func (n *NICPacketStats) beforeSave() {} + +// +checklocksignore +func (n *NICPacketStats) StateSave(stateSinkObject state.Sink) { + n.beforeSave() + stateSinkObject.Save(0, &n.Packets) + stateSinkObject.Save(1, &n.Bytes) +} + +func (n *NICPacketStats) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NICPacketStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &n.Packets) + stateSourceObject.Load(1, &n.Bytes) +} + +func (m *IntegralStatCounterMap) StateTypeName() string { + return "pkg/tcpip.IntegralStatCounterMap" +} + +func (m *IntegralStatCounterMap) StateFields() []string { + return []string{ + "counterMap", + } +} + +func (m *IntegralStatCounterMap) beforeSave() {} + +// +checklocksignore +func (m *IntegralStatCounterMap) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.counterMap) +} + +func (m *IntegralStatCounterMap) afterLoad(context.Context) {} + +// +checklocksignore +func (m *IntegralStatCounterMap) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.counterMap) +} + +func (m *MultiIntegralStatCounterMap) StateTypeName() string { + return "pkg/tcpip.MultiIntegralStatCounterMap" +} + +func (m *MultiIntegralStatCounterMap) StateFields() []string { + return []string{ + "a", + "b", + } +} + +func (m *MultiIntegralStatCounterMap) beforeSave() {} + +// +checklocksignore +func (m *MultiIntegralStatCounterMap) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.a) + stateSinkObject.Save(1, &m.b) +} + +func (m *MultiIntegralStatCounterMap) afterLoad(context.Context) {} + +// +checklocksignore +func (m *MultiIntegralStatCounterMap) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.a) + stateSourceObject.Load(1, &m.b) +} + +func (s *NICStats) StateTypeName() string { + return "pkg/tcpip.NICStats" +} + +func (s *NICStats) StateFields() []string { + return []string{ + "UnknownL3ProtocolRcvdPacketCounts", + "UnknownL4ProtocolRcvdPacketCounts", + "MalformedL4RcvdPackets", + "Tx", + "TxPacketsDroppedNoBufferSpace", + "Rx", + "DisabledRx", + "Neighbor", + } +} + +func (s *NICStats) beforeSave() {} + +// +checklocksignore +func (s *NICStats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.UnknownL3ProtocolRcvdPacketCounts) + stateSinkObject.Save(1, &s.UnknownL4ProtocolRcvdPacketCounts) + stateSinkObject.Save(2, &s.MalformedL4RcvdPackets) + stateSinkObject.Save(3, &s.Tx) + stateSinkObject.Save(4, &s.TxPacketsDroppedNoBufferSpace) + stateSinkObject.Save(5, &s.Rx) + stateSinkObject.Save(6, &s.DisabledRx) + stateSinkObject.Save(7, &s.Neighbor) +} + +func (s *NICStats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *NICStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.UnknownL3ProtocolRcvdPacketCounts) + stateSourceObject.Load(1, &s.UnknownL4ProtocolRcvdPacketCounts) + stateSourceObject.Load(2, &s.MalformedL4RcvdPackets) + stateSourceObject.Load(3, &s.Tx) + stateSourceObject.Load(4, &s.TxPacketsDroppedNoBufferSpace) + stateSourceObject.Load(5, &s.Rx) + stateSourceObject.Load(6, &s.DisabledRx) + stateSourceObject.Load(7, &s.Neighbor) +} + +func (s *Stats) StateTypeName() string { + return "pkg/tcpip.Stats" +} + +func (s *Stats) StateFields() []string { + return []string{ + "DroppedPackets", + "NICs", + "ICMP", + "IGMP", + "IP", + "ARP", + "TCP", + "UDP", + } +} + +func (s *Stats) beforeSave() {} + +// +checklocksignore +func (s *Stats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.DroppedPackets) + stateSinkObject.Save(1, &s.NICs) + stateSinkObject.Save(2, &s.ICMP) + stateSinkObject.Save(3, &s.IGMP) + stateSinkObject.Save(4, &s.IP) + stateSinkObject.Save(5, &s.ARP) + stateSinkObject.Save(6, &s.TCP) + stateSinkObject.Save(7, &s.UDP) +} + +func (s *Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.DroppedPackets) + stateSourceObject.Load(1, &s.NICs) + stateSourceObject.Load(2, &s.ICMP) + stateSourceObject.Load(3, &s.IGMP) + stateSourceObject.Load(4, &s.IP) + stateSourceObject.Load(5, &s.ARP) + stateSourceObject.Load(6, &s.TCP) + stateSourceObject.Load(7, &s.UDP) +} + +func (r *ReceiveErrors) StateTypeName() string { + return "pkg/tcpip.ReceiveErrors" +} + +func (r *ReceiveErrors) StateFields() []string { + return []string{ + "ReceiveBufferOverflow", + "MalformedPacketsReceived", + "ClosedReceiver", + "ChecksumErrors", + } +} + +func (r *ReceiveErrors) beforeSave() {} + +// +checklocksignore +func (r *ReceiveErrors) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.ReceiveBufferOverflow) + stateSinkObject.Save(1, &r.MalformedPacketsReceived) + stateSinkObject.Save(2, &r.ClosedReceiver) + stateSinkObject.Save(3, &r.ChecksumErrors) +} + +func (r *ReceiveErrors) afterLoad(context.Context) {} + +// +checklocksignore +func (r *ReceiveErrors) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.ReceiveBufferOverflow) + stateSourceObject.Load(1, &r.MalformedPacketsReceived) + stateSourceObject.Load(2, &r.ClosedReceiver) + stateSourceObject.Load(3, &r.ChecksumErrors) +} + +func (s *SendErrors) StateTypeName() string { + return "pkg/tcpip.SendErrors" +} + +func (s *SendErrors) StateFields() []string { + return []string{ + "SendToNetworkFailed", + "NoRoute", + } +} + +func (s *SendErrors) beforeSave() {} + +// +checklocksignore +func (s *SendErrors) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.SendToNetworkFailed) + stateSinkObject.Save(1, &s.NoRoute) +} + +func (s *SendErrors) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SendErrors) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.SendToNetworkFailed) + stateSourceObject.Load(1, &s.NoRoute) +} + +func (r *ReadErrors) StateTypeName() string { + return "pkg/tcpip.ReadErrors" +} + +func (r *ReadErrors) StateFields() []string { + return []string{ + "ReadClosed", + "InvalidEndpointState", + "NotConnected", + } +} + +func (r *ReadErrors) beforeSave() {} + +// +checklocksignore +func (r *ReadErrors) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.ReadClosed) + stateSinkObject.Save(1, &r.InvalidEndpointState) + stateSinkObject.Save(2, &r.NotConnected) +} + +func (r *ReadErrors) afterLoad(context.Context) {} + +// +checklocksignore +func (r *ReadErrors) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.ReadClosed) + stateSourceObject.Load(1, &r.InvalidEndpointState) + stateSourceObject.Load(2, &r.NotConnected) +} + +func (w *WriteErrors) StateTypeName() string { + return "pkg/tcpip.WriteErrors" +} + +func (w *WriteErrors) StateFields() []string { + return []string{ + "WriteClosed", + "InvalidEndpointState", + "InvalidArgs", + } +} + +func (w *WriteErrors) beforeSave() {} + +// +checklocksignore +func (w *WriteErrors) StateSave(stateSinkObject state.Sink) { + w.beforeSave() + stateSinkObject.Save(0, &w.WriteClosed) + stateSinkObject.Save(1, &w.InvalidEndpointState) + stateSinkObject.Save(2, &w.InvalidArgs) +} + +func (w *WriteErrors) afterLoad(context.Context) {} + +// +checklocksignore +func (w *WriteErrors) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &w.WriteClosed) + stateSourceObject.Load(1, &w.InvalidEndpointState) + stateSourceObject.Load(2, &w.InvalidArgs) +} + +func (src *TransportEndpointStats) StateTypeName() string { + return "pkg/tcpip.TransportEndpointStats" +} + +func (src *TransportEndpointStats) StateFields() []string { + return []string{ + "PacketsReceived", + "PacketsSent", + "ReceiveErrors", + "ReadErrors", + "SendErrors", + "WriteErrors", + } +} + +func (src *TransportEndpointStats) beforeSave() {} + +// +checklocksignore +func (src *TransportEndpointStats) StateSave(stateSinkObject state.Sink) { + src.beforeSave() + stateSinkObject.Save(0, &src.PacketsReceived) + stateSinkObject.Save(1, &src.PacketsSent) + stateSinkObject.Save(2, &src.ReceiveErrors) + stateSinkObject.Save(3, &src.ReadErrors) + stateSinkObject.Save(4, &src.SendErrors) + stateSinkObject.Save(5, &src.WriteErrors) +} + +func (src *TransportEndpointStats) afterLoad(context.Context) {} + +// +checklocksignore +func (src *TransportEndpointStats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &src.PacketsReceived) + stateSourceObject.Load(1, &src.PacketsSent) + stateSourceObject.Load(2, &src.ReceiveErrors) + stateSourceObject.Load(3, &src.ReadErrors) + stateSourceObject.Load(4, &src.SendErrors) + stateSourceObject.Load(5, &src.WriteErrors) +} + +func (a *AddressWithPrefix) StateTypeName() string { + return "pkg/tcpip.AddressWithPrefix" +} + +func (a *AddressWithPrefix) StateFields() []string { + return []string{ + "Address", + "PrefixLen", + } +} + +func (a *AddressWithPrefix) beforeSave() {} + +// +checklocksignore +func (a *AddressWithPrefix) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + stateSinkObject.Save(0, &a.Address) + stateSinkObject.Save(1, &a.PrefixLen) +} + +func (a *AddressWithPrefix) afterLoad(context.Context) {} + +// +checklocksignore +func (a *AddressWithPrefix) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &a.Address) + stateSourceObject.Load(1, &a.PrefixLen) +} + +func (p *ProtocolAddress) StateTypeName() string { + return "pkg/tcpip.ProtocolAddress" +} + +func (p *ProtocolAddress) StateFields() []string { + return []string{ + "Protocol", + "AddressWithPrefix", + } +} + +func (p *ProtocolAddress) beforeSave() {} + +// +checklocksignore +func (p *ProtocolAddress) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.Protocol) + stateSinkObject.Save(1, &p.AddressWithPrefix) +} + +func (p *ProtocolAddress) afterLoad(context.Context) {} + +// +checklocksignore +func (p *ProtocolAddress) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.Protocol) + stateSourceObject.Load(1, &p.AddressWithPrefix) +} + +func (j *jobInstance) StateTypeName() string { + return "pkg/tcpip.jobInstance" +} + +func (j *jobInstance) StateFields() []string { + return []string{ + "timer", + "earlyReturn", + } +} + +func (j *jobInstance) beforeSave() {} + +// +checklocksignore +func (j *jobInstance) StateSave(stateSinkObject state.Sink) { + j.beforeSave() + stateSinkObject.Save(0, &j.timer) + stateSinkObject.Save(1, &j.earlyReturn) +} + +func (j *jobInstance) afterLoad(context.Context) {} + +// +checklocksignore +func (j *jobInstance) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &j.timer) + stateSourceObject.Load(1, &j.earlyReturn) +} + +func (j *Job) StateTypeName() string { + return "pkg/tcpip.Job" +} + +func (j *Job) StateFields() []string { + return []string{ + "clock", + "instance", + } +} + +func (j *Job) beforeSave() {} + +// +checklocksignore +func (j *Job) StateSave(stateSinkObject state.Sink) { + j.beforeSave() + stateSinkObject.Save(0, &j.clock) + stateSinkObject.Save(1, &j.instance) +} + +func (j *Job) afterLoad(context.Context) {} + +// +checklocksignore +func (j *Job) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &j.clock) + stateSourceObject.Load(1, &j.instance) +} + +func init() { + state.Register((*ErrAborted)(nil)) + state.Register((*ErrAddressFamilyNotSupported)(nil)) + state.Register((*ErrAlreadyBound)(nil)) + state.Register((*ErrAlreadyConnected)(nil)) + state.Register((*ErrAlreadyConnecting)(nil)) + state.Register((*ErrBadAddress)(nil)) + state.Register((*ErrBadBuffer)(nil)) + state.Register((*ErrBadLocalAddress)(nil)) + state.Register((*ErrBroadcastDisabled)(nil)) + state.Register((*ErrClosedForReceive)(nil)) + state.Register((*ErrClosedForSend)(nil)) + state.Register((*ErrConnectStarted)(nil)) + state.Register((*ErrConnectionAborted)(nil)) + state.Register((*ErrConnectionRefused)(nil)) + state.Register((*ErrConnectionReset)(nil)) + state.Register((*ErrDestinationRequired)(nil)) + state.Register((*ErrDuplicateAddress)(nil)) + state.Register((*ErrDuplicateNICID)(nil)) + state.Register((*ErrInvalidNICID)(nil)) + state.Register((*ErrInvalidEndpointState)(nil)) + state.Register((*ErrInvalidOptionValue)(nil)) + state.Register((*ErrInvalidPortRange)(nil)) + state.Register((*ErrMalformedHeader)(nil)) + state.Register((*ErrMessageTooLong)(nil)) + state.Register((*ErrNetworkUnreachable)(nil)) + state.Register((*ErrNoBufferSpace)(nil)) + state.Register((*ErrNoPortAvailable)(nil)) + state.Register((*ErrHostUnreachable)(nil)) + state.Register((*ErrHostDown)(nil)) + state.Register((*ErrNoNet)(nil)) + state.Register((*ErrNoSuchFile)(nil)) + state.Register((*ErrNotConnected)(nil)) + state.Register((*ErrNotPermitted)(nil)) + state.Register((*ErrNotSupported)(nil)) + state.Register((*ErrPortInUse)(nil)) + state.Register((*ErrQueueSizeNotSupported)(nil)) + state.Register((*ErrTimeout)(nil)) + state.Register((*ErrUnknownDevice)(nil)) + state.Register((*ErrUnknownNICID)(nil)) + state.Register((*ErrUnknownProtocol)(nil)) + state.Register((*ErrUnknownProtocolOption)(nil)) + state.Register((*ErrWouldBlock)(nil)) + state.Register((*ErrMissingRequiredFields)(nil)) + state.Register((*ErrMulticastInputCannotBeOutput)(nil)) + state.Register((*ErrEndpointBusy)(nil)) + state.Register((*RouteList)(nil)) + state.Register((*RouteEntry)(nil)) + state.Register((*sockErrorList)(nil)) + state.Register((*sockErrorEntry)(nil)) + state.Register((*SocketOptions)(nil)) + state.Register((*LocalSockError)(nil)) + state.Register((*SockError)(nil)) + state.Register((*stdClock)(nil)) + state.Register((*stdTimer)(nil)) + state.Register((*MonotonicTime)(nil)) + state.Register((*Address)(nil)) + state.Register((*AddressMask)(nil)) + state.Register((*Subnet)(nil)) + state.Register((*FullAddress)(nil)) + state.Register((*SendableControlMessages)(nil)) + state.Register((*ReceivableControlMessages)(nil)) + state.Register((*LinkPacketInfo)(nil)) + state.Register((*TCPSendBufferSizeRangeOption)(nil)) + state.Register((*TCPReceiveBufferSizeRangeOption)(nil)) + state.Register((*ICMPv6Filter)(nil)) + state.Register((*TpacketReq)(nil)) + state.Register((*TpacketStats)(nil)) + state.Register((*LingerOption)(nil)) + state.Register((*IPPacketInfo)(nil)) + state.Register((*IPv6PacketInfo)(nil)) + state.Register((*SendBufferSizeOption)(nil)) + state.Register((*ReceiveBufferSizeOption)(nil)) + state.Register((*Route)(nil)) + state.Register((*StatCounter)(nil)) + state.Register((*MultiCounterStat)(nil)) + state.Register((*ICMPv4PacketStats)(nil)) + state.Register((*ICMPv4SentPacketStats)(nil)) + state.Register((*ICMPv4ReceivedPacketStats)(nil)) + state.Register((*ICMPv4Stats)(nil)) + state.Register((*ICMPv6PacketStats)(nil)) + state.Register((*ICMPv6SentPacketStats)(nil)) + state.Register((*ICMPv6ReceivedPacketStats)(nil)) + state.Register((*ICMPv6Stats)(nil)) + state.Register((*ICMPStats)(nil)) + state.Register((*IGMPPacketStats)(nil)) + state.Register((*IGMPSentPacketStats)(nil)) + state.Register((*IGMPReceivedPacketStats)(nil)) + state.Register((*IGMPStats)(nil)) + state.Register((*IPForwardingStats)(nil)) + state.Register((*IPStats)(nil)) + state.Register((*ARPStats)(nil)) + state.Register((*TCPStats)(nil)) + state.Register((*UDPStats)(nil)) + state.Register((*NICNeighborStats)(nil)) + state.Register((*NICPacketStats)(nil)) + state.Register((*IntegralStatCounterMap)(nil)) + state.Register((*MultiIntegralStatCounterMap)(nil)) + state.Register((*NICStats)(nil)) + state.Register((*Stats)(nil)) + state.Register((*ReceiveErrors)(nil)) + state.Register((*SendErrors)(nil)) + state.Register((*ReadErrors)(nil)) + state.Register((*WriteErrors)(nil)) + state.Register((*TransportEndpointStats)(nil)) + state.Register((*AddressWithPrefix)(nil)) + state.Register((*ProtocolAddress)(nil)) + state.Register((*jobInstance)(nil)) + state.Register((*Job)(nil)) +} diff --git a/pkg/tcpip/timer.go b/pkg/tcpip/timer.go new file mode 100644 index 0000000..91bc001 --- /dev/null +++ b/pkg/tcpip/timer.go @@ -0,0 +1,212 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpip + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/sync" +) + +// jobInstance is a specific instance of Job. +// +// Different instances are created each time Job is scheduled so each timer has +// its own earlyReturn signal. This is to address a bug when a Job is stopped +// and reset in quick succession resulting in a timer instance's earlyReturn +// signal being affected or seen by another timer instance. +// +// Consider the following sceneario where timer instances share a common +// earlyReturn signal (T1 creates, stops and resets a Cancellable timer under a +// lock L; T2, T3, T4 and T5 are goroutines that handle the first (A), second +// (B), third (C), and fourth (D) instance of the timer firing, respectively): +// +// T1: Obtain L +// T1: Create a new Job w/ lock L (create instance A) +// T2: instance A fires, blocked trying to obtain L. +// T1: Attempt to stop instance A (set earlyReturn = true) +// T1: Schedule timer (create instance B) +// T3: instance B fires, blocked trying to obtain L. +// T1: Attempt to stop instance B (set earlyReturn = true) +// T1: Schedule timer (create instance C) +// T4: instance C fires, blocked trying to obtain L. +// T1: Attempt to stop instance C (set earlyReturn = true) +// T1: Schedule timer (create instance D) +// T5: instance D fires, blocked trying to obtain L. +// T1: Release L +// +// Now that T1 has released L, any of the 4 timer instances can take L and +// check earlyReturn. If the timers simply check earlyReturn and then do +// nothing further, then instance D will never early return even though it was +// not requested to stop. If the timers reset earlyReturn before early +// returning, then all but one of the timers will do work when only one was +// expected to. If Job resets earlyReturn when resetting, then all the timers +// will fire (again, when only one was expected to). +// +// To address the above concerns the simplest solution was to give each timer +// its own earlyReturn signal. +// +// +stateify savable +type jobInstance struct { + timer Timer + + // Used to inform the timer to early return when it gets stopped while the + // lock the timer tries to obtain when fired is held (T1 is a goroutine that + // tries to cancel the timer and T2 is the goroutine that handles the timer + // firing): + // T1: Obtain the lock, then call Cancel() + // T2: timer fires, and gets blocked on obtaining the lock + // T1: Releases lock + // T2: Obtains lock does unintended work + // + // To resolve this, T1 will check to see if the timer already fired, and + // inform the timer using earlyReturn to return early so that once T2 obtains + // the lock, it will see that it is set to true and do nothing further. + earlyReturn *bool +} + +// stop stops the job instance j from firing if it hasn't fired already. If it +// has fired and is blocked at obtaining the lock, earlyReturn will be set to +// true so that it will early return when it obtains the lock. +func (j *jobInstance) stop() { + if j.timer != nil { + j.timer.Stop() + *j.earlyReturn = true + } +} + +// Job represents some work that can be scheduled for execution. The work can +// be safely cancelled when it fires at the same time some "related work" is +// being done. +// +// The term "related work" is defined as some work that needs to be done while +// holding some lock that the timer must also hold while doing some work. +// +// Note, it is not safe to copy a Job as its timer instance creates +// a closure over the address of the Job. +// +// +stateify savable +type Job struct { + _ sync.NoCopy + + // The clock used to schedule the backing timer + clock Clock + + // The active instance of a cancellable timer. + instance jobInstance + + // locker is the lock taken by the timer immediately after it fires and must + // be held when attempting to stop the timer. + // + // Must never change after being assigned. + locker sync.Locker `state:"nosave"` + + // fn is the function that will be called when a timer fires and has not been + // signaled to early return. + // + // fn MUST NOT attempt to lock locker. + // + // Must never change after being assigned. + // TODO(b/341946753): Restore when netstack is savable. + fn func() `state:"nosave"` +} + +// Cancel prevents the Job from executing if it has not executed already. +// +// Cancel requires appropriate locking to be in place for any resources managed +// by the Job. If the Job is blocked on obtaining the lock when Cancel is +// called, it will early return. +// +// Note, t will be modified. +// +// j.locker MUST be locked. +func (j *Job) Cancel() { + j.instance.stop() + + // Nothing to do with the stopped instance anymore. + j.instance = jobInstance{} +} + +// Schedule schedules the Job for execution after duration d. This can be +// called on cancelled or completed Jobs to schedule them again. +// +// Schedule should be invoked only on unscheduled, cancelled, or completed +// Jobs. To be safe, callers should always call Cancel before calling Schedule. +// +// Note, j will be modified. +func (j *Job) Schedule(d time.Duration) { + // Create a new instance. + earlyReturn := false + + // Capture the locker so that updating the timer does not cause a data race + // when a timer fires and tries to obtain the lock (read the timer's locker). + locker := j.locker + j.instance = jobInstance{ + timer: j.clock.AfterFunc(d, func() { + locker.Lock() + defer locker.Unlock() + + if earlyReturn { + // If we reach this point, it means that the timer fired while another + // goroutine called Cancel while it had the lock. Simply return here + // and do nothing further. + earlyReturn = false + return + } + + j.fn() + }), + earlyReturn: &earlyReturn, + } +} + +// NewJob returns a new Job that can be used to schedule f to run in its own +// gorountine. l will be locked before calling f then unlocked after f returns. +// +// var clock tcpip.StdClock +// var mu sync.Mutex +// message := "foo" +// job := tcpip.NewJob(&clock, &mu, func() { +// fmt.Println(message) +// }) +// job.Schedule(time.Second) +// +// mu.Lock() +// message = "bar" +// mu.Unlock() +// +// // Output: bar +// +// f MUST NOT attempt to lock l. +// +// l MUST be locked prior to calling the returned job's Cancel(). +// +// var clock tcpip.StdClock +// var mu sync.Mutex +// message := "foo" +// job := tcpip.NewJob(&clock, &mu, func() { +// fmt.Println(message) +// }) +// job.Schedule(time.Second) +// +// mu.Lock() +// job.Cancel() +// mu.Unlock() +func NewJob(c Clock, l sync.Locker, f func()) *Job { + return &Job{ + clock: c, + locker: l, + fn: f, + } +} diff --git a/pkg/tcpip/transport/datagram.go b/pkg/tcpip/transport/datagram.go new file mode 100644 index 0000000..db7ceb2 --- /dev/null +++ b/pkg/tcpip/transport/datagram.go @@ -0,0 +1,49 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// DatagramEndpointState is the state of a datagram-based endpoint. +type DatagramEndpointState tcpip.EndpointState + +// The states a datagram-based endpoint may be in. +const ( + _ DatagramEndpointState = iota + DatagramEndpointStateInitial + DatagramEndpointStateBound + DatagramEndpointStateConnected + DatagramEndpointStateClosed +) + +// String implements fmt.Stringer. +func (s DatagramEndpointState) String() string { + switch s { + case DatagramEndpointStateInitial: + return "INITIAL" + case DatagramEndpointStateBound: + return "BOUND" + case DatagramEndpointStateConnected: + return "CONNECTED" + case DatagramEndpointStateClosed: + return "CLOSED" + default: + panic(fmt.Sprintf("unhandled %[1]T variant = %[1]d", s)) + } +} diff --git a/pkg/tcpip/transport/icmp/endpoint.go b/pkg/tcpip/transport/icmp/endpoint.go new file mode 100644 index 0000000..8ec5f70 --- /dev/null +++ b/pkg/tcpip/transport/icmp/endpoint.go @@ -0,0 +1,828 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package icmp + +import ( + "fmt" + "io" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" + "github.com/sagernet/gvisor/pkg/tcpip/transport/internal/network" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// +stateify savable +type icmpPacket struct { + icmpPacketEntry + senderAddress tcpip.FullAddress + packetInfo tcpip.IPPacketInfo + data *stack.PacketBuffer + receivedAt time.Time `state:".(int64)"` + + // tosOrTClass stores either the Type of Service for IPv4 or the Traffic Class + // for IPv6. + tosOrTClass uint8 + // ttlOrHopLimit stores either the TTL for IPv4 or the HopLimit for IPv6 + ttlOrHopLimit uint8 +} + +// endpoint represents an ICMP endpoint. This struct serves as the interface +// between users of the endpoint and the protocol implementation; it is legal to +// have concurrent goroutines make calls into the endpoint, they are properly +// synchronized. +// +// +stateify savable +type endpoint struct { + tcpip.DefaultSocketOptionsHandler + + // The following fields are initialized at creation time and are + // immutable. + stack *stack.Stack + transProto tcpip.TransportProtocolNumber + waiterQueue *waiter.Queue + net network.Endpoint + stats tcpip.TransportEndpointStats + ops tcpip.SocketOptions + + // The following fields are used to manage the receive queue, and are + // protected by rcvMu. + rcvMu sync.Mutex `state:"nosave"` + rcvReady bool + rcvList icmpPacketList + rcvBufSize int + rcvClosed bool + + // The following fields are protected by the mu mutex. + mu sync.RWMutex `state:"nosave"` + // frozen indicates if the packets should be delivered to the endpoint + // during restore. + frozen bool + ident uint16 +} + +func newEndpoint(s *stack.Stack, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + ep := &endpoint{ + stack: s, + transProto: transProto, + waiterQueue: waiterQueue, + } + ep.ops.InitHandler(ep, ep.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + ep.ops.SetSendBufferSize(32*1024, false /* notify */) + ep.ops.SetReceiveBufferSize(32*1024, false /* notify */) + ep.net.Init(s, netProto, transProto, &ep.ops, waiterQueue) + + // Override with stack defaults. + var ss tcpip.SendBufferSizeOption + if err := s.Option(&ss); err == nil { + ep.ops.SetSendBufferSize(int64(ss.Default), false /* notify */) + } + var rs tcpip.ReceiveBufferSizeOption + if err := s.Option(&rs); err == nil { + ep.ops.SetReceiveBufferSize(int64(rs.Default), false /* notify */) + } + return ep, nil +} + +// WakeupWriters implements tcpip.SocketOptionsHandler. +func (e *endpoint) WakeupWriters() { + e.net.MaybeSignalWritable() +} + +// Abort implements stack.TransportEndpoint.Abort. +func (e *endpoint) Abort() { + e.Close() +} + +// Close puts the endpoint in a closed state and frees all resources +// associated with it. +func (e *endpoint) Close() { + notify := func() bool { + e.mu.Lock() + defer e.mu.Unlock() + + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial: + case transport.DatagramEndpointStateClosed: + return false + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + info := e.net.Info() + info.ID.LocalPort = e.ident + e.stack.UnregisterTransportEndpoint([]tcpip.NetworkProtocolNumber{info.NetProto}, e.transProto, info.ID, e, ports.Flags{}, tcpip.NICID(e.ops.GetBindToDevice())) + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } + + e.net.Shutdown() + e.net.Close() + + e.rcvMu.Lock() + defer e.rcvMu.Unlock() + e.rcvClosed = true + e.rcvBufSize = 0 + for !e.rcvList.Empty() { + p := e.rcvList.Front() + e.rcvList.Remove(p) + p.data.DecRef() + } + + return true + }() + + if notify { + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + } +} + +// ModerateRecvBuf implements tcpip.Endpoint.ModerateRecvBuf. +func (*endpoint) ModerateRecvBuf(int) {} + +// SetOwner implements tcpip.Endpoint.SetOwner. +func (e *endpoint) SetOwner(owner tcpip.PacketOwner) { + e.net.SetOwner(owner) +} + +// Read implements tcpip.Endpoint.Read. +func (e *endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult, tcpip.Error) { + e.rcvMu.Lock() + + if e.rcvList.Empty() { + var err tcpip.Error = &tcpip.ErrWouldBlock{} + if e.rcvClosed { + e.stats.ReadErrors.ReadClosed.Increment() + err = &tcpip.ErrClosedForReceive{} + } + e.rcvMu.Unlock() + return tcpip.ReadResult{}, err + } + + p := e.rcvList.Front() + if !opts.Peek { + e.rcvList.Remove(p) + defer p.data.DecRef() + e.rcvBufSize -= p.data.Data().Size() + } + + e.rcvMu.Unlock() + + // Control Messages + // TODO(https://gvisor.dev/issue/7012): Share control message code with other + // network endpoints. + cm := tcpip.ReceivableControlMessages{ + HasTimestamp: true, + Timestamp: p.receivedAt, + } + switch netProto := e.net.NetProto(); netProto { + case header.IPv4ProtocolNumber: + if e.ops.GetReceiveTOS() { + cm.HasTOS = true + cm.TOS = p.tosOrTClass + } + if e.ops.GetReceivePacketInfo() { + cm.HasIPPacketInfo = true + cm.PacketInfo = p.packetInfo + } + if e.ops.GetReceiveTTL() { + cm.HasTTL = true + cm.TTL = p.ttlOrHopLimit + } + case header.IPv6ProtocolNumber: + if e.ops.GetReceiveTClass() { + cm.HasTClass = true + // Although TClass is an 8-bit value it's read in the CMsg as a uint32. + cm.TClass = uint32(p.tosOrTClass) + } + if e.ops.GetIPv6ReceivePacketInfo() { + cm.HasIPv6PacketInfo = true + cm.IPv6PacketInfo = tcpip.IPv6PacketInfo{ + NIC: p.packetInfo.NIC, + Addr: p.packetInfo.DestinationAddr, + } + } + if e.ops.GetReceiveHopLimit() { + cm.HasHopLimit = true + cm.HopLimit = p.ttlOrHopLimit + } + default: + panic(fmt.Sprintf("unrecognized network protocol = %d", netProto)) + } + + res := tcpip.ReadResult{ + Total: p.data.Data().Size(), + ControlMessages: cm, + } + if opts.NeedRemoteAddr { + res.RemoteAddr = p.senderAddress + } + + n, err := p.data.Data().ReadTo(dst, opts.Peek) + if n == 0 && err != nil { + return res, &tcpip.ErrBadBuffer{} + } + res.Count = n + return res, nil +} + +// prepareForWrite prepares the endpoint for sending data. In particular, it +// binds it if it's still in the initial state. To do so, it must first +// reacquire the mutex in exclusive mode. +// +// Returns true for retry if preparation should be retried. +// +checklocksread:e.mu +func (e *endpoint) prepareForWriteInner(to *tcpip.FullAddress) (retry bool, err tcpip.Error) { + switch e.net.State() { + case transport.DatagramEndpointStateInitial: + case transport.DatagramEndpointStateConnected: + return false, nil + case transport.DatagramEndpointStateBound: + if to == nil { + return false, &tcpip.ErrDestinationRequired{} + } + return false, nil + default: + return false, &tcpip.ErrInvalidEndpointState{} + } + + e.mu.RUnlock() + e.mu.Lock() + defer e.mu.DowngradeLock() + + // The state changed when we released the shared locked and re-acquired + // it in exclusive mode. Try again. + if e.net.State() != transport.DatagramEndpointStateInitial { + return true, nil + } + + // The state is still 'initial', so try to bind the endpoint. + if err := e.bindLocked(tcpip.FullAddress{}); err != nil { + return false, err + } + + return true, nil +} + +// Write writes data to the endpoint's peer. This method does not block +// if the data cannot be written. +func (e *endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + n, err := e.write(p, opts) + switch err.(type) { + case nil: + e.stats.PacketsSent.Increment() + case *tcpip.ErrMessageTooLong, *tcpip.ErrInvalidOptionValue: + e.stats.WriteErrors.InvalidArgs.Increment() + case *tcpip.ErrClosedForSend: + e.stats.WriteErrors.WriteClosed.Increment() + case *tcpip.ErrInvalidEndpointState: + e.stats.WriteErrors.InvalidEndpointState.Increment() + case *tcpip.ErrHostUnreachable, *tcpip.ErrBroadcastDisabled, *tcpip.ErrNetworkUnreachable: + // Errors indicating any problem with IP routing of the packet. + e.stats.SendErrors.NoRoute.Increment() + default: + // For all other errors when writing to the network layer. + e.stats.SendErrors.SendToNetworkFailed.Increment() + } + return n, err +} + +func (e *endpoint) prepareForWrite(opts tcpip.WriteOptions) (network.WriteContext, uint16, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + // Prepare for write. + for { + retry, err := e.prepareForWriteInner(opts.To) + if err != nil { + return network.WriteContext{}, 0, err + } + + if !retry { + break + } + } + + ctx, err := e.net.AcquireContextForWrite(opts) + return ctx, e.ident, err +} + +func (e *endpoint) write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + ctx, ident, err := e.prepareForWrite(opts) + if err != nil { + return 0, err + } + defer ctx.Release() + + // Prevents giant buffer allocations. + if p.Len() > header.DatagramMaximumSize { + return 0, &tcpip.ErrMessageTooLong{} + } + + v := buffer.NewView(p.Len()) + defer v.Release() + if _, err := io.CopyN(v, p, int64(p.Len())); err != nil { + return 0, &tcpip.ErrBadBuffer{} + } + n := v.Size() + + switch netProto, pktInfo := e.net.NetProto(), ctx.PacketInfo(); netProto { + case header.IPv4ProtocolNumber: + if err := send4(e.stack, &ctx, ident, v, pktInfo.MaxHeaderLength); err != nil { + return 0, err + } + + case header.IPv6ProtocolNumber: + if err := send6(e.stack, &ctx, ident, v, pktInfo.LocalAddress, pktInfo.RemoteAddress, pktInfo.MaxHeaderLength); err != nil { + return 0, err + } + default: + panic(fmt.Sprintf("unhandled network protocol = %d", netProto)) + } + + return int64(n), nil +} + +var _ tcpip.SocketOptionsHandler = (*endpoint)(nil) + +// HasNIC implements tcpip.SocketOptionsHandler. +func (e *endpoint) HasNIC(id int32) bool { + return e.stack.HasNIC(tcpip.NICID(id)) +} + +// SetSockOpt implements tcpip.Endpoint. +func (e *endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { + return e.net.SetSockOpt(opt) +} + +// SetSockOptInt implements tcpip.Endpoint. +func (e *endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { + return e.net.SetSockOptInt(opt, v) +} + +// GetSockOptInt implements tcpip.Endpoint. +func (e *endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { + switch opt { + case tcpip.ReceiveQueueSizeOption: + v := 0 + e.rcvMu.Lock() + if !e.rcvList.Empty() { + p := e.rcvList.Front() + v = p.data.Data().Size() + } + e.rcvMu.Unlock() + return v, nil + + default: + return e.net.GetSockOptInt(opt) + } +} + +// GetSockOpt implements tcpip.Endpoint. +func (e *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { + return e.net.GetSockOpt(opt) +} + +func send4(s *stack.Stack, ctx *network.WriteContext, ident uint16, data *buffer.View, maxHeaderLength uint16) tcpip.Error { + if data.Size() < header.ICMPv4MinimumSize { + return &tcpip.ErrInvalidEndpointState{} + } + + pkt := ctx.TryNewPacketBuffer(header.ICMPv4MinimumSize+int(maxHeaderLength), buffer.Buffer{}) + if pkt == nil { + return &tcpip.ErrWouldBlock{} + } + defer pkt.DecRef() + + icmpv4 := header.ICMPv4(pkt.TransportHeader().Push(header.ICMPv4MinimumSize)) + pkt.TransportProtocolNumber = header.ICMPv4ProtocolNumber + copy(icmpv4, data.AsSlice()) + // Set the ident to the user-specified port. Sequence number should + // already be set by the user. + icmpv4.SetIdent(ident) + data.TrimFront(header.ICMPv4MinimumSize) + + // Linux performs these basic checks. + if icmpv4.Type() != header.ICMPv4Echo || icmpv4.Code() != 0 { + return &tcpip.ErrInvalidEndpointState{} + } + + icmpv4.SetChecksum(0) + icmpv4.SetChecksum(^checksum.Checksum(icmpv4, checksum.Checksum(data.AsSlice(), 0))) + pkt.Data().AppendView(data.Clone()) + + // Because this icmp endpoint is implemented in the transport layer, we can + // only increment the 'stack-wide' stats but we can't increment the + // 'per-NetworkEndpoint' stats. + stats := s.Stats().ICMP.V4.PacketsSent + + if err := ctx.WritePacket(pkt, false /* headerIncluded */); err != nil { + stats.Dropped.Increment() + return err + } + + stats.EchoRequest.Increment() + return nil +} + +func send6(s *stack.Stack, ctx *network.WriteContext, ident uint16, data *buffer.View, src, dst tcpip.Address, maxHeaderLength uint16) tcpip.Error { + if data.Size() < header.ICMPv6EchoMinimumSize { + return &tcpip.ErrInvalidEndpointState{} + } + + pkt := ctx.TryNewPacketBuffer(header.ICMPv6MinimumSize+int(maxHeaderLength), buffer.Buffer{}) + if pkt == nil { + return &tcpip.ErrWouldBlock{} + } + defer pkt.DecRef() + + icmpv6 := header.ICMPv6(pkt.TransportHeader().Push(header.ICMPv6MinimumSize)) + pkt.TransportProtocolNumber = header.ICMPv6ProtocolNumber + copy(icmpv6, data.AsSlice()) + // Set the ident. Sequence number is provided by the user. + icmpv6.SetIdent(ident) + data.TrimFront(header.ICMPv6MinimumSize) + + if icmpv6.Type() != header.ICMPv6EchoRequest || icmpv6.Code() != 0 { + return &tcpip.ErrInvalidEndpointState{} + } + + pkt.Data().AppendView(data.Clone()) + pktData := pkt.Data() + icmpv6.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmpv6, + Src: src, + Dst: dst, + PayloadCsum: pktData.Checksum(), + PayloadLen: pktData.Size(), + })) + + // Because this icmp endpoint is implemented in the transport layer, we can + // only increment the 'stack-wide' stats but we can't increment the + // 'per-NetworkEndpoint' stats. + stats := s.Stats().ICMP.V6.PacketsSent + + if err := ctx.WritePacket(pkt, false /* headerIncluded */); err != nil { + stats.Dropped.Increment() + return err + } + + stats.EchoRequest.Increment() + return nil +} + +// Disconnect implements tcpip.Endpoint.Disconnect. +func (*endpoint) Disconnect() tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Connect connects the endpoint to its peer. Specifying a NIC is optional. +func (e *endpoint) Connect(addr tcpip.FullAddress) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + err := e.net.ConnectAndThen(addr, func(netProto tcpip.NetworkProtocolNumber, previousID, nextID stack.TransportEndpointID) tcpip.Error { + nextID.LocalPort = e.ident + + nextID, err := e.registerWithStack(netProto, nextID) + if err != nil { + return err + } + + e.ident = nextID.LocalPort + return nil + }) + if err != nil { + return err + } + + e.rcvMu.Lock() + e.rcvReady = true + e.rcvMu.Unlock() + + return nil +} + +// ConnectEndpoint is not supported. +func (*endpoint) ConnectEndpoint(tcpip.Endpoint) tcpip.Error { + return &tcpip.ErrInvalidEndpointState{} +} + +// Shutdown closes the read and/or write end of the endpoint connection +// to its peer. +func (e *endpoint) Shutdown(flags tcpip.ShutdownFlags) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: + return &tcpip.ErrNotConnected{} + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } + + if flags&tcpip.ShutdownWrite != 0 { + if err := e.net.Shutdown(); err != nil { + return err + } + } + + if flags&tcpip.ShutdownRead != 0 { + e.rcvMu.Lock() + wasClosed := e.rcvClosed + e.rcvClosed = true + e.rcvMu.Unlock() + + if !wasClosed { + e.waiterQueue.Notify(waiter.ReadableEvents) + } + } + + return nil +} + +// Listen is not supported by UDP, it just fails. +func (*endpoint) Listen(int) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Accept is not supported by UDP, it just fails. +func (*endpoint) Accept(*tcpip.FullAddress) (tcpip.Endpoint, *waiter.Queue, tcpip.Error) { + return nil, nil, &tcpip.ErrNotSupported{} +} + +func (e *endpoint) registerWithStack(netProto tcpip.NetworkProtocolNumber, id stack.TransportEndpointID) (stack.TransportEndpointID, tcpip.Error) { + bindToDevice := tcpip.NICID(e.ops.GetBindToDevice()) + if id.LocalPort != 0 { + // The endpoint already has a local port, just attempt to + // register it. + return id, e.stack.RegisterTransportEndpoint([]tcpip.NetworkProtocolNumber{netProto}, e.transProto, id, e, ports.Flags{}, bindToDevice) + } + + // We need to find a port for the endpoint. + _, err := e.stack.PickEphemeralPort(e.stack.SecureRNG(), func(p uint16) (bool, tcpip.Error) { + id.LocalPort = p + err := e.stack.RegisterTransportEndpoint([]tcpip.NetworkProtocolNumber{netProto}, e.transProto, id, e, ports.Flags{}, bindToDevice) + switch err.(type) { + case nil: + return true, nil + case *tcpip.ErrPortInUse: + return false, nil + default: + return false, err + } + }) + + return id, err +} + +func (e *endpoint) bindLocked(addr tcpip.FullAddress) tcpip.Error { + // Don't allow binding once endpoint is not in the initial state + // anymore. + if e.net.State() != transport.DatagramEndpointStateInitial { + return &tcpip.ErrInvalidEndpointState{} + } + + err := e.net.BindAndThen(addr, func(boundNetProto tcpip.NetworkProtocolNumber, boundAddr tcpip.Address) tcpip.Error { + id := stack.TransportEndpointID{ + LocalPort: addr.Port, + LocalAddress: addr.Addr, + } + id, err := e.registerWithStack(boundNetProto, id) + if err != nil { + return err + } + + e.ident = id.LocalPort + return nil + }) + if err != nil { + return err + } + + e.rcvMu.Lock() + e.rcvReady = true + e.rcvMu.Unlock() + + return nil +} + +func (e *endpoint) isBroadcastOrMulticast(nicID tcpip.NICID, addr tcpip.Address) bool { + return addr == header.IPv4Broadcast || + header.IsV4MulticastAddress(addr) || + header.IsV6MulticastAddress(addr) || + e.stack.IsSubnetBroadcast(nicID, e.net.NetProto(), addr) +} + +// Bind binds the endpoint to a specific local address and port. +// Specifying a NIC is optional. +func (e *endpoint) Bind(addr tcpip.FullAddress) tcpip.Error { + if addr.Addr.BitLen() != 0 && e.isBroadcastOrMulticast(addr.NIC, addr.Addr) { + return &tcpip.ErrBadLocalAddress{} + } + + e.mu.Lock() + defer e.mu.Unlock() + + return e.bindLocked(addr) +} + +// GetLocalAddress returns the address to which the endpoint is bound. +func (e *endpoint) GetLocalAddress() (tcpip.FullAddress, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + addr := e.net.GetLocalAddress() + addr.Port = e.ident + return addr, nil +} + +// GetRemoteAddress returns the address to which the endpoint is connected. +func (e *endpoint) GetRemoteAddress() (tcpip.FullAddress, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + if addr, connected := e.net.GetRemoteAddress(); connected { + return addr, nil + } + + return tcpip.FullAddress{}, &tcpip.ErrNotConnected{} +} + +// Readiness returns the current readiness of the endpoint. For example, if +// waiter.EventIn is set, the endpoint is immediately readable. +func (e *endpoint) Readiness(mask waiter.EventMask) waiter.EventMask { + var result waiter.EventMask + + if e.net.HasSendSpace() { + result |= waiter.WritableEvents & mask + } + + // Determine if the endpoint is readable if requested. + if (mask & waiter.ReadableEvents) != 0 { + e.rcvMu.Lock() + if !e.rcvList.Empty() || e.rcvClosed { + result |= waiter.ReadableEvents + } + e.rcvMu.Unlock() + } + + return result +} + +// HandlePacket is called by the stack when new packets arrive to this transport +// endpoint. +func (e *endpoint) HandlePacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) { + // Only accept echo replies. + switch e.net.NetProto() { + case header.IPv4ProtocolNumber: + h := header.ICMPv4(pkt.TransportHeader().Slice()) + if len(h) < header.ICMPv4MinimumSize || h.Type() != header.ICMPv4EchoReply { + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.MalformedPacketsReceived.Increment() + return + } + case header.IPv6ProtocolNumber: + h := header.ICMPv6(pkt.TransportHeader().Slice()) + if len(h) < header.ICMPv6MinimumSize || h.Type() != header.ICMPv6EchoReply { + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.MalformedPacketsReceived.Increment() + return + } + } + + e.rcvMu.Lock() + + // Drop the packet if our buffer is currently full. + if !e.rcvReady || e.rcvClosed { + e.rcvMu.Unlock() + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.ClosedReceiver.Increment() + return + } + + rcvBufSize := e.ops.GetReceiveBufferSize() + if e.frozen || e.rcvBufSize >= int(rcvBufSize) { + e.rcvMu.Unlock() + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.ReceiveBufferOverflow.Increment() + return + } + + wasEmpty := e.rcvBufSize == 0 + + net := pkt.Network() + dstAddr := net.DestinationAddress() + // Push new packet into receive list and increment the buffer size. + packet := &icmpPacket{ + senderAddress: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: id.RemoteAddress, + }, + packetInfo: tcpip.IPPacketInfo{ + // Linux does not 'prepare' [1] in_pktinfo on socket buffers destined to + // ping sockets (unlike UDP/RAW sockets). However the interface index [2] + // and the Header Destination Address [3] are always filled. + // [1] https://github.com/torvalds/linux/blob/dcb85f85fa6/net/ipv4/ip_sockglue.c#L1392 + // [2] https://github.com/torvalds/linux/blob/dcb85f85fa6/net/ipv4/ip_input.c#L510 + // [3] https://github.com/torvalds/linux/blob/dcb85f85fa6/net/ipv4/ip_sockglue.c#L60 + NIC: pkt.NICID, + DestinationAddr: dstAddr, + }, + } + + // Save any useful information from the network header to the packet. + packet.tosOrTClass, _ = net.TOS() + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + packet.ttlOrHopLimit = header.IPv4(pkt.NetworkHeader().Slice()).TTL() + case header.IPv6ProtocolNumber: + packet.ttlOrHopLimit = header.IPv6(pkt.NetworkHeader().Slice()).HopLimit() + } + + // ICMP socket's data includes ICMP header but no others. Trim all other + // headers from the front of the packet. + pktBuf := pkt.ToBuffer() + pktBuf.TrimFront(int64(pkt.HeaderSize() - len(pkt.TransportHeader().Slice()))) + packet.data = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: pktBuf}) + + e.rcvList.PushBack(packet) + e.rcvBufSize += packet.data.Data().Size() + + packet.receivedAt = e.stack.Clock().Now() + + e.rcvMu.Unlock() + e.stats.PacketsReceived.Increment() + // Notify any waiters that there's data to be read now. + if wasEmpty { + e.waiterQueue.Notify(waiter.ReadableEvents) + } +} + +// HandleError implements stack.TransportEndpoint. +func (*endpoint) HandleError(stack.TransportError, *stack.PacketBuffer) {} + +// State implements tcpip.Endpoint.State. The ICMP endpoint currently doesn't +// expose internal socket state. +func (e *endpoint) State() uint32 { + return uint32(e.net.State()) +} + +// Info returns a copy of the endpoint info. +func (e *endpoint) Info() tcpip.EndpointInfo { + e.mu.RLock() + defer e.mu.RUnlock() + ret := e.net.Info() + ret.ID.LocalPort = e.ident + return &ret +} + +// Stats returns a pointer to the endpoint stats. +func (e *endpoint) Stats() tcpip.EndpointStats { + return &e.stats +} + +// Wait implements stack.TransportEndpoint.Wait. +func (*endpoint) Wait() {} + +// LastError implements tcpip.Endpoint.LastError. +func (*endpoint) LastError() tcpip.Error { + return nil +} + +// SocketOptions implements tcpip.Endpoint.SocketOptions. +func (e *endpoint) SocketOptions() *tcpip.SocketOptions { + return &e.ops +} + +// freeze prevents any more packets from being delivered to the endpoint. +func (e *endpoint) freeze() { + e.mu.Lock() + e.frozen = true + e.mu.Unlock() +} + +// thaw unfreezes a previously frozen endpoint using endpoint.freeze() allows +// new packets to be delivered again. +func (e *endpoint) thaw() { + e.mu.Lock() + e.frozen = false + e.mu.Unlock() +} diff --git a/pkg/tcpip/transport/icmp/endpoint_state.go b/pkg/tcpip/transport/icmp/endpoint_state.go new file mode 100644 index 0000000..122f9d6 --- /dev/null +++ b/pkg/tcpip/transport/icmp/endpoint_state.go @@ -0,0 +1,92 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package icmp + +import ( + "context" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" +) + +// saveReceivedAt is invoked by stateify. +func (p *icmpPacket) saveReceivedAt() int64 { + return p.receivedAt.UnixNano() +} + +// loadReceivedAt is invoked by stateify. +func (p *icmpPacket) loadReceivedAt(_ context.Context, nsec int64) { + p.receivedAt = time.Unix(0, nsec) +} + +// afterLoad is invoked by stateify. +func (e *endpoint) afterLoad(ctx context.Context) { + if e.stack.IsSaveRestoreEnabled() { + e.stack.RegisterRestoredEndpoint(e) + } else { + stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) + } +} + +// beforeSave is invoked by stateify. +func (e *endpoint) beforeSave() { + e.freeze() + e.stack.RegisterResumableEndpoint(e) +} + +// Restore implements tcpip.RestoredEndpoint.Restore. +func (e *endpoint) Restore(s *stack.Stack) { + if err := e.net.Resume(s); err != nil { + log.Warningf("Closing the ICMP endpoint as it cannot be restored, err: %v", err) + e.Close() + return + } + + e.thaw() + if e.stack.IsSaveRestoreEnabled() { + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + return + } + + e.stack = s + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + + e.mu.Lock() + defer e.mu.Unlock() + + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + var err tcpip.Error + info := e.net.Info() + info.ID.LocalPort = e.ident + info.ID, err = e.registerWithStack(info.NetProto, info.ID) + if err != nil { + panic(fmt.Sprintf("e.registerWithStack(%d, %#v): %s", info.NetProto, info.ID, err)) + } + e.ident = info.ID.LocalPort + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } +} + +// Resume implements tcpip.ResumableEndpoint.Resume. +func (e *endpoint) Resume() { + e.thaw() +} diff --git a/pkg/tcpip/transport/icmp/icmp_packet_list.go b/pkg/tcpip/transport/icmp/icmp_packet_list.go new file mode 100644 index 0000000..59de994 --- /dev/null +++ b/pkg/tcpip/transport/icmp/icmp_packet_list.go @@ -0,0 +1,239 @@ +package icmp + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type icmpPacketElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (icmpPacketElementMapper) linkerFor(elem *icmpPacket) *icmpPacket { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type icmpPacketList struct { + head *icmpPacket + tail *icmpPacket +} + +// Reset resets list l to the empty state. +func (l *icmpPacketList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *icmpPacketList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *icmpPacketList) Front() *icmpPacket { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *icmpPacketList) Back() *icmpPacket { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *icmpPacketList) Len() (count int) { + for e := l.Front(); e != nil; e = (icmpPacketElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *icmpPacketList) PushFront(e *icmpPacket) { + linker := icmpPacketElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + icmpPacketElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *icmpPacketList) PushFrontList(m *icmpPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + icmpPacketElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + icmpPacketElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *icmpPacketList) PushBack(e *icmpPacket) { + linker := icmpPacketElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + icmpPacketElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *icmpPacketList) PushBackList(m *icmpPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + icmpPacketElementMapper{}.linkerFor(l.tail).SetNext(m.head) + icmpPacketElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *icmpPacketList) InsertAfter(b, e *icmpPacket) { + bLinker := icmpPacketElementMapper{}.linkerFor(b) + eLinker := icmpPacketElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + icmpPacketElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *icmpPacketList) InsertBefore(a, e *icmpPacket) { + aLinker := icmpPacketElementMapper{}.linkerFor(a) + eLinker := icmpPacketElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + icmpPacketElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *icmpPacketList) Remove(e *icmpPacket) { + linker := icmpPacketElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + icmpPacketElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + icmpPacketElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type icmpPacketEntry struct { + next *icmpPacket + prev *icmpPacket +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *icmpPacketEntry) Next() *icmpPacket { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *icmpPacketEntry) Prev() *icmpPacket { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *icmpPacketEntry) SetNext(elem *icmpPacket) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *icmpPacketEntry) SetPrev(elem *icmpPacket) { + e.prev = elem +} diff --git a/pkg/tcpip/transport/icmp/icmp_state_autogen.go b/pkg/tcpip/transport/icmp/icmp_state_autogen.go new file mode 100644 index 0000000..33af387 --- /dev/null +++ b/pkg/tcpip/transport/icmp/icmp_state_autogen.go @@ -0,0 +1,204 @@ +// automatically generated by stateify. + +package icmp + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *icmpPacket) StateTypeName() string { + return "pkg/tcpip/transport/icmp.icmpPacket" +} + +func (p *icmpPacket) StateFields() []string { + return []string{ + "icmpPacketEntry", + "senderAddress", + "packetInfo", + "data", + "receivedAt", + "tosOrTClass", + "ttlOrHopLimit", + } +} + +func (p *icmpPacket) beforeSave() {} + +// +checklocksignore +func (p *icmpPacket) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + var receivedAtValue int64 + receivedAtValue = p.saveReceivedAt() + stateSinkObject.SaveValue(4, receivedAtValue) + stateSinkObject.Save(0, &p.icmpPacketEntry) + stateSinkObject.Save(1, &p.senderAddress) + stateSinkObject.Save(2, &p.packetInfo) + stateSinkObject.Save(3, &p.data) + stateSinkObject.Save(5, &p.tosOrTClass) + stateSinkObject.Save(6, &p.ttlOrHopLimit) +} + +func (p *icmpPacket) afterLoad(context.Context) {} + +// +checklocksignore +func (p *icmpPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.icmpPacketEntry) + stateSourceObject.Load(1, &p.senderAddress) + stateSourceObject.Load(2, &p.packetInfo) + stateSourceObject.Load(3, &p.data) + stateSourceObject.Load(5, &p.tosOrTClass) + stateSourceObject.Load(6, &p.ttlOrHopLimit) + stateSourceObject.LoadValue(4, new(int64), func(y any) { p.loadReceivedAt(ctx, y.(int64)) }) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/transport/icmp.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "DefaultSocketOptionsHandler", + "stack", + "transProto", + "waiterQueue", + "net", + "stats", + "ops", + "rcvReady", + "rcvList", + "rcvBufSize", + "rcvClosed", + "frozen", + "ident", + } +} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.DefaultSocketOptionsHandler) + stateSinkObject.Save(1, &e.stack) + stateSinkObject.Save(2, &e.transProto) + stateSinkObject.Save(3, &e.waiterQueue) + stateSinkObject.Save(4, &e.net) + stateSinkObject.Save(5, &e.stats) + stateSinkObject.Save(6, &e.ops) + stateSinkObject.Save(7, &e.rcvReady) + stateSinkObject.Save(8, &e.rcvList) + stateSinkObject.Save(9, &e.rcvBufSize) + stateSinkObject.Save(10, &e.rcvClosed) + stateSinkObject.Save(11, &e.frozen) + stateSinkObject.Save(12, &e.ident) +} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.DefaultSocketOptionsHandler) + stateSourceObject.Load(1, &e.stack) + stateSourceObject.Load(2, &e.transProto) + stateSourceObject.Load(3, &e.waiterQueue) + stateSourceObject.Load(4, &e.net) + stateSourceObject.Load(5, &e.stats) + stateSourceObject.Load(6, &e.ops) + stateSourceObject.Load(7, &e.rcvReady) + stateSourceObject.Load(8, &e.rcvList) + stateSourceObject.Load(9, &e.rcvBufSize) + stateSourceObject.Load(10, &e.rcvClosed) + stateSourceObject.Load(11, &e.frozen) + stateSourceObject.Load(12, &e.ident) + stateSourceObject.AfterLoad(func() { e.afterLoad(ctx) }) +} + +func (l *icmpPacketList) StateTypeName() string { + return "pkg/tcpip/transport/icmp.icmpPacketList" +} + +func (l *icmpPacketList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *icmpPacketList) beforeSave() {} + +// +checklocksignore +func (l *icmpPacketList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *icmpPacketList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *icmpPacketList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *icmpPacketEntry) StateTypeName() string { + return "pkg/tcpip/transport/icmp.icmpPacketEntry" +} + +func (e *icmpPacketEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *icmpPacketEntry) beforeSave() {} + +// +checklocksignore +func (e *icmpPacketEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *icmpPacketEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *icmpPacketEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func (p *protocol) StateTypeName() string { + return "pkg/tcpip/transport/icmp.protocol" +} + +func (p *protocol) StateFields() []string { + return []string{ + "stack", + "number", + } +} + +func (p *protocol) beforeSave() {} + +// +checklocksignore +func (p *protocol) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.stack) + stateSinkObject.Save(1, &p.number) +} + +func (p *protocol) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.stack) + stateSourceObject.Load(1, &p.number) +} + +func init() { + state.Register((*icmpPacket)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*icmpPacketList)(nil)) + state.Register((*icmpPacketEntry)(nil)) + state.Register((*protocol)(nil)) +} diff --git a/pkg/tcpip/transport/icmp/protocol.go b/pkg/tcpip/transport/icmp/protocol.go new file mode 100644 index 0000000..b96ab4b --- /dev/null +++ b/pkg/tcpip/transport/icmp/protocol.go @@ -0,0 +1,150 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package icmp contains the implementation of the ICMP and IPv6-ICMP transport +// protocols for use in ping. +package icmp + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport/raw" + "github.com/sagernet/gvisor/pkg/waiter" +) + +const ( + // ProtocolNumber4 is the ICMP protocol number. + ProtocolNumber4 = header.ICMPv4ProtocolNumber + + // ProtocolNumber6 is the IPv6-ICMP protocol number. + ProtocolNumber6 = header.ICMPv6ProtocolNumber +) + +// protocol implements stack.TransportProtocol. +// +// +stateify savable +type protocol struct { + stack *stack.Stack + + number tcpip.TransportProtocolNumber +} + +// Number returns the ICMP protocol number. +func (p *protocol) Number() tcpip.TransportProtocolNumber { + return p.number +} + +func (p *protocol) netProto() tcpip.NetworkProtocolNumber { + switch p.number { + case ProtocolNumber4: + return header.IPv4ProtocolNumber + case ProtocolNumber6: + return header.IPv6ProtocolNumber + } + panic(fmt.Sprint("unknown protocol number: ", p.number)) +} + +// NewEndpoint creates a new icmp endpoint. It implements +// stack.TransportProtocol.NewEndpoint. +func (p *protocol) NewEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + if netProto != p.netProto() { + return nil, &tcpip.ErrUnknownProtocol{} + } + return newEndpoint(p.stack, netProto, p.number, waiterQueue) +} + +// NewRawEndpoint creates a new raw icmp endpoint. It implements +// stack.TransportProtocol.NewRawEndpoint. +func (p *protocol) NewRawEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + if netProto != p.netProto() { + return nil, &tcpip.ErrUnknownProtocol{} + } + return raw.NewEndpoint(p.stack, netProto, p.number, waiterQueue) +} + +// MinimumPacketSize returns the minimum valid icmp packet size. +func (p *protocol) MinimumPacketSize() int { + switch p.number { + case ProtocolNumber4: + return header.ICMPv4MinimumSize + case ProtocolNumber6: + return header.ICMPv6MinimumSize + } + panic(fmt.Sprint("unknown protocol number: ", p.number)) +} + +// ParsePorts in case of ICMP sets src to 0, dst to ICMP ID, and err to nil. +func (p *protocol) ParsePorts(v []byte) (src, dst uint16, err tcpip.Error) { + switch p.number { + case ProtocolNumber4: + hdr := header.ICMPv4(v) + return 0, hdr.Ident(), nil + case ProtocolNumber6: + hdr := header.ICMPv6(v) + return 0, hdr.Ident(), nil + } + panic(fmt.Sprint("unknown protocol number: ", p.number)) +} + +// HandleUnknownDestinationPacket handles packets targeted at this protocol but +// that don't match any existing endpoint. +func (*protocol) HandleUnknownDestinationPacket(stack.TransportEndpointID, *stack.PacketBuffer) stack.UnknownDestinationPacketDisposition { + return stack.UnknownDestinationPacketHandled +} + +// SetOption implements stack.TransportProtocol.SetOption. +func (*protocol) SetOption(tcpip.SettableTransportProtocolOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// Option implements stack.TransportProtocol.Option. +func (*protocol) Option(tcpip.GettableTransportProtocolOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// Close implements stack.TransportProtocol.Close. +func (*protocol) Close() {} + +// Wait implements stack.TransportProtocol.Wait. +func (*protocol) Wait() {} + +// Pause implements stack.TransportProtocol.Pause. +func (*protocol) Pause() {} + +// Resume implements stack.TransportProtocol.Resume. +func (*protocol) Resume() {} + +// Restore implements stack.TransportProtocol.Restore. +func (*protocol) Restore() {} + +// Parse implements stack.TransportProtocol.Parse. +func (*protocol) Parse(pkt *stack.PacketBuffer) bool { + // Right now, the Parse() method is tied to enabled protocols passed into + // stack.New. This works for UDP and TCP, but we handle ICMP traffic even + // when netstack users don't pass ICMP as a supported protocol. + return false +} + +// NewProtocol4 returns an ICMPv4 transport protocol. +func NewProtocol4(s *stack.Stack) stack.TransportProtocol { + return &protocol{stack: s, number: ProtocolNumber4} +} + +// NewProtocol6 returns an ICMPv6 transport protocol. +func NewProtocol6(s *stack.Stack) stack.TransportProtocol { + return &protocol{stack: s, number: ProtocolNumber6} +} diff --git a/pkg/tcpip/transport/internal/network/endpoint.go b/pkg/tcpip/transport/internal/network/endpoint.go new file mode 100644 index 0000000..da60506 --- /dev/null +++ b/pkg/tcpip/transport/internal/network/endpoint.go @@ -0,0 +1,1065 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package network provides facilities to support tcpip.Endpoints that operate +// at the network layer or above. +package network + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// Endpoint is a datagram-based endpoint. It only supports sending datagrams to +// a peer. +// +// +stateify savable +type Endpoint struct { + // The following fields must only be set once then never changed. + stack *stack.Stack + ops *tcpip.SocketOptions + netProto tcpip.NetworkProtocolNumber + transProto tcpip.TransportProtocolNumber + waiterQueue *waiter.Queue + + mu sync.RWMutex `state:"nosave"` + // +checklocks:mu + wasBound bool + // owner is the owner of transmitted packets. + // + // +checklocks:mu + owner tcpip.PacketOwner + // +checklocks:mu + writeShutdown bool + // +checklocks:mu + effectiveNetProto tcpip.NetworkProtocolNumber + // +checklocks:mu + connectedRoute *stack.Route `state:"nosave"` + // +checklocks:mu + multicastMemberships map[multicastMembership]struct{} + // +checklocks:mu + ipv4TTL uint8 + // +checklocks:mu + ipv6HopLimit int16 + // TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6. + // +checklocks:mu + multicastTTL uint8 + // TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6. + // +checklocks:mu + multicastAddr tcpip.Address + // TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6. + // +checklocks:mu + multicastNICID tcpip.NICID + // +checklocks:mu + ipv4TOS uint8 + // +checklocks:mu + ipv6TClass uint8 + + // Lock ordering: mu > infoMu. + infoMu sync.RWMutex `state:"nosave"` + // info has a dedicated mutex so that we can avoid lock ordering violations + // when reading the endpoint's info. If we used mu, we need to guarantee + // that any lock taken while mu is held is not held when calling Info() + // which is not true as of writing (we hold mu while registering transport + // endpoints (taking the transport demuxer lock but we also hold the demuxer + // lock when delivering packets/errors to endpoints). + // + // Writes must be performed through setInfo. + // + // +checklocks:infoMu + info stack.TransportEndpointInfo + + // state holds a transport.DatagramBasedEndpointState. + // + // state must be accessed with atomics so that we can avoid lock ordering + // violations when reading the state. If we used mu, we need to guarantee + // that any lock taken while mu is held is not held when calling State() + // which is not true as of writing (we hold mu while registering transport + // endpoints (taking the transport demuxer lock but we also hold the demuxer + // lock when delivering packets/errors to endpoints). + // + // Writes must be performed through setEndpointState. + state atomicbitops.Uint32 + + // Callers should not attempt to obtain sendBufferSizeInUseMu while holding + // another lock on Endpoint. + sendBufferSizeInUseMu sync.RWMutex `state:"nosave"` + // sendBufferSizeInUse keeps track of the bytes in use by in-flight packets. + // + // +checklocks:sendBufferSizeInUseMu + sendBufferSizeInUse int64 `state:"nosave"` +} + +// +stateify savable +type multicastMembership struct { + nicID tcpip.NICID + multicastAddr tcpip.Address +} + +// Init initializes the endpoint. +func (e *Endpoint) Init(s *stack.Stack, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, ops *tcpip.SocketOptions, waiterQueue *waiter.Queue) { + e.mu.Lock() + defer e.mu.Unlock() + if e.multicastMemberships != nil { + panic(fmt.Sprintf("endpoint is already initialized; got e.multicastMemberships = %#v, want = nil", e.multicastMemberships)) + } + + switch netProto { + case header.IPv4ProtocolNumber, header.IPv6ProtocolNumber: + default: + panic(fmt.Sprintf("invalid protocol number = %d", netProto)) + } + + e.stack = s + e.ops = ops + e.netProto = netProto + e.transProto = transProto + e.waiterQueue = waiterQueue + e.infoMu.Lock() + e.info = stack.TransportEndpointInfo{ + NetProto: netProto, + TransProto: transProto, + } + e.infoMu.Unlock() + e.effectiveNetProto = netProto + e.ipv4TTL = tcpip.UseDefaultIPv4TTL + e.ipv6HopLimit = tcpip.UseDefaultIPv6HopLimit + + // Linux defaults to TTL=1. + e.multicastTTL = 1 + e.multicastMemberships = make(map[multicastMembership]struct{}) + e.setEndpointState(transport.DatagramEndpointStateInitial) +} + +// NetProto returns the network protocol the endpoint was initialized with. +func (e *Endpoint) NetProto() tcpip.NetworkProtocolNumber { + return e.netProto +} + +// setEndpointState sets the state of the endpoint. +// +// e.mu must be held to synchronize changes to state with the rest of the +// endpoint. +// +// +checklocks:e.mu +func (e *Endpoint) setEndpointState(state transport.DatagramEndpointState) { + e.state.Store(uint32(state)) +} + +// State returns the state of the endpoint. +func (e *Endpoint) State() transport.DatagramEndpointState { + return transport.DatagramEndpointState(e.state.Load()) +} + +// Close cleans the endpoint's resources and leaves the endpoint in a closed +// state. +func (e *Endpoint) Close() { + e.mu.Lock() + defer e.mu.Unlock() + + if e.State() == transport.DatagramEndpointStateClosed { + return + } + + for mem := range e.multicastMemberships { + e.stack.LeaveGroup(e.netProto, mem.nicID, mem.multicastAddr) + } + e.multicastMemberships = nil + + if e.connectedRoute != nil { + e.connectedRoute.Release() + e.connectedRoute = nil + } + + e.setEndpointState(transport.DatagramEndpointStateClosed) +} + +// SetOwner sets the owner of transmitted packets. +func (e *Endpoint) SetOwner(owner tcpip.PacketOwner) { + e.mu.Lock() + defer e.mu.Unlock() + e.owner = owner +} + +// +checklocksread:e.mu +func (e *Endpoint) calculateTTL(route *stack.Route) uint8 { + remoteAddress := route.RemoteAddress() + if header.IsV4MulticastAddress(remoteAddress) || header.IsV6MulticastAddress(remoteAddress) { + return e.multicastTTL + } + + switch netProto := route.NetProto(); netProto { + case header.IPv4ProtocolNumber: + if e.ipv4TTL == 0 { + return route.DefaultTTL() + } + return e.ipv4TTL + case header.IPv6ProtocolNumber: + if e.ipv6HopLimit == -1 { + return route.DefaultTTL() + } + return uint8(e.ipv6HopLimit) + default: + panic(fmt.Sprintf("invalid protocol number = %d", netProto)) + } +} + +// WriteContext holds the context for a write. +type WriteContext struct { + e *Endpoint + route *stack.Route + ttl uint8 + tos uint8 +} + +func (c *WriteContext) MTU() uint32 { + return c.route.MTU() +} + +// Release releases held resources. +func (c *WriteContext) Release() { + c.route.Release() + *c = WriteContext{} +} + +// WritePacketInfo is the properties of a packet that may be written. +type WritePacketInfo struct { + NetProto tcpip.NetworkProtocolNumber + LocalAddress, RemoteAddress tcpip.Address + MaxHeaderLength uint16 + RequiresTXTransportChecksum bool +} + +// PacketInfo returns the properties of a packet that will be written. +func (c *WriteContext) PacketInfo() WritePacketInfo { + return WritePacketInfo{ + NetProto: c.route.NetProto(), + LocalAddress: c.route.LocalAddress(), + RemoteAddress: c.route.RemoteAddress(), + MaxHeaderLength: c.route.MaxHeaderLength(), + RequiresTXTransportChecksum: c.route.RequiresTXTransportChecksum(), + } +} + +// TryNewPacketBuffer returns a new packet buffer iff the endpoint's send buffer +// is not full. +// +// If this method returns nil, the caller should wait for the endpoint to become +// writable. +func (c *WriteContext) TryNewPacketBuffer(reserveHdrBytes int, data buffer.Buffer) *stack.PacketBuffer { + e := c.e + + e.sendBufferSizeInUseMu.Lock() + defer e.sendBufferSizeInUseMu.Unlock() + + if !e.hasSendSpaceRLocked() { + return nil + } + return c.newPacketBufferLocked(reserveHdrBytes, data) +} + +// TryNewPacketBufferFromPayloader returns a new packet buffer iff the endpoint's send buffer +// is not full. Otherwise, data from `payloader` isn't read. +// +// If this method returns nil, the caller should wait for the endpoint to become +// writable. +func (c *WriteContext) TryNewPacketBufferFromPayloader(reserveHdrBytes int, payloader tcpip.Payloader) *stack.PacketBuffer { + e := c.e + + e.sendBufferSizeInUseMu.Lock() + defer e.sendBufferSizeInUseMu.Unlock() + + if !e.hasSendSpaceRLocked() { + return nil + } + var data buffer.Buffer + if _, err := data.WriteFromReader(payloader, int64(payloader.Len())); err != nil { + data.Release() + return nil + } + return c.newPacketBufferLocked(reserveHdrBytes, data) +} + +// +checklocks:c.e.sendBufferSizeInUseMu +func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Buffer) *stack.PacketBuffer { + e := c.e + // Note that we allow oversubscription - if there is any space at all in the + // send buffer, we accept the full packet which may be larger than the space + // available. This is because if the endpoint reports that it is writable, + // a write operation should succeed. + // + // This matches Linux behaviour: + // https://github.com/torvalds/linux/blob/38d741cb70b/include/net/sock.h#L2519 + // https://github.com/torvalds/linux/blob/38d741cb70b/net/core/sock.c#L2588 + var expOptVal uint16 + if nic, err := c.e.stack.GetNICByID(c.route.OutgoingNIC()); err == nil && nic.GetExperimentIPOptionEnabled() { + expOptVal = c.e.ops.GetExperimentOptionValue() + } + if c.route.NetProto() == header.IPv6ProtocolNumber && expOptVal != 0 { + reserveHdrBytes += header.IPv6ExperimentHdrLength + } + pktSize := int64(reserveHdrBytes) + int64(data.Size()) + e.sendBufferSizeInUse += pktSize + + return stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: reserveHdrBytes, + Payload: data, + OnRelease: func() { + e.sendBufferSizeInUseMu.Lock() + if got := e.sendBufferSizeInUse; got < pktSize { + e.sendBufferSizeInUseMu.Unlock() + panic(fmt.Sprintf("e.sendBufferSizeInUse=(%d) < pktSize(=%d)", got, pktSize)) + } + e.sendBufferSizeInUse -= pktSize + signal := e.hasSendSpaceRLocked() + e.sendBufferSizeInUseMu.Unlock() + + // Let waiters know if we now have space in the send buffer. + if signal { + e.waiterQueue.Notify(waiter.WritableEvents) + } + }, + }) +} + +// WritePacket attempts to write the packet. +func (c *WriteContext) WritePacket(pkt *stack.PacketBuffer, headerIncluded bool) tcpip.Error { + c.e.mu.RLock() + pkt.Owner = c.e.owner + c.e.mu.RUnlock() + + if headerIncluded { + return c.route.WriteHeaderIncludedPacket(pkt) + } + + var expOptVal uint16 + if nic, err := c.e.stack.GetNICByID(c.route.OutgoingNIC()); err == nil && nic.GetExperimentIPOptionEnabled() { + expOptVal = c.e.ops.GetExperimentOptionValue() + } + + err := c.route.WritePacket(stack.NetworkHeaderParams{ + Protocol: c.e.transProto, + TTL: c.ttl, + TOS: c.tos, + ExperimentOptionValue: expOptVal, + }, pkt) + + if _, ok := err.(*tcpip.ErrNoBufferSpace); ok { + var recvErr bool + switch netProto := c.route.NetProto(); netProto { + case header.IPv4ProtocolNumber: + recvErr = c.e.ops.GetIPv4RecvError() + case header.IPv6ProtocolNumber: + recvErr = c.e.ops.GetIPv6RecvError() + default: + panic(fmt.Sprintf("unhandled network protocol number = %d", netProto)) + } + + // Linux only returns ENOBUFS to the caller if IP{,V6}_RECVERR is set. + // + // https://github.com/torvalds/linux/blob/3e71713c9e75c/net/ipv4/udp.c#L969 + // https://github.com/torvalds/linux/blob/3e71713c9e75c/net/ipv6/udp.c#L1260 + if !recvErr { + err = nil + } + } + + return err +} + +// MaybeSignalWritable signals waiters with writable events if the send buffer +// has space. +func (e *Endpoint) MaybeSignalWritable() { + e.sendBufferSizeInUseMu.RLock() + signal := e.hasSendSpaceRLocked() + e.sendBufferSizeInUseMu.RUnlock() + + if signal { + e.waiterQueue.Notify(waiter.WritableEvents) + } +} + +// HasSendSpace returns whether or not the send buffer has space. +func (e *Endpoint) HasSendSpace() bool { + e.sendBufferSizeInUseMu.RLock() + defer e.sendBufferSizeInUseMu.RUnlock() + return e.hasSendSpaceRLocked() +} + +// +checklocksread:e.sendBufferSizeInUseMu +func (e *Endpoint) hasSendSpaceRLocked() bool { + return e.ops.GetSendBufferSize() > e.sendBufferSizeInUse +} + +// AcquireContextForWrite acquires a WriteContext. +func (e *Endpoint) AcquireContextForWrite(opts tcpip.WriteOptions) (WriteContext, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + // MSG_MORE is unimplemented. This also means that MSG_EOR is a no-op. + if opts.More { + return WriteContext{}, &tcpip.ErrInvalidOptionValue{} + } + + if e.State() == transport.DatagramEndpointStateClosed { + return WriteContext{}, &tcpip.ErrInvalidEndpointState{} + } + + if e.writeShutdown { + return WriteContext{}, &tcpip.ErrClosedForSend{} + } + + ipv6PktInfoValid := e.effectiveNetProto == header.IPv6ProtocolNumber && opts.ControlMessages.HasIPv6PacketInfo + + route := e.connectedRoute + to := opts.To + info := e.Info() + switch { + case to == nil: + // If the user doesn't specify a destination, they should have + // connected to another address. + if e.State() != transport.DatagramEndpointStateConnected { + return WriteContext{}, &tcpip.ErrDestinationRequired{} + } + + if !ipv6PktInfoValid { + route.Acquire() + break + } + + // We are connected and the caller did not specify the destination but + // we have an IPv6 packet info structure which may change our local + // interface/address used to send the packet so we need to construct + // a new route instead of using the connected route. + // + // Construct a destination matching the remote the endpoint is connected + // to. + to = &tcpip.FullAddress{ + // RegisterNICID is set when the endpoint is connected. It is usually + // only set for link-local addresses or multicast addresses if the + // multicast interface was specified (see e.multicastNICID, + // e.connectRouteRLocked and e.ConnectAndThen). + NIC: info.RegisterNICID, + Addr: info.ID.RemoteAddress, + } + fallthrough + default: + // Reject destination address if it goes through a different + // NIC than the endpoint was bound to. + nicID := to.NIC + if nicID == 0 { + nicID = tcpip.NICID(e.ops.GetBindToDevice()) + } + + var localAddr tcpip.Address + if ipv6PktInfoValid { + // Uphold strong-host semantics since (as of writing) the stack follows + // the strong host model. + + pktInfoNICID := opts.ControlMessages.IPv6PacketInfo.NIC + pktInfoAddr := opts.ControlMessages.IPv6PacketInfo.Addr + + if pktInfoNICID != 0 { + // If we are bound to an interface or specified the destination + // interface (usually when using link-local addresses), make sure the + // interface matches the specified local interface. + if nicID != 0 && nicID != pktInfoNICID { + return WriteContext{}, &tcpip.ErrHostUnreachable{} + } + + // If a local address is not specified, then we need to make sure the + // bound address belongs to the specified local interface. + if pktInfoAddr.BitLen() == 0 { + // If the bound interface is different from the specified local + // interface, the bound address obviously does not belong to the + // specified local interface. + // + // The bound interface is usually only set for link-local addresses. + if info.BindNICID != 0 && info.BindNICID != pktInfoNICID { + return WriteContext{}, &tcpip.ErrHostUnreachable{} + } + if info.ID.LocalAddress.BitLen() != 0 && e.stack.CheckLocalAddress(pktInfoNICID, header.IPv6ProtocolNumber, info.ID.LocalAddress) == 0 { + return WriteContext{}, &tcpip.ErrBadLocalAddress{} + } + } + + nicID = pktInfoNICID + } + + if pktInfoAddr.BitLen() != 0 { + // The local address must belong to the stack. If an outgoing interface + // is specified as a result of binding the endpoint to a device, or + // specifying the outgoing interface in the destination address/pkt info + // structure, the address must belong to that interface. + if e.stack.CheckLocalAddress(nicID, header.IPv6ProtocolNumber, pktInfoAddr) == 0 { + return WriteContext{}, &tcpip.ErrBadLocalAddress{} + } + + localAddr = pktInfoAddr + } + } else { + if info.BindNICID != 0 { + if nicID != 0 && nicID != info.BindNICID { + return WriteContext{}, &tcpip.ErrHostUnreachable{} + } + + nicID = info.BindNICID + } + if nicID == 0 { + nicID = info.RegisterNICID + } + } + + dst, netProto, err := e.checkV4Mapped(*to, false /* bind */) + if err != nil { + return WriteContext{}, err + } + + route, _, err = e.connectRouteRLocked(nicID, localAddr, dst, netProto) + if err != nil { + return WriteContext{}, err + } + } + + if !e.ops.GetBroadcast() && route.IsOutboundBroadcast() { + route.Release() + return WriteContext{}, &tcpip.ErrBroadcastDisabled{} + } + + var tos uint8 + var ttl uint8 + switch netProto := route.NetProto(); netProto { + case header.IPv4ProtocolNumber: + tos = e.ipv4TOS + if opts.ControlMessages.HasTTL { + ttl = opts.ControlMessages.TTL + } else { + ttl = e.calculateTTL(route) + } + case header.IPv6ProtocolNumber: + tos = e.ipv6TClass + if opts.ControlMessages.HasHopLimit { + ttl = opts.ControlMessages.HopLimit + } else { + ttl = e.calculateTTL(route) + } + default: + panic(fmt.Sprintf("invalid protocol number = %d", netProto)) + } + + return WriteContext{ + e: e, + route: route, + ttl: ttl, + tos: tos, + }, nil +} + +// Disconnect disconnects the endpoint from its peer. +func (e *Endpoint) Disconnect() { + e.mu.Lock() + defer e.mu.Unlock() + + if e.State() != transport.DatagramEndpointStateConnected { + return + } + + info := e.Info() + // Exclude ephemerally bound endpoints. + if e.wasBound { + info.ID = stack.TransportEndpointID{ + LocalAddress: info.BindAddr, + } + e.setEndpointState(transport.DatagramEndpointStateBound) + } else { + info.ID = stack.TransportEndpointID{} + e.setEndpointState(transport.DatagramEndpointStateInitial) + } + e.setInfo(info) + + e.connectedRoute.Release() + e.connectedRoute = nil +} + +// connectRouteRLocked establishes a route to the specified interface or the +// configured multicast interface if no interface is specified and the +// specified address is a multicast address. +// +// +checklocksread:e.mu +func (e *Endpoint) connectRouteRLocked(nicID tcpip.NICID, localAddr tcpip.Address, addr tcpip.FullAddress, netProto tcpip.NetworkProtocolNumber) (*stack.Route, tcpip.NICID, tcpip.Error) { + if localAddr.BitLen() == 0 { + localAddr = e.Info().ID.LocalAddress + if e.isBroadcastOrMulticast(nicID, netProto, localAddr) { + // A packet can only originate from a unicast address (i.e., an interface). + localAddr = tcpip.Address{} + } + + if header.IsV4MulticastAddress(addr.Addr) || header.IsV6MulticastAddress(addr.Addr) { + if nicID == 0 { + nicID = e.multicastNICID + } + if localAddr == (tcpip.Address{}) && nicID == 0 { + localAddr = e.multicastAddr + } + } + } + + // Find a route to the desired destination. + r, err := e.stack.FindRoute(nicID, localAddr, addr.Addr, netProto, e.ops.GetMulticastLoop()) + if err != nil { + return nil, 0, err + } + return r, nicID, nil +} + +// Connect connects the endpoint to the address. +func (e *Endpoint) Connect(addr tcpip.FullAddress) tcpip.Error { + return e.ConnectAndThen(addr, func(_ tcpip.NetworkProtocolNumber, _, _ stack.TransportEndpointID) tcpip.Error { + return nil + }) +} + +// ConnectAndThen connects the endpoint to the address and then calls the +// provided function. +// +// If the function returns an error, the endpoint's state does not change. The +// function will be called with the network protocol used to connect to the peer +// and the source and destination addresses that will be used to send traffic to +// the peer. +func (e *Endpoint) ConnectAndThen(addr tcpip.FullAddress, f func(netProto tcpip.NetworkProtocolNumber, previousID, nextID stack.TransportEndpointID) tcpip.Error) tcpip.Error { + addr.Port = 0 + + e.mu.Lock() + defer e.mu.Unlock() + + info := e.Info() + nicID := addr.NIC + switch e.State() { + case transport.DatagramEndpointStateInitial: + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + if info.BindNICID == 0 { + break + } + + if nicID != 0 && nicID != info.BindNICID { + return &tcpip.ErrInvalidEndpointState{} + } + + nicID = info.BindNICID + default: + return &tcpip.ErrInvalidEndpointState{} + } + + addr, netProto, err := e.checkV4Mapped(addr, false /* bind */) + if err != nil { + return err + } + + r, nicID, err := e.connectRouteRLocked(nicID, tcpip.Address{}, addr, netProto) + if err != nil { + return err + } + + id := stack.TransportEndpointID{ + LocalAddress: info.ID.LocalAddress, + RemoteAddress: r.RemoteAddress(), + } + if e.State() == transport.DatagramEndpointStateInitial { + id.LocalAddress = r.LocalAddress() + } + + if err := f(r.NetProto(), info.ID, id); err != nil { + r.Release() + return err + } + + if e.connectedRoute != nil { + // If the endpoint was previously connected then release any previous route. + e.connectedRoute.Release() + } + e.connectedRoute = r + info.ID = id + info.RegisterNICID = nicID + e.setInfo(info) + e.effectiveNetProto = netProto + e.setEndpointState(transport.DatagramEndpointStateConnected) + return nil +} + +// Shutdown shutsdown the endpoint. +func (e *Endpoint) Shutdown() tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + switch state := e.State(); state { + case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: + return &tcpip.ErrNotConnected{} + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + e.writeShutdown = true + return nil + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } +} + +// checkV4MappedRLocked determines the effective network protocol and converts +// addr to its canonical form. +func (e *Endpoint) checkV4Mapped(addr tcpip.FullAddress, bind bool) (tcpip.FullAddress, tcpip.NetworkProtocolNumber, tcpip.Error) { + info := e.Info() + unwrapped, netProto, err := info.AddrNetProtoLocked(addr, e.ops.GetV6Only(), bind) + if err != nil { + return tcpip.FullAddress{}, 0, err + } + return unwrapped, netProto, nil +} + +func (e *Endpoint) isBroadcastOrMulticast(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, addr tcpip.Address) bool { + return addr == header.IPv4Broadcast || header.IsV4MulticastAddress(addr) || header.IsV6MulticastAddress(addr) || e.stack.IsSubnetBroadcast(nicID, netProto, addr) +} + +// Bind binds the endpoint to the address. +func (e *Endpoint) Bind(addr tcpip.FullAddress) tcpip.Error { + return e.BindAndThen(addr, func(tcpip.NetworkProtocolNumber, tcpip.Address) tcpip.Error { + return nil + }) +} + +// BindAndThen binds the endpoint to the address and then calls the provided +// function. +// +// If the function returns an error, the endpoint's state does not change. The +// function will be called with the bound network protocol and address. +func (e *Endpoint) BindAndThen(addr tcpip.FullAddress, f func(tcpip.NetworkProtocolNumber, tcpip.Address) tcpip.Error) tcpip.Error { + addr.Port = 0 + + e.mu.Lock() + defer e.mu.Unlock() + + // Don't allow binding once endpoint is not in the initial state + // anymore. + if e.State() != transport.DatagramEndpointStateInitial { + return &tcpip.ErrInvalidEndpointState{} + } + + addr, netProto, err := e.checkV4Mapped(addr, true /* bind */) + if err != nil { + return err + } + + nicID := addr.NIC + if addr.Addr.BitLen() != 0 && !e.isBroadcastOrMulticast(addr.NIC, netProto, addr.Addr) { + nicID = e.stack.CheckLocalAddress(nicID, netProto, addr.Addr) + if nicID == 0 { + return &tcpip.ErrBadLocalAddress{} + } + } + + if err := f(netProto, addr.Addr); err != nil { + return err + } + + e.wasBound = true + + info := e.Info() + info.ID = stack.TransportEndpointID{ + LocalAddress: addr.Addr, + } + info.BindNICID = addr.NIC + info.RegisterNICID = nicID + info.BindAddr = addr.Addr + e.setInfo(info) + e.effectiveNetProto = netProto + e.setEndpointState(transport.DatagramEndpointStateBound) + return nil +} + +// WasBound returns true iff the endpoint was ever bound. +func (e *Endpoint) WasBound() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.wasBound +} + +// GetLocalAddress returns the address that the endpoint is bound to. +func (e *Endpoint) GetLocalAddress() tcpip.FullAddress { + e.mu.RLock() + defer e.mu.RUnlock() + + info := e.Info() + addr := info.BindAddr + if e.State() == transport.DatagramEndpointStateConnected { + addr = e.connectedRoute.LocalAddress() + } + + return tcpip.FullAddress{ + NIC: info.RegisterNICID, + Addr: addr, + } +} + +// GetRemoteAddress returns the address that the endpoint is connected to. +func (e *Endpoint) GetRemoteAddress() (tcpip.FullAddress, bool) { + e.mu.RLock() + defer e.mu.RUnlock() + + if e.State() != transport.DatagramEndpointStateConnected { + return tcpip.FullAddress{}, false + } + + return tcpip.FullAddress{ + Addr: e.connectedRoute.RemoteAddress(), + NIC: e.Info().RegisterNICID, + }, true +} + +// SetSockOptInt sets the socket option. +func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { + switch opt { + case tcpip.MTUDiscoverOption: + // Return not supported if the value is not disabling path + // MTU discovery. + if tcpip.PMTUDStrategy(v) != tcpip.PMTUDiscoveryDont { + return &tcpip.ErrNotSupported{} + } + + case tcpip.MulticastTTLOption: + e.mu.Lock() + e.multicastTTL = uint8(v) + e.mu.Unlock() + + case tcpip.IPv4TTLOption: + e.mu.Lock() + e.ipv4TTL = uint8(v) + e.mu.Unlock() + + case tcpip.IPv6HopLimitOption: + e.mu.Lock() + e.ipv6HopLimit = int16(v) + e.mu.Unlock() + + case tcpip.IPv4TOSOption: + e.mu.Lock() + e.ipv4TOS = uint8(v) + e.mu.Unlock() + + case tcpip.IPv6TrafficClassOption: + e.mu.Lock() + e.ipv6TClass = uint8(v) + e.mu.Unlock() + } + + return nil +} + +// GetSockOptInt returns the socket option. +func (e *Endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { + switch opt { + case tcpip.MTUDiscoverOption: + // The only supported setting is path MTU discovery disabled. + return int(tcpip.PMTUDiscoveryDont), nil + + case tcpip.MulticastTTLOption: + e.mu.Lock() + v := int(e.multicastTTL) + e.mu.Unlock() + return v, nil + + case tcpip.IPv4TTLOption: + e.mu.Lock() + v := int(e.ipv4TTL) + e.mu.Unlock() + return v, nil + + case tcpip.IPv6HopLimitOption: + e.mu.Lock() + v := int(e.ipv6HopLimit) + e.mu.Unlock() + return v, nil + + case tcpip.IPv4TOSOption: + e.mu.RLock() + v := int(e.ipv4TOS) + e.mu.RUnlock() + return v, nil + + case tcpip.IPv6TrafficClassOption: + e.mu.RLock() + v := int(e.ipv6TClass) + e.mu.RUnlock() + return v, nil + + default: + return -1, &tcpip.ErrUnknownProtocolOption{} + } +} + +// SetSockOpt sets the socket option. +func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { + switch v := opt.(type) { + case *tcpip.MulticastInterfaceOption: + e.mu.Lock() + defer e.mu.Unlock() + + fa := tcpip.FullAddress{Addr: v.InterfaceAddr} + fa, netProto, err := e.checkV4Mapped(fa, true /* bind */) + if err != nil { + return err + } + nic := v.NIC + addr := fa.Addr + + if nic == 0 && addr == (tcpip.Address{}) { + e.multicastAddr = tcpip.Address{} + e.multicastNICID = 0 + break + } + + if nic != 0 { + if !e.stack.CheckNIC(nic) { + return &tcpip.ErrBadLocalAddress{} + } + } else { + nic = e.stack.CheckLocalAddress(0, netProto, addr) + if nic == 0 { + return &tcpip.ErrBadLocalAddress{} + } + } + + if info := e.Info(); info.BindNICID != 0 && info.BindNICID != nic { + return &tcpip.ErrInvalidEndpointState{} + } + + e.multicastNICID = nic + e.multicastAddr = addr + + case *tcpip.AddMembershipOption: + if !(header.IsV4MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv4ProtocolNumber) && !(header.IsV6MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv6ProtocolNumber) { + return &tcpip.ErrInvalidOptionValue{} + } + + nicID := v.NIC + + if v.InterfaceAddr.Unspecified() { + if nicID == 0 { + if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, e.netProto, false /* multicastLoop */); err == nil { + nicID = r.NICID() + r.Release() + } + } + } else { + nicID = e.stack.CheckLocalAddress(nicID, e.netProto, v.InterfaceAddr) + } + if nicID == 0 { + return &tcpip.ErrUnknownDevice{} + } + + memToInsert := multicastMembership{nicID: nicID, multicastAddr: v.MulticastAddr} + + e.mu.Lock() + defer e.mu.Unlock() + + if _, ok := e.multicastMemberships[memToInsert]; ok { + return &tcpip.ErrPortInUse{} + } + + if err := e.stack.JoinGroup(e.netProto, nicID, v.MulticastAddr); err != nil { + return err + } + + e.multicastMemberships[memToInsert] = struct{}{} + + case *tcpip.RemoveMembershipOption: + if !(header.IsV4MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv4ProtocolNumber) && !(header.IsV6MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv6ProtocolNumber) { + return &tcpip.ErrInvalidOptionValue{} + } + + nicID := v.NIC + if v.InterfaceAddr.Unspecified() { + if nicID == 0 { + if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, e.netProto, false /* multicastLoop */); err == nil { + nicID = r.NICID() + r.Release() + } + } + } else { + nicID = e.stack.CheckLocalAddress(nicID, e.netProto, v.InterfaceAddr) + } + if nicID == 0 { + return &tcpip.ErrUnknownDevice{} + } + + memToRemove := multicastMembership{nicID: nicID, multicastAddr: v.MulticastAddr} + + e.mu.Lock() + defer e.mu.Unlock() + + if _, ok := e.multicastMemberships[memToRemove]; !ok { + return &tcpip.ErrBadLocalAddress{} + } + + if err := e.stack.LeaveGroup(e.netProto, nicID, v.MulticastAddr); err != nil { + return err + } + + delete(e.multicastMemberships, memToRemove) + + case *tcpip.SocketDetachFilterOption: + return nil + } + return nil +} + +// GetSockOpt returns the socket option. +func (e *Endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { + switch o := opt.(type) { + case *tcpip.MulticastInterfaceOption: + e.mu.Lock() + *o = tcpip.MulticastInterfaceOption{ + NIC: e.multicastNICID, + InterfaceAddr: e.multicastAddr, + } + e.mu.Unlock() + + default: + return &tcpip.ErrUnknownProtocolOption{} + } + return nil +} + +// Info returns a copy of the endpoint info. +func (e *Endpoint) Info() stack.TransportEndpointInfo { + e.infoMu.RLock() + defer e.infoMu.RUnlock() + return e.info +} + +// setInfo sets the endpoint's info. +// +// e.mu must be held to synchronize changes to info with the rest of the +// endpoint. +// +// +checklocks:e.mu +func (e *Endpoint) setInfo(info stack.TransportEndpointInfo) { + e.infoMu.Lock() + defer e.infoMu.Unlock() + e.info = info +} diff --git a/pkg/tcpip/transport/internal/network/endpoint_state.go b/pkg/tcpip/transport/internal/network/endpoint_state.go new file mode 100644 index 0000000..c63c8b3 --- /dev/null +++ b/pkg/tcpip/transport/internal/network/endpoint_state.go @@ -0,0 +1,62 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package network + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" +) + +// Resume implements tcpip.ResumableEndpoint.Resume. +func (e *Endpoint) Resume(s *stack.Stack) error { + e.mu.Lock() + defer e.mu.Unlock() + + e.stack = s + for m := range e.multicastMemberships { + if err := e.stack.JoinGroup(e.netProto, m.nicID, m.multicastAddr); err != nil { + return fmt.Errorf("e.stack.JoinGroup(%d, %d, %s): %s", e.netProto, m.nicID, m.multicastAddr, err) + } + } + + info := e.Info() + + switch state := e.State(); state { + case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: + case transport.DatagramEndpointStateBound: + if info.ID.LocalAddress.BitLen() != 0 && !e.isBroadcastOrMulticast(info.RegisterNICID, e.effectiveNetProto, info.ID.LocalAddress) { + if e.stack.CheckLocalAddress(info.RegisterNICID, e.effectiveNetProto, info.ID.LocalAddress) == 0 { + return fmt.Errorf("got e.stack.CheckLocalAddress(%d, %d, %s) = 0, want != 0", info.RegisterNICID, e.effectiveNetProto, info.ID.LocalAddress) + } + } + case transport.DatagramEndpointStateConnected: + var err tcpip.Error + multicastLoop := e.ops.GetMulticastLoop() + // Release the connectedRoute if present. + if e.connectedRoute != nil { + e.connectedRoute.Release() + } + e.connectedRoute, err = e.stack.FindRoute(info.RegisterNICID, info.ID.LocalAddress, info.ID.RemoteAddress, e.effectiveNetProto, multicastLoop) + if err != nil { + return fmt.Errorf("e.stack.FindRoute(%d, %s, %s, %d, %t): %s", info.RegisterNICID, info.ID.LocalAddress, info.ID.RemoteAddress, e.effectiveNetProto, multicastLoop, err) + } + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } + return nil +} diff --git a/pkg/tcpip/transport/internal/network/network_state_autogen.go b/pkg/tcpip/transport/internal/network/network_state_autogen.go new file mode 100644 index 0000000..11234a9 --- /dev/null +++ b/pkg/tcpip/transport/internal/network/network_state_autogen.go @@ -0,0 +1,121 @@ +// automatically generated by stateify. + +package network + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/transport/internal/network.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "stack", + "ops", + "netProto", + "transProto", + "waiterQueue", + "wasBound", + "owner", + "writeShutdown", + "effectiveNetProto", + "multicastMemberships", + "ipv4TTL", + "ipv6HopLimit", + "multicastTTL", + "multicastAddr", + "multicastNICID", + "ipv4TOS", + "ipv6TClass", + "info", + "state", + } +} + +func (e *Endpoint) beforeSave() {} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.stack) + stateSinkObject.Save(1, &e.ops) + stateSinkObject.Save(2, &e.netProto) + stateSinkObject.Save(3, &e.transProto) + stateSinkObject.Save(4, &e.waiterQueue) + stateSinkObject.Save(5, &e.wasBound) + stateSinkObject.Save(6, &e.owner) + stateSinkObject.Save(7, &e.writeShutdown) + stateSinkObject.Save(8, &e.effectiveNetProto) + stateSinkObject.Save(9, &e.multicastMemberships) + stateSinkObject.Save(10, &e.ipv4TTL) + stateSinkObject.Save(11, &e.ipv6HopLimit) + stateSinkObject.Save(12, &e.multicastTTL) + stateSinkObject.Save(13, &e.multicastAddr) + stateSinkObject.Save(14, &e.multicastNICID) + stateSinkObject.Save(15, &e.ipv4TOS) + stateSinkObject.Save(16, &e.ipv6TClass) + stateSinkObject.Save(17, &e.info) + stateSinkObject.Save(18, &e.state) +} + +func (e *Endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.stack) + stateSourceObject.Load(1, &e.ops) + stateSourceObject.Load(2, &e.netProto) + stateSourceObject.Load(3, &e.transProto) + stateSourceObject.Load(4, &e.waiterQueue) + stateSourceObject.Load(5, &e.wasBound) + stateSourceObject.Load(6, &e.owner) + stateSourceObject.Load(7, &e.writeShutdown) + stateSourceObject.Load(8, &e.effectiveNetProto) + stateSourceObject.Load(9, &e.multicastMemberships) + stateSourceObject.Load(10, &e.ipv4TTL) + stateSourceObject.Load(11, &e.ipv6HopLimit) + stateSourceObject.Load(12, &e.multicastTTL) + stateSourceObject.Load(13, &e.multicastAddr) + stateSourceObject.Load(14, &e.multicastNICID) + stateSourceObject.Load(15, &e.ipv4TOS) + stateSourceObject.Load(16, &e.ipv6TClass) + stateSourceObject.Load(17, &e.info) + stateSourceObject.Load(18, &e.state) +} + +func (m *multicastMembership) StateTypeName() string { + return "pkg/tcpip/transport/internal/network.multicastMembership" +} + +func (m *multicastMembership) StateFields() []string { + return []string{ + "nicID", + "multicastAddr", + } +} + +func (m *multicastMembership) beforeSave() {} + +// +checklocksignore +func (m *multicastMembership) StateSave(stateSinkObject state.Sink) { + m.beforeSave() + stateSinkObject.Save(0, &m.nicID) + stateSinkObject.Save(1, &m.multicastAddr) +} + +func (m *multicastMembership) afterLoad(context.Context) {} + +// +checklocksignore +func (m *multicastMembership) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &m.nicID) + stateSourceObject.Load(1, &m.multicastAddr) +} + +func init() { + state.Register((*Endpoint)(nil)) + state.Register((*multicastMembership)(nil)) +} diff --git a/pkg/tcpip/transport/internal/noop/endpoint.go b/pkg/tcpip/transport/internal/noop/endpoint.go new file mode 100644 index 0000000..df736b5 --- /dev/null +++ b/pkg/tcpip/transport/internal/noop/endpoint.go @@ -0,0 +1,177 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package noop contains an endpoint that implements all tcpip.Endpoint +// functions as noops. +package noop + +import ( + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// endpoint can be created, but all interactions have no effect or +// return errors. +// +// +stateify savable +type endpoint struct { + tcpip.DefaultSocketOptionsHandler + ops tcpip.SocketOptions +} + +// New returns an initialized noop endpoint. +func New(stk *stack.Stack) tcpip.Endpoint { + // ep.ops must be in a valid, initialized state for callers of + // ep.SocketOptions. + var ep endpoint + ep.ops.InitHandler(&ep, stk, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + return &ep +} + +// Abort implements stack.TransportEndpoint.Abort. +func (*endpoint) Abort() { + // No-op. +} + +// Close implements tcpip.Endpoint.Close. +func (*endpoint) Close() { + // No-op. +} + +// ModerateRecvBuf implements tcpip.Endpoint.ModerateRecvBuf. +func (*endpoint) ModerateRecvBuf(int) { + // No-op. +} + +func (*endpoint) SetOwner(tcpip.PacketOwner) { + // No-op. +} + +// Read implements tcpip.Endpoint.Read. +func (*endpoint) Read(io.Writer, tcpip.ReadOptions) (tcpip.ReadResult, tcpip.Error) { + return tcpip.ReadResult{}, &tcpip.ErrNotPermitted{} +} + +// Write implements tcpip.Endpoint.Write. +func (*endpoint) Write(tcpip.Payloader, tcpip.WriteOptions) (int64, tcpip.Error) { + return 0, &tcpip.ErrNotPermitted{} +} + +// Disconnect implements tcpip.Endpoint.Disconnect. +func (*endpoint) Disconnect() tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Connect implements tcpip.Endpoint.Connect. +func (*endpoint) Connect(tcpip.FullAddress) tcpip.Error { + return &tcpip.ErrNotPermitted{} +} + +// Shutdown implements tcpip.Endpoint.Shutdown. +func (*endpoint) Shutdown(tcpip.ShutdownFlags) tcpip.Error { + return &tcpip.ErrNotPermitted{} +} + +// Listen implements tcpip.Endpoint.Listen. +func (*endpoint) Listen(int) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Accept implements tcpip.Endpoint.Accept. +func (*endpoint) Accept(*tcpip.FullAddress) (tcpip.Endpoint, *waiter.Queue, tcpip.Error) { + return nil, nil, &tcpip.ErrNotSupported{} +} + +// Bind implements tcpip.Endpoint.Bind. +func (*endpoint) Bind(tcpip.FullAddress) tcpip.Error { + return &tcpip.ErrNotPermitted{} +} + +// GetLocalAddress implements tcpip.Endpoint.GetLocalAddress. +func (*endpoint) GetLocalAddress() (tcpip.FullAddress, tcpip.Error) { + return tcpip.FullAddress{}, &tcpip.ErrNotSupported{} +} + +// GetRemoteAddress implements tcpip.Endpoint.GetRemoteAddress. +func (*endpoint) GetRemoteAddress() (tcpip.FullAddress, tcpip.Error) { + return tcpip.FullAddress{}, &tcpip.ErrNotConnected{} +} + +// Readiness implements tcpip.Endpoint.Readiness. +func (*endpoint) Readiness(waiter.EventMask) waiter.EventMask { + return 0 +} + +// SetSockOpt implements tcpip.Endpoint.SetSockOpt. +func (*endpoint) SetSockOpt(tcpip.SettableSocketOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +func (*endpoint) SetSockOptInt(tcpip.SockOptInt, int) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// GetSockOpt implements tcpip.Endpoint.GetSockOpt. +func (*endpoint) GetSockOpt(tcpip.GettableSocketOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// GetSockOptInt implements tcpip.Endpoint.GetSockOptInt. +func (*endpoint) GetSockOptInt(tcpip.SockOptInt) (int, tcpip.Error) { + return 0, &tcpip.ErrUnknownProtocolOption{} +} + +// HandlePacket implements stack.RawTransportEndpoint.HandlePacket. +func (*endpoint) HandlePacket(pkt *stack.PacketBuffer) { + panic(fmt.Sprintf("unreachable: noop.endpoint should never be registered, but got packet: %+v", pkt)) +} + +// State implements socket.Socket.State. +func (*endpoint) State() uint32 { + return 0 +} + +// Wait implements stack.TransportEndpoint.Wait. +func (*endpoint) Wait() { + // No-op. +} + +// Release implements stack.TransportEndpoint.Release. +func (*endpoint) Release() { + // No-op. +} + +// LastError implements tcpip.Endpoint.LastError. +func (*endpoint) LastError() tcpip.Error { + return nil +} + +// SocketOptions implements tcpip.Endpoint.SocketOptions. +func (ep *endpoint) SocketOptions() *tcpip.SocketOptions { + return &ep.ops +} + +// Info implements tcpip.Endpoint.Info. +func (*endpoint) Info() tcpip.EndpointInfo { + return &stack.TransportEndpointInfo{} +} + +// Stats returns a pointer to the endpoint stats. +func (*endpoint) Stats() tcpip.EndpointStats { + return &tcpip.TransportEndpointStats{} +} diff --git a/pkg/tcpip/transport/internal/noop/noop_state_autogen.go b/pkg/tcpip/transport/internal/noop/noop_state_autogen.go new file mode 100644 index 0000000..3fb69f3 --- /dev/null +++ b/pkg/tcpip/transport/internal/noop/noop_state_autogen.go @@ -0,0 +1,41 @@ +// automatically generated by stateify. + +package noop + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (ep *endpoint) StateTypeName() string { + return "pkg/tcpip/transport/internal/noop.endpoint" +} + +func (ep *endpoint) StateFields() []string { + return []string{ + "DefaultSocketOptionsHandler", + "ops", + } +} + +func (ep *endpoint) beforeSave() {} + +// +checklocksignore +func (ep *endpoint) StateSave(stateSinkObject state.Sink) { + ep.beforeSave() + stateSinkObject.Save(0, &ep.DefaultSocketOptionsHandler) + stateSinkObject.Save(1, &ep.ops) +} + +func (ep *endpoint) afterLoad(context.Context) {} + +// +checklocksignore +func (ep *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ep.DefaultSocketOptionsHandler) + stateSourceObject.Load(1, &ep.ops) +} + +func init() { + state.Register((*endpoint)(nil)) +} diff --git a/pkg/tcpip/transport/packet/endpoint.go b/pkg/tcpip/transport/packet/endpoint.go new file mode 100644 index 0000000..1e311d3 --- /dev/null +++ b/pkg/tcpip/transport/packet/endpoint.go @@ -0,0 +1,623 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package packet provides the implementation of packet sockets (see +// packet(7)). Packet sockets allow applications to: +// +// - manually write and inspect link, network, and transport headers +// - receive all traffic of a given network protocol, or all protocols +// +// Packet sockets are similar to raw sockets, but provide even more power to +// users, letting them effectively talk directly to the network device. +// +// Packet sockets skip the input and output iptables chains. +package packet + +import ( + "io" + "math" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +type tpacketVersion int + +const ( + tpacketVersion1 tpacketVersion = iota + tpacketVersion2 +) + +var _ stack.MappablePacketEndpoint = (*endpoint)(nil) + +// +stateify savable +type packet struct { + packetEntry + // data holds the actual packet data, including any headers and payload. + data *stack.PacketBuffer + receivedAt time.Time `state:".(int64)"` + // senderAddr is the network address of the sender. + senderAddr tcpip.FullAddress + // packetInfo holds additional information like the protocol + // of the packet etc. + packetInfo tcpip.LinkPacketInfo +} + +// endpoint is the packet socket implementation of tcpip.Endpoint. It is legal +// to have goroutines make concurrent calls into the endpoint. +// +// Lock order: +// +// endpoint.mu +// endpoint.rcvMu +// endpoint.packetMmapMu +// +// +stateify savable +type endpoint struct { + tcpip.DefaultSocketOptionsHandler + + // The following fields are initialized at creation time and are + // immutable. + stack *stack.Stack + waiterQueue *waiter.Queue + cooked bool + ops tcpip.SocketOptions + stats tcpip.TransportEndpointStats + + // The following fields are used to manage the receive queue. + rcvMu rcvMutex `state:"nosave"` + // +checklocks:rcvMu + rcvList packetList + // +checklocks:rcvMu + rcvBufSize int + // +checklocks:rcvMu + rcvClosed bool + // +checklocks:rcvMu + rcvDisabled bool + + mu endpointRWMutex `state:"nosave"` + // +checklocks:mu + closed bool + // +checklocks:mu + boundNetProto tcpip.NetworkProtocolNumber + // +checklocks:mu + boundNIC tcpip.NICID + + lastErrorMu lastErrorMutex `state:"nosave"` + // +checklocks:lastErrorMu + lastError tcpip.Error + + packetMmapMu packetMmapRWMutex `state:"nosave"` + // +checklocks:packetMmapMu + packetMMapVersion tpacketVersion + // +checklocks:packetMmapMu + packetMMapReserve int + // +checklocks:packetMmapMu + packetMMapEp stack.PacketMMapEndpoint +} + +// NewEndpoint returns a new packet endpoint. +func NewEndpoint(s *stack.Stack, cooked bool, netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) tcpip.Endpoint { + ep := &endpoint{ + stack: s, + cooked: cooked, + boundNetProto: netProto, + waiterQueue: waiterQueue, + } + ep.ops.InitHandler(ep, ep.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + ep.ops.SetReceiveBufferSize(32*1024, false /* notify */) + + // Override with stack defaults. + var ss tcpip.SendBufferSizeOption + if err := s.Option(&ss); err == nil { + ep.ops.SetSendBufferSize(int64(ss.Default), false /* notify */) + } + + var rs tcpip.ReceiveBufferSizeOption + if err := s.Option(&rs); err == nil { + ep.ops.SetReceiveBufferSize(int64(rs.Default), false /* notify */) + } + + s.RegisterPacketEndpoint(0, netProto, ep) + + return ep +} + +// Abort implements stack.TransportEndpoint.Abort. +func (ep *endpoint) Abort() { + ep.Close() +} + +// Close implements tcpip.Endpoint.Close. +func (ep *endpoint) Close() { + ep.mu.Lock() + defer ep.mu.Unlock() + if ep.closed { + return + } + ep.stack.UnregisterPacketEndpoint(ep.boundNIC, ep.boundNetProto, ep) + + ep.packetMmapMu.Lock() + if ep.packetMMapEp != nil { + ep.packetMMapEp.Close() + ep.packetMMapEp = nil + } + ep.packetMmapMu.Unlock() + + ep.rcvMu.Lock() + defer ep.rcvMu.Unlock() + + // Clear the receive list. + ep.rcvClosed = true + ep.rcvBufSize = 0 + for !ep.rcvList.Empty() { + p := ep.rcvList.Front() + ep.rcvList.Remove(p) + p.data.DecRef() + } + + ep.closed = true + ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) +} + +// ModerateRecvBuf implements tcpip.Endpoint.ModerateRecvBuf. +func (*endpoint) ModerateRecvBuf(int) {} + +// Read implements tcpip.Endpoint.Read. +func (ep *endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult, tcpip.Error) { + ep.rcvMu.Lock() + + // If there's no data to read, return that read would block or that the + // endpoint is closed. + if ep.rcvList.Empty() { + var err tcpip.Error = &tcpip.ErrWouldBlock{} + if ep.rcvClosed { + ep.stats.ReadErrors.ReadClosed.Increment() + err = &tcpip.ErrClosedForReceive{} + } + ep.rcvMu.Unlock() + return tcpip.ReadResult{}, err + } + + packet := ep.rcvList.Front() + if !opts.Peek { + ep.rcvList.Remove(packet) + defer packet.data.DecRef() + ep.rcvBufSize -= packet.data.Size() + } + + ep.rcvMu.Unlock() + + res := tcpip.ReadResult{ + Total: packet.data.Size(), + ControlMessages: tcpip.ReceivableControlMessages{ + HasTimestamp: true, + Timestamp: packet.receivedAt, + }, + } + if opts.NeedRemoteAddr { + res.RemoteAddr = packet.senderAddr + } + if opts.NeedLinkPacketInfo { + res.LinkPacketInfo = packet.packetInfo + } + + n, err := packet.data.Data().ReadTo(dst, opts.Peek) + if n == 0 && err != nil { + return res, &tcpip.ErrBadBuffer{} + } + res.Count = n + return res, nil +} + +func (ep *endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + if !ep.stack.PacketEndpointWriteSupported() { + return 0, &tcpip.ErrNotSupported{} + } + + ep.mu.Lock() + closed := ep.closed + nicID := ep.boundNIC + proto := ep.boundNetProto + ep.mu.Unlock() + if closed { + return 0, &tcpip.ErrClosedForSend{} + } + + var remote tcpip.LinkAddress + if to := opts.To; to != nil { + remote = to.LinkAddr + + if n := to.NIC; n != 0 { + nicID = n + } + + if p := to.Port; p != 0 { + proto = tcpip.NetworkProtocolNumber(p) + } + } + + if nicID == 0 { + return 0, &tcpip.ErrInvalidOptionValue{} + } + + // Prevents giant buffer allocations. + if p.Len() > header.DatagramMaximumSize { + return 0, &tcpip.ErrMessageTooLong{} + } + + var payload buffer.Buffer + if _, err := payload.WriteFromReader(p, int64(p.Len())); err != nil { + return 0, &tcpip.ErrBadBuffer{} + } + payloadSz := payload.Size() + + if err := func() tcpip.Error { + if ep.cooked { + return ep.stack.WritePacketToRemote(nicID, remote, proto, payload) + } + return ep.stack.WriteRawPacket(nicID, proto, payload) + }(); err != nil { + return 0, err + } + return payloadSz, nil +} + +// Disconnect implements tcpip.Endpoint.Disconnect. Packet sockets cannot be +// disconnected, and this function always returns tpcip.ErrNotSupported. +func (*endpoint) Disconnect() tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Connect implements tcpip.Endpoint.Connect. Packet sockets cannot be +// connected, and this function always returns *tcpip.ErrNotSupported. +func (*endpoint) Connect(tcpip.FullAddress) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Shutdown implements tcpip.Endpoint.Shutdown. Packet sockets cannot be used +// with Shutdown, and this function always returns *tcpip.ErrNotSupported. +func (*endpoint) Shutdown(tcpip.ShutdownFlags) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Listen implements tcpip.Endpoint.Listen. Packet sockets cannot be used with +// Listen, and this function always returns *tcpip.ErrNotSupported. +func (*endpoint) Listen(int) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Accept implements tcpip.Endpoint.Accept. Packet sockets cannot be used with +// Accept, and this function always returns *tcpip.ErrNotSupported. +func (*endpoint) Accept(*tcpip.FullAddress) (tcpip.Endpoint, *waiter.Queue, tcpip.Error) { + return nil, nil, &tcpip.ErrNotSupported{} +} + +// Bind implements tcpip.Endpoint.Bind. +func (ep *endpoint) Bind(addr tcpip.FullAddress) tcpip.Error { + // "By default, all packets of the specified protocol type are passed + // to a packet socket. To get packets only from a specific interface + // use bind(2) specifying an address in a struct sockaddr_ll to bind + // the packet socket to an interface. Fields used for binding are + // sll_family (should be AF_PACKET), sll_protocol, and sll_ifindex." + // - packet(7). + + ep.mu.Lock() + defer ep.mu.Unlock() + + netProto := tcpip.NetworkProtocolNumber(addr.Port) + if netProto == 0 { + // Do not allow unbinding the network protocol. + netProto = ep.boundNetProto + } + + if ep.boundNIC == addr.NIC && ep.boundNetProto == netProto { + // Already bound to the requested NIC and network protocol. + return nil + } + + // TODO(https://gvisor.dev/issue/6618): Unregister after registering the new + // binding. + ep.stack.UnregisterPacketEndpoint(ep.boundNIC, ep.boundNetProto, ep) + ep.boundNIC = 0 + ep.boundNetProto = 0 + + // Bind endpoint to receive packets from specific interface. + if err := ep.stack.RegisterPacketEndpoint(addr.NIC, netProto, ep); err != nil { + return err + } + + ep.boundNIC = addr.NIC + ep.boundNetProto = netProto + return nil +} + +// GetLocalAddress implements tcpip.Endpoint.GetLocalAddress. +func (ep *endpoint) GetLocalAddress() (tcpip.FullAddress, tcpip.Error) { + ep.mu.RLock() + defer ep.mu.RUnlock() + + return tcpip.FullAddress{ + NIC: ep.boundNIC, + Port: uint16(ep.boundNetProto), + }, nil +} + +// GetRemoteAddress implements tcpip.Endpoint.GetRemoteAddress. +func (*endpoint) GetRemoteAddress() (tcpip.FullAddress, tcpip.Error) { + // Even a connected socket doesn't return a remote address. + return tcpip.FullAddress{}, &tcpip.ErrNotConnected{} +} + +// Readiness implements tcpip.Endpoint.Readiness. +func (ep *endpoint) Readiness(mask waiter.EventMask) waiter.EventMask { + // The endpoint is always writable. + result := waiter.WritableEvents & mask + + // Determine whether the endpoint is readable. + if (mask & waiter.ReadableEvents) != 0 { + ep.packetMmapMu.RLock() + if ep.packetMMapEp != nil { + result |= ep.packetMMapEp.Readiness(mask) + } + ep.packetMmapMu.RUnlock() + ep.rcvMu.Lock() + if !ep.rcvList.Empty() || ep.rcvClosed { + result |= waiter.ReadableEvents + } + ep.rcvMu.Unlock() + } + + return result +} + +// SetSockOpt implements tcpip.Endpoint.SetSockOpt. +func (ep *endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { + switch opt.(type) { + case *tcpip.SocketDetachFilterOption: + return nil + case *tcpip.TpacketReq: + ep.rcvMu.Lock() + defer ep.rcvMu.Unlock() + if !ep.rcvList.Empty() { + return &tcpip.ErrWouldBlock{} + } + return nil + + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// SetSockOptInt implements tcpip.Endpoint.SetSockOptInt. +func (ep *endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { + switch opt { + case tcpip.PacketMMapVersionOption: + ep.packetMmapMu.Lock() + defer ep.packetMmapMu.Unlock() + // We support up to TPACKET_V2. + version := tpacketVersion(v) + switch version { + case tpacketVersion1, tpacketVersion2: + if ep.packetMMapEp != nil { + return &tcpip.ErrEndpointBusy{} + } + ep.packetMMapVersion = version + return nil + default: + return &tcpip.ErrInvalidOptionValue{} + } + case tcpip.PacketMMapReserveOption: + ep.packetMmapMu.Lock() + defer ep.packetMmapMu.Unlock() + if ep.packetMMapEp != nil { + return &tcpip.ErrEndpointBusy{} + } + if uint32(v) > uint32(math.MaxInt32) { + return &tcpip.ErrInvalidOptionValue{} + } + ep.packetMMapReserve = v + return nil + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +func (ep *endpoint) LastError() tcpip.Error { + ep.lastErrorMu.Lock() + defer ep.lastErrorMu.Unlock() + + err := ep.lastError + ep.lastError = nil + return err +} + +// UpdateLastError implements tcpip.SocketOptionsHandler.UpdateLastError. +func (ep *endpoint) UpdateLastError(err tcpip.Error) { + ep.lastErrorMu.Lock() + ep.lastError = err + ep.lastErrorMu.Unlock() +} + +// GetSockOpt implements tcpip.Endpoint.GetSockOpt. +func (ep *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { + switch opt.(type) { + case *tcpip.TpacketStats: + ep.packetMmapMu.RLock() + defer ep.packetMmapMu.RUnlock() + if ep.packetMMapEp == nil { + return nil + } + *(opt.(*tcpip.TpacketStats)) = ep.packetMMapEp.Stats() + return nil + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// GetSockOptInt implements tcpip.Endpoint.GetSockOptInt. +func (ep *endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { + switch opt { + case tcpip.ReceiveQueueSizeOption: + v := 0 + ep.rcvMu.Lock() + if !ep.rcvList.Empty() { + p := ep.rcvList.Front() + v = p.data.Size() + } + ep.rcvMu.Unlock() + return v, nil + + default: + return -1, &tcpip.ErrUnknownProtocolOption{} + } +} + +// handlePacket implements stack.PacketEndpoint.HandlePacket +func (ep *endpoint) HandlePacket(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + ep.packetMmapMu.RLock() + if ep.packetMMapEp != nil { + if handled := ep.packetMMapEp.HandlePacket(nicID, netProto, pkt); handled { + ep.packetMmapMu.RUnlock() + return + } + } + ep.packetMmapMu.RUnlock() + + wasEmpty := ep.handlePacketInner(nicID, netProto, pkt) + + ep.stats.PacketsReceived.Increment() + // Notify waiters that there's data to be read. + if wasEmpty { + ep.waiterQueue.Notify(waiter.ReadableEvents) + } +} + +func (ep *endpoint) HandlePacketMMapCopy(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { + _ = ep.handlePacketInner(nicID, netProto, pkt) +} + +func (ep *endpoint) handlePacketInner(nicID tcpip.NICID, netProto tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) bool { + ep.rcvMu.Lock() + + // Drop the packet if our buffer is currently full. + if ep.rcvClosed { + ep.rcvMu.Unlock() + ep.stack.Stats().DroppedPackets.Increment() + ep.stats.ReceiveErrors.ClosedReceiver.Increment() + return false + } + + rcvBufSize := ep.ops.GetReceiveBufferSize() + if ep.rcvDisabled || ep.rcvBufSize >= int(rcvBufSize) { + ep.rcvMu.Unlock() + ep.stack.Stats().DroppedPackets.Increment() + ep.stats.ReceiveErrors.ReceiveBufferOverflow.Increment() + return false + } + + wasEmpty := ep.rcvBufSize == 0 + + rcvdPkt := packet{ + packetInfo: tcpip.LinkPacketInfo{ + Protocol: netProto, + PktType: pkt.PktType, + }, + senderAddr: tcpip.FullAddress{ + NIC: nicID, + }, + receivedAt: ep.stack.Clock().Now(), + } + + if len(pkt.LinkHeader().Slice()) != 0 { + hdr := header.Ethernet(pkt.LinkHeader().Slice()) + rcvdPkt.senderAddr.LinkAddr = hdr.SourceAddress() + } + + // Raw packet endpoints include link-headers in received packets. + pktBuf := pkt.ToBuffer() + if ep.cooked { + // Cooked packet endpoints don't include the link-headers in received + // packets. + pktBuf.TrimFront(int64(len(pkt.LinkHeader().Slice()) + len(pkt.VirtioNetHeader().Slice()))) + } + rcvdPkt.data = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: pktBuf}) + + ep.rcvList.PushBack(&rcvdPkt) + ep.rcvBufSize += rcvdPkt.data.Size() + ep.rcvMu.Unlock() + return wasEmpty +} + +// State implements socket.Socket.State. +func (*endpoint) State() uint32 { + return 0 +} + +// Info returns a copy of the endpoint info. +func (ep *endpoint) Info() tcpip.EndpointInfo { + ep.mu.RLock() + defer ep.mu.RUnlock() + return &stack.TransportEndpointInfo{NetProto: ep.boundNetProto} +} + +// Stats returns a pointer to the endpoint stats. +func (ep *endpoint) Stats() tcpip.EndpointStats { + return &ep.stats +} + +// SetOwner implements tcpip.Endpoint.SetOwner. +func (*endpoint) SetOwner(tcpip.PacketOwner) {} + +// SocketOptions implements tcpip.Endpoint.SocketOptions. +func (ep *endpoint) SocketOptions() *tcpip.SocketOptions { + return &ep.ops +} + +// GetPacketMMapOpts implements stack.MappablePacketEndpoint.GetPacketMMapOpts. +func (ep *endpoint) GetPacketMMapOpts(req *tcpip.TpacketReq, isRx bool) stack.PacketMMapOpts { + ep.packetMmapMu.Lock() + defer ep.packetMmapMu.Unlock() + + return stack.PacketMMapOpts{ + Req: req, + IsRx: isRx, + Cooked: ep.cooked, + Stack: ep.stack, + Wq: ep.waiterQueue, + PacketEndpoint: ep, + Version: int(ep.packetMMapVersion), + Reserve: uint32(ep.packetMMapReserve), + } +} + +// SetPacketMMapEndpoint implements +// stack.MappablePacketEndpoint.SetPacketMMapEndpoint. +func (ep *endpoint) SetPacketMMapEndpoint(m stack.PacketMMapEndpoint) { + ep.packetMmapMu.Lock() + defer ep.packetMmapMu.Unlock() + ep.packetMMapEp = m +} + +// GetPacketMMapEndpoint implements +// stack.MappablePacketEndpoint.GetPacketMMapEndpoint. +func (ep *endpoint) GetPacketMMapEndpoint() stack.PacketMMapEndpoint { + ep.packetMmapMu.RLock() + defer ep.packetMmapMu.RUnlock() + return ep.packetMMapEp +} diff --git a/pkg/tcpip/transport/packet/endpoint_mutex.go b/pkg/tcpip/transport/packet/endpoint_mutex.go new file mode 100644 index 0000000..685f878 --- /dev/null +++ b/pkg/tcpip/transport/packet/endpoint_mutex.go @@ -0,0 +1,96 @@ +package packet + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type endpointRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var endpointlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type endpointlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *endpointRWMutex) Lock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) { + locking.AddGLock(endpointprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *endpointRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) { + m.mu.Unlock() + locking.DelGLock(endpointprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *endpointRWMutex) RLock() { + locking.AddGLock(endpointprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *endpointRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(endpointprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *endpointRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *endpointRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *endpointRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var endpointprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func endpointinitLockNames() {} + +func init() { + endpointinitLockNames() + endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) +} diff --git a/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go b/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go new file mode 100644 index 0000000..c34ffc3 --- /dev/null +++ b/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go @@ -0,0 +1,64 @@ +package packet + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type rcvMutex struct { + mu sync.Mutex +} + +var rcvprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var rcvlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type rcvlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *rcvMutex) Lock() { + locking.AddGLock(rcvprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *rcvMutex) NestedLock(i rcvlockNameIndex) { + locking.AddGLock(rcvprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *rcvMutex) Unlock() { + locking.DelGLock(rcvprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *rcvMutex) NestedUnlock(i rcvlockNameIndex) { + locking.DelGLock(rcvprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func rcvinitLockNames() {} + +func init() { + rcvinitLockNames() + rcvprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvMutex{}), rcvlockNames) +} diff --git a/pkg/tcpip/transport/packet/endpoint_state.go b/pkg/tcpip/transport/packet/endpoint_state.go new file mode 100644 index 0000000..40751c8 --- /dev/null +++ b/pkg/tcpip/transport/packet/endpoint_state.go @@ -0,0 +1,74 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packet + +import ( + "context" + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// saveReceivedAt is invoked by stateify. +func (p *packet) saveReceivedAt() int64 { + return p.receivedAt.UnixNano() +} + +// loadReceivedAt is invoked by stateify. +func (p *packet) loadReceivedAt(_ context.Context, nsec int64) { + p.receivedAt = time.Unix(0, nsec) +} + +// beforeSave is invoked by stateify. +func (ep *endpoint) beforeSave() { + ep.rcvMu.Lock() + ep.rcvDisabled = true + ep.rcvMu.Unlock() + ep.stack.RegisterResumableEndpoint(ep) +} + +// afterLoad is invoked by stateify. +func (ep *endpoint) afterLoad(ctx context.Context) { + if !ep.stack.IsSaveRestoreEnabled() { + ep.mu.Lock() + ep.stack = stack.RestoreStackFromContext(ctx) + ep.mu.Unlock() + } + ep.stack.RegisterRestoredEndpoint(ep) +} + +// Restore implements tcpip.RestoredEndpoint.Restore. +func (ep *endpoint) Restore(_ *stack.Stack) { + ep.mu.Lock() + defer ep.mu.Unlock() + + ep.ops.InitHandler(ep, ep.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + if err := ep.stack.RegisterPacketEndpoint(ep.boundNIC, ep.boundNetProto, ep); err != nil { + panic(fmt.Sprintf("RegisterPacketEndpoint(%d, %d, _): %s", ep.boundNIC, ep.boundNetProto, err)) + } + + ep.rcvMu.Lock() + ep.rcvDisabled = false + ep.rcvMu.Unlock() +} + +// Resume implements tcpip.ResumableEndpoint.Resume. +func (ep *endpoint) Resume() { + ep.rcvMu.Lock() + defer ep.rcvMu.Unlock() + ep.rcvDisabled = false +} diff --git a/pkg/tcpip/transport/packet/last_error_mutex.go b/pkg/tcpip/transport/packet/last_error_mutex.go new file mode 100644 index 0000000..241cc2d --- /dev/null +++ b/pkg/tcpip/transport/packet/last_error_mutex.go @@ -0,0 +1,64 @@ +package packet + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type lastErrorMutex struct { + mu sync.Mutex +} + +var lastErrorprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var lastErrorlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type lastErrorlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *lastErrorMutex) Lock() { + locking.AddGLock(lastErrorprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *lastErrorMutex) NestedLock(i lastErrorlockNameIndex) { + locking.AddGLock(lastErrorprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *lastErrorMutex) Unlock() { + locking.DelGLock(lastErrorprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *lastErrorMutex) NestedUnlock(i lastErrorlockNameIndex) { + locking.DelGLock(lastErrorprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func lastErrorinitLockNames() {} + +func init() { + lastErrorinitLockNames() + lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeOf(lastErrorMutex{}), lastErrorlockNames) +} diff --git a/pkg/tcpip/transport/packet/packet_list.go b/pkg/tcpip/transport/packet/packet_list.go new file mode 100644 index 0000000..74bed44 --- /dev/null +++ b/pkg/tcpip/transport/packet/packet_list.go @@ -0,0 +1,239 @@ +package packet + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type packetElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (packetElementMapper) linkerFor(elem *packet) *packet { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type packetList struct { + head *packet + tail *packet +} + +// Reset resets list l to the empty state. +func (l *packetList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *packetList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *packetList) Front() *packet { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *packetList) Back() *packet { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *packetList) Len() (count int) { + for e := l.Front(); e != nil; e = (packetElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *packetList) PushFront(e *packet) { + linker := packetElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + packetElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *packetList) PushFrontList(m *packetList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + packetElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + packetElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *packetList) PushBack(e *packet) { + linker := packetElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + packetElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *packetList) PushBackList(m *packetList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + packetElementMapper{}.linkerFor(l.tail).SetNext(m.head) + packetElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *packetList) InsertAfter(b, e *packet) { + bLinker := packetElementMapper{}.linkerFor(b) + eLinker := packetElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + packetElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *packetList) InsertBefore(a, e *packet) { + aLinker := packetElementMapper{}.linkerFor(a) + eLinker := packetElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + packetElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *packetList) Remove(e *packet) { + linker := packetElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + packetElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + packetElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type packetEntry struct { + next *packet + prev *packet +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *packetEntry) Next() *packet { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *packetEntry) Prev() *packet { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *packetEntry) SetNext(elem *packet) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *packetEntry) SetPrev(elem *packet) { + e.prev = elem +} diff --git a/pkg/tcpip/transport/packet/packet_mmap_mutex.go b/pkg/tcpip/transport/packet/packet_mmap_mutex.go new file mode 100644 index 0000000..853ed73 --- /dev/null +++ b/pkg/tcpip/transport/packet/packet_mmap_mutex.go @@ -0,0 +1,96 @@ +package packet + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type packetMmapRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var packetMmaplockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type packetMmaplockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *packetMmapRWMutex) Lock() { + locking.AddGLock(packetMmapprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetMmapRWMutex) NestedLock(i packetMmaplockNameIndex) { + locking.AddGLock(packetMmapprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *packetMmapRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(packetMmapprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *packetMmapRWMutex) NestedUnlock(i packetMmaplockNameIndex) { + m.mu.Unlock() + locking.DelGLock(packetMmapprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *packetMmapRWMutex) RLock() { + locking.AddGLock(packetMmapprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *packetMmapRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(packetMmapprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *packetMmapRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *packetMmapRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *packetMmapRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var packetMmapprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func packetMmapinitLockNames() {} + +func init() { + packetMmapinitLockNames() + packetMmapprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetMmapRWMutex{}), packetMmaplockNames) +} diff --git a/pkg/tcpip/transport/packet/packet_state_autogen.go b/pkg/tcpip/transport/packet/packet_state_autogen.go new file mode 100644 index 0000000..0d62c0a --- /dev/null +++ b/pkg/tcpip/transport/packet/packet_state_autogen.go @@ -0,0 +1,181 @@ +// automatically generated by stateify. + +package packet + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *packet) StateTypeName() string { + return "pkg/tcpip/transport/packet.packet" +} + +func (p *packet) StateFields() []string { + return []string{ + "packetEntry", + "data", + "receivedAt", + "senderAddr", + "packetInfo", + } +} + +func (p *packet) beforeSave() {} + +// +checklocksignore +func (p *packet) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + var receivedAtValue int64 + receivedAtValue = p.saveReceivedAt() + stateSinkObject.SaveValue(2, receivedAtValue) + stateSinkObject.Save(0, &p.packetEntry) + stateSinkObject.Save(1, &p.data) + stateSinkObject.Save(3, &p.senderAddr) + stateSinkObject.Save(4, &p.packetInfo) +} + +func (p *packet) afterLoad(context.Context) {} + +// +checklocksignore +func (p *packet) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.packetEntry) + stateSourceObject.Load(1, &p.data) + stateSourceObject.Load(3, &p.senderAddr) + stateSourceObject.Load(4, &p.packetInfo) + stateSourceObject.LoadValue(2, new(int64), func(y any) { p.loadReceivedAt(ctx, y.(int64)) }) +} + +func (ep *endpoint) StateTypeName() string { + return "pkg/tcpip/transport/packet.endpoint" +} + +func (ep *endpoint) StateFields() []string { + return []string{ + "DefaultSocketOptionsHandler", + "stack", + "waiterQueue", + "cooked", + "ops", + "stats", + "rcvList", + "rcvBufSize", + "rcvClosed", + "rcvDisabled", + "closed", + "boundNetProto", + "boundNIC", + "lastError", + "packetMMapVersion", + "packetMMapReserve", + "packetMMapEp", + } +} + +// +checklocksignore +func (ep *endpoint) StateSave(stateSinkObject state.Sink) { + ep.beforeSave() + stateSinkObject.Save(0, &ep.DefaultSocketOptionsHandler) + stateSinkObject.Save(1, &ep.stack) + stateSinkObject.Save(2, &ep.waiterQueue) + stateSinkObject.Save(3, &ep.cooked) + stateSinkObject.Save(4, &ep.ops) + stateSinkObject.Save(5, &ep.stats) + stateSinkObject.Save(6, &ep.rcvList) + stateSinkObject.Save(7, &ep.rcvBufSize) + stateSinkObject.Save(8, &ep.rcvClosed) + stateSinkObject.Save(9, &ep.rcvDisabled) + stateSinkObject.Save(10, &ep.closed) + stateSinkObject.Save(11, &ep.boundNetProto) + stateSinkObject.Save(12, &ep.boundNIC) + stateSinkObject.Save(13, &ep.lastError) + stateSinkObject.Save(14, &ep.packetMMapVersion) + stateSinkObject.Save(15, &ep.packetMMapReserve) + stateSinkObject.Save(16, &ep.packetMMapEp) +} + +// +checklocksignore +func (ep *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &ep.DefaultSocketOptionsHandler) + stateSourceObject.Load(1, &ep.stack) + stateSourceObject.Load(2, &ep.waiterQueue) + stateSourceObject.Load(3, &ep.cooked) + stateSourceObject.Load(4, &ep.ops) + stateSourceObject.Load(5, &ep.stats) + stateSourceObject.Load(6, &ep.rcvList) + stateSourceObject.Load(7, &ep.rcvBufSize) + stateSourceObject.Load(8, &ep.rcvClosed) + stateSourceObject.Load(9, &ep.rcvDisabled) + stateSourceObject.Load(10, &ep.closed) + stateSourceObject.Load(11, &ep.boundNetProto) + stateSourceObject.Load(12, &ep.boundNIC) + stateSourceObject.Load(13, &ep.lastError) + stateSourceObject.Load(14, &ep.packetMMapVersion) + stateSourceObject.Load(15, &ep.packetMMapReserve) + stateSourceObject.Load(16, &ep.packetMMapEp) + stateSourceObject.AfterLoad(func() { ep.afterLoad(ctx) }) +} + +func (l *packetList) StateTypeName() string { + return "pkg/tcpip/transport/packet.packetList" +} + +func (l *packetList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *packetList) beforeSave() {} + +// +checklocksignore +func (l *packetList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *packetList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *packetList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *packetEntry) StateTypeName() string { + return "pkg/tcpip/transport/packet.packetEntry" +} + +func (e *packetEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *packetEntry) beforeSave() {} + +// +checklocksignore +func (e *packetEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *packetEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *packetEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*packet)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*packetList)(nil)) + state.Register((*packetEntry)(nil)) +} diff --git a/pkg/tcpip/transport/raw/endpoint.go b/pkg/tcpip/transport/raw/endpoint.go new file mode 100644 index 0000000..5fa1820 --- /dev/null +++ b/pkg/tcpip/transport/raw/endpoint.go @@ -0,0 +1,783 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package raw provides the implementation of raw sockets (see raw(7)). Raw +// sockets allow applications to: +// +// - manually write and inspect transport layer headers and payloads +// - receive all traffic of a given transport protocol (e.g. ICMP or UDP) +// - optionally write and inspect network layer headers of packets +// +// Raw sockets don't have any notion of ports, and incoming packets are +// demultiplexed solely by protocol number. Thus, a raw UDP endpoint will +// receive every UDP packet received by netstack. bind(2) and connect(2) can be +// used to filter incoming packets by source and destination. +package raw + +import ( + "fmt" + "io" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" + "github.com/sagernet/gvisor/pkg/tcpip/transport/internal/network" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// +stateify savable +type rawPacket struct { + rawPacketEntry + // data holds the actual packet data, including any headers and + // payload. + data *stack.PacketBuffer + receivedAt time.Time `state:".(int64)"` + // senderAddr is the network address of the sender. + senderAddr tcpip.FullAddress + packetInfo tcpip.IPPacketInfo + + // tosOrTClass stores either the Type of Service for IPv4 or the Traffic Class + // for IPv6. + tosOrTClass uint8 + // ttlOrHopLimit stores either the TTL for IPv4 or the HopLimit for IPv6 + ttlOrHopLimit uint8 +} + +// endpoint is the raw socket implementation of tcpip.Endpoint. It is legal to +// have goroutines make concurrent calls into the endpoint. +// +// Lock order: +// +// endpoint.mu +// endpoint.rcvMu +// +// +stateify savable +type endpoint struct { + tcpip.DefaultSocketOptionsHandler + + // The following fields are initialized at creation time and are + // immutable. + stack *stack.Stack + transProto tcpip.TransportProtocolNumber + waiterQueue *waiter.Queue + associated bool + + net network.Endpoint + stats tcpip.TransportEndpointStats + ops tcpip.SocketOptions + + rcvMu sync.Mutex `state:"nosave"` + // +checklocks:rcvMu + rcvList rawPacketList + // +checklocks:rcvMu + rcvBufSize int + // +checklocks:rcvMu + rcvClosed bool + // +checklocks:rcvMu + rcvDisabled bool + + mu sync.RWMutex `state:"nosave"` + + // ipv6ChecksumOffset indicates the offset to populate the IPv6 checksum at. + // + // A negative value indicates no checksum should be calculated. + // + // +checklocks:mu + ipv6ChecksumOffset int + // icmp6Filter holds the filter for ICMPv6 packets. + // + // +checklocks:mu + icmpv6Filter tcpip.ICMPv6Filter +} + +// NewEndpoint returns a raw endpoint for the given protocols. +func NewEndpoint(stack *stack.Stack, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return newEndpoint(stack, netProto, transProto, waiterQueue, true /* associated */) +} + +func newEndpoint(s *stack.Stack, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, waiterQueue *waiter.Queue, associated bool) (tcpip.Endpoint, tcpip.Error) { + // Calculating the upper-layer checksum is disabled by default for raw IPv6 + // endpoints, unless the upper-layer protocol is ICMPv6. + // + // As per RFC 3542 section 3.1, + // + // The kernel will calculate and insert the ICMPv6 checksum for ICMPv6 + // raw sockets, since this checksum is mandatory. + ipv6ChecksumOffset := -1 + if netProto == header.IPv6ProtocolNumber && transProto == header.ICMPv6ProtocolNumber { + ipv6ChecksumOffset = header.ICMPv6ChecksumOffset + } + + e := &endpoint{ + stack: s, + transProto: transProto, + waiterQueue: waiterQueue, + associated: associated, + ipv6ChecksumOffset: ipv6ChecksumOffset, + } + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + e.ops.SetMulticastLoop(true) + e.ops.SetHeaderIncluded(!associated) + e.ops.SetSendBufferSize(32*1024, false /* notify */) + e.ops.SetReceiveBufferSize(32*1024, false /* notify */) + e.net.Init(s, netProto, transProto, &e.ops, waiterQueue) + + // Override with stack defaults. + var ss tcpip.SendBufferSizeOption + if err := s.Option(&ss); err == nil { + e.ops.SetSendBufferSize(int64(ss.Default), false /* notify */) + } + + var rs tcpip.ReceiveBufferSizeOption + if err := s.Option(&rs); err == nil { + e.ops.SetReceiveBufferSize(int64(rs.Default), false /* notify */) + } + + // Unassociated endpoints are write-only and users call Write() with IP + // headers included. Because they're write-only, We don't need to + // register with the stack. + if !associated { + e.ops.SetReceiveBufferSize(0, false /* notify */) + e.waiterQueue = nil + return e, nil + } + + if err := e.stack.RegisterRawTransportEndpoint(netProto, e.transProto, e); err != nil { + return nil, err + } + + return e, nil +} + +// WakeupWriters implements tcpip.SocketOptionsHandler. +func (e *endpoint) WakeupWriters() { + e.net.MaybeSignalWritable() +} + +// HasNIC implements tcpip.SocketOptionsHandler. +func (e *endpoint) HasNIC(id int32) bool { + return e.stack.HasNIC(tcpip.NICID(id)) +} + +// Abort implements stack.TransportEndpoint.Abort. +func (e *endpoint) Abort() { + e.Close() +} + +// Close implements tcpip.Endpoint.Close. +func (e *endpoint) Close() { + e.mu.Lock() + defer e.mu.Unlock() + + if e.net.State() == transport.DatagramEndpointStateClosed { + return + } + + e.net.Close() + + if !e.associated { + return + } + + e.stack.UnregisterRawTransportEndpoint(e.net.NetProto(), e.transProto, e) + + e.rcvMu.Lock() + defer e.rcvMu.Unlock() + + // Clear the receive list. + e.rcvClosed = true + e.rcvBufSize = 0 + for !e.rcvList.Empty() { + p := e.rcvList.Front() + e.rcvList.Remove(p) + p.data.DecRef() + } + + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) +} + +// ModerateRecvBuf implements tcpip.Endpoint.ModerateRecvBuf. +func (*endpoint) ModerateRecvBuf(int) {} + +func (e *endpoint) SetOwner(owner tcpip.PacketOwner) { + e.net.SetOwner(owner) +} + +// Read implements tcpip.Endpoint.Read. +func (e *endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult, tcpip.Error) { + e.rcvMu.Lock() + + // If there's no data to read, return that read would block or that the + // endpoint is closed. + if e.rcvList.Empty() { + var err tcpip.Error = &tcpip.ErrWouldBlock{} + if e.rcvClosed { + e.stats.ReadErrors.ReadClosed.Increment() + err = &tcpip.ErrClosedForReceive{} + } + e.rcvMu.Unlock() + return tcpip.ReadResult{}, err + } + + pkt := e.rcvList.Front() + if !opts.Peek { + e.rcvList.Remove(pkt) + defer pkt.data.DecRef() + e.rcvBufSize -= pkt.data.Data().Size() + } + + e.rcvMu.Unlock() + + // Control Messages + // TODO(https://gvisor.dev/issue/7012): Share control message code with other + // network endpoints. + cm := tcpip.ReceivableControlMessages{ + HasTimestamp: true, + Timestamp: pkt.receivedAt, + } + switch netProto := e.net.NetProto(); netProto { + case header.IPv4ProtocolNumber: + if e.ops.GetReceiveTOS() { + cm.HasTOS = true + cm.TOS = pkt.tosOrTClass + } + if e.ops.GetReceiveTTL() { + cm.HasTTL = true + cm.TTL = pkt.ttlOrHopLimit + } + if e.ops.GetReceivePacketInfo() { + cm.HasIPPacketInfo = true + cm.PacketInfo = pkt.packetInfo + } + case header.IPv6ProtocolNumber: + if e.ops.GetReceiveTClass() { + cm.HasTClass = true + // Although TClass is an 8-bit value it's read in the CMsg as a uint32. + cm.TClass = uint32(pkt.tosOrTClass) + } + if e.ops.GetReceiveHopLimit() { + cm.HasHopLimit = true + cm.HopLimit = pkt.ttlOrHopLimit + } + if e.ops.GetIPv6ReceivePacketInfo() { + cm.HasIPv6PacketInfo = true + cm.IPv6PacketInfo = tcpip.IPv6PacketInfo{ + NIC: pkt.packetInfo.NIC, + Addr: pkt.packetInfo.DestinationAddr, + } + } + default: + panic(fmt.Sprintf("unrecognized network protocol = %d", netProto)) + } + + res := tcpip.ReadResult{ + Total: pkt.data.Data().Size(), + ControlMessages: cm, + } + if opts.NeedRemoteAddr { + res.RemoteAddr = pkt.senderAddr + } + + n, err := pkt.data.Data().ReadTo(dst, opts.Peek) + if n == 0 && err != nil { + return res, &tcpip.ErrBadBuffer{} + } + res.Count = n + return res, nil +} + +// Write implements tcpip.Endpoint.Write. +func (e *endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + netProto := e.net.NetProto() + // We can create, but not write to, unassociated IPv6 endpoints. + if !e.associated && netProto == header.IPv6ProtocolNumber { + return 0, &tcpip.ErrInvalidOptionValue{} + } + + if opts.To != nil { + // Raw sockets do not support sending to a IPv4 address on a IPv6 endpoint. + if netProto == header.IPv6ProtocolNumber && opts.To.Addr.BitLen() != header.IPv6AddressSizeBits { + return 0, &tcpip.ErrInvalidOptionValue{} + } + } + + n, err := e.write(p, opts) + switch err.(type) { + case nil: + e.stats.PacketsSent.Increment() + case *tcpip.ErrMessageTooLong, *tcpip.ErrInvalidOptionValue: + e.stats.WriteErrors.InvalidArgs.Increment() + case *tcpip.ErrClosedForSend: + e.stats.WriteErrors.WriteClosed.Increment() + case *tcpip.ErrInvalidEndpointState: + e.stats.WriteErrors.InvalidEndpointState.Increment() + case *tcpip.ErrHostUnreachable, *tcpip.ErrBroadcastDisabled, *tcpip.ErrNetworkUnreachable: + // Errors indicating any problem with IP routing of the packet. + e.stats.SendErrors.NoRoute.Increment() + default: + // For all other errors when writing to the network layer. + e.stats.SendErrors.SendToNetworkFailed.Increment() + } + return n, err +} + +func (e *endpoint) write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + e.mu.Lock() + ctx, err := e.net.AcquireContextForWrite(opts) + ipv6ChecksumOffset := e.ipv6ChecksumOffset + e.mu.Unlock() + if err != nil { + return 0, err + } + defer ctx.Release() + + if p.Len() > int(ctx.MTU()) { + return 0, &tcpip.ErrMessageTooLong{} + } + + // Prevents giant buffer allocations. + if p.Len() > header.DatagramMaximumSize { + return 0, &tcpip.ErrMessageTooLong{} + } + + var payload buffer.Buffer + defer payload.Release() + if _, err := payload.WriteFromReader(p, int64(p.Len())); err != nil { + return 0, &tcpip.ErrBadBuffer{} + } + payloadSz := payload.Size() + + if packetInfo := ctx.PacketInfo(); packetInfo.NetProto == header.IPv6ProtocolNumber && ipv6ChecksumOffset >= 0 { + // Make sure we can fit the checksum. + if payload.Size() < int64(ipv6ChecksumOffset+checksum.Size) { + return 0, &tcpip.ErrInvalidOptionValue{} + } + + payloadView, _ := payload.PullUp(ipv6ChecksumOffset, int(payload.Size())-ipv6ChecksumOffset) + xsum := header.PseudoHeaderChecksum(e.transProto, packetInfo.LocalAddress, packetInfo.RemoteAddress, uint16(payload.Size())) + checksum.Put(payloadView.AsSlice(), 0) + xsum = checksum.Combine(payload.Checksum(0), xsum) + checksum.Put(payloadView.AsSlice(), ^xsum) + } + + pkt := ctx.TryNewPacketBuffer(int(ctx.PacketInfo().MaxHeaderLength), payload.Clone()) + if pkt == nil { + return 0, &tcpip.ErrWouldBlock{} + } + defer pkt.DecRef() + + if err := ctx.WritePacket(pkt, e.ops.GetHeaderIncluded()); err != nil { + return 0, err + } + + return payloadSz, nil +} + +// Disconnect implements tcpip.Endpoint.Disconnect. +func (*endpoint) Disconnect() tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Connect implements tcpip.Endpoint.Connect. +func (e *endpoint) Connect(addr tcpip.FullAddress) tcpip.Error { + netProto := e.net.NetProto() + + // Raw sockets do not support connecting to a IPv4 address on a IPv6 endpoint. + if netProto == header.IPv6ProtocolNumber && addr.Addr.BitLen() != header.IPv6AddressSizeBits { + return &tcpip.ErrAddressFamilyNotSupported{} + } + + return e.net.ConnectAndThen(addr, func(_ tcpip.NetworkProtocolNumber, _, _ stack.TransportEndpointID) tcpip.Error { + if e.associated { + // Re-register the endpoint with the appropriate NIC. + if err := e.stack.RegisterRawTransportEndpoint(netProto, e.transProto, e); err != nil { + return err + } + e.stack.UnregisterRawTransportEndpoint(netProto, e.transProto, e) + } + + return nil + }) +} + +// Shutdown implements tcpip.Endpoint.Shutdown. It's a noop for raw sockets. +func (e *endpoint) Shutdown(tcpip.ShutdownFlags) tcpip.Error { + if e.net.State() != transport.DatagramEndpointStateConnected { + return &tcpip.ErrNotConnected{} + } + return nil +} + +// Listen implements tcpip.Endpoint.Listen. +func (*endpoint) Listen(int) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Accept implements tcpip.Endpoint.Accept. +func (*endpoint) Accept(*tcpip.FullAddress) (tcpip.Endpoint, *waiter.Queue, tcpip.Error) { + return nil, nil, &tcpip.ErrNotSupported{} +} + +// Bind implements tcpip.Endpoint.Bind. +func (e *endpoint) Bind(addr tcpip.FullAddress) tcpip.Error { + return e.net.BindAndThen(addr, func(netProto tcpip.NetworkProtocolNumber, _ tcpip.Address) tcpip.Error { + if !e.associated { + return nil + } + + // Re-register the endpoint with the appropriate NIC. + if err := e.stack.RegisterRawTransportEndpoint(netProto, e.transProto, e); err != nil { + return err + } + e.stack.UnregisterRawTransportEndpoint(netProto, e.transProto, e) + return nil + }) +} + +// GetLocalAddress implements tcpip.Endpoint.GetLocalAddress. +func (e *endpoint) GetLocalAddress() (tcpip.FullAddress, tcpip.Error) { + a := e.net.GetLocalAddress() + // Linux returns the protocol in the port field. + a.Port = uint16(e.transProto) + return a, nil +} + +// GetRemoteAddress implements tcpip.Endpoint.GetRemoteAddress. +func (*endpoint) GetRemoteAddress() (tcpip.FullAddress, tcpip.Error) { + // Even a connected socket doesn't return a remote address. + return tcpip.FullAddress{}, &tcpip.ErrNotConnected{} +} + +// Readiness implements tcpip.Endpoint.Readiness. +func (e *endpoint) Readiness(mask waiter.EventMask) waiter.EventMask { + var result waiter.EventMask + + if e.net.HasSendSpace() { + result |= waiter.WritableEvents & mask + } + + // Determine whether the endpoint is readable. + if (mask & waiter.ReadableEvents) != 0 { + e.rcvMu.Lock() + if !e.rcvList.Empty() || e.rcvClosed { + result |= waiter.ReadableEvents + } + e.rcvMu.Unlock() + } + + return result +} + +// SetSockOpt implements tcpip.Endpoint.SetSockOpt. +func (e *endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { + switch opt := opt.(type) { + case *tcpip.SocketDetachFilterOption: + return nil + + case *tcpip.ICMPv6Filter: + if e.net.NetProto() != header.IPv6ProtocolNumber { + return &tcpip.ErrUnknownProtocolOption{} + } + + if e.transProto != header.ICMPv6ProtocolNumber { + return &tcpip.ErrInvalidOptionValue{} + } + + e.mu.Lock() + defer e.mu.Unlock() + e.icmpv6Filter = *opt + return nil + default: + return e.net.SetSockOpt(opt) + } +} + +func (e *endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { + switch opt { + case tcpip.IPv6Checksum: + if e.net.NetProto() != header.IPv6ProtocolNumber { + return &tcpip.ErrUnknownProtocolOption{} + } + + if e.transProto == header.ICMPv6ProtocolNumber { + // As per RFC 3542 section 3.1, + // + // An attempt to set IPV6_CHECKSUM for an ICMPv6 socket will fail. + return &tcpip.ErrInvalidOptionValue{} + } + + // Make sure the offset is aligned properly if checksum is requested. + if v > 0 && v%checksum.Size != 0 { + return &tcpip.ErrInvalidOptionValue{} + } + + e.mu.Lock() + defer e.mu.Unlock() + e.ipv6ChecksumOffset = v + return nil + default: + return e.net.SetSockOptInt(opt, v) + } +} + +// GetSockOpt implements tcpip.Endpoint.GetSockOpt. +func (e *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { + switch opt := opt.(type) { + case *tcpip.ICMPv6Filter: + if e.net.NetProto() != header.IPv6ProtocolNumber { + return &tcpip.ErrUnknownProtocolOption{} + } + + if e.transProto != header.ICMPv6ProtocolNumber { + return &tcpip.ErrInvalidOptionValue{} + } + + e.mu.RLock() + defer e.mu.RUnlock() + *opt = e.icmpv6Filter + return nil + + default: + return e.net.GetSockOpt(opt) + } +} + +// GetSockOptInt implements tcpip.Endpoint.GetSockOptInt. +func (e *endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { + switch opt { + case tcpip.ReceiveQueueSizeOption: + v := 0 + e.rcvMu.Lock() + if !e.rcvList.Empty() { + p := e.rcvList.Front() + v = p.data.Data().Size() + } + e.rcvMu.Unlock() + return v, nil + + case tcpip.IPv6Checksum: + if e.net.NetProto() != header.IPv6ProtocolNumber { + return 0, &tcpip.ErrUnknownProtocolOption{} + } + + e.mu.Lock() + defer e.mu.Unlock() + return e.ipv6ChecksumOffset, nil + + default: + return e.net.GetSockOptInt(opt) + } +} + +// HandlePacket implements stack.RawTransportEndpoint.HandlePacket. +func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { + notifyReadableEvents := func() bool { + e.mu.RLock() + defer e.mu.RUnlock() + e.rcvMu.Lock() + defer e.rcvMu.Unlock() + + // Drop the packet if our buffer is currently full or if this is an unassociated + // endpoint (i.e endpoint created w/ IPPROTO_RAW). Such endpoints are send only + // See: https://man7.org/linux/man-pages/man7/raw.7.html + // + // An IPPROTO_RAW socket is send only. If you really want to receive + // all IP packets, use a packet(7) socket with the ETH_P_IP protocol. + // Note that packet sockets don't reassemble IP fragments, unlike raw + // sockets. + if e.rcvClosed || !e.associated { + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.ClosedReceiver.Increment() + return false + } + + rcvBufSize := e.ops.GetReceiveBufferSize() + if e.rcvDisabled || e.rcvBufSize >= int(rcvBufSize) { + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.ReceiveBufferOverflow.Increment() + return false + } + + net := pkt.Network() + dstAddr := net.DestinationAddress() + srcAddr := net.SourceAddress() + info := e.net.Info() + + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial: + case transport.DatagramEndpointStateConnected: + // If connected, only accept packets from the remote address we + // connected to. + if info.ID.RemoteAddress != srcAddr { + return false + } + + // Connected sockets may also have been bound to a specific + // address/NIC. + fallthrough + case transport.DatagramEndpointStateBound: + // If bound to a NIC, only accept data for that NIC. + if info.BindNICID != 0 && info.BindNICID != pkt.NICID { + return false + } + + // If bound to an address, only accept data for that address. + if info.BindAddr != (tcpip.Address{}) && info.BindAddr != dstAddr { + return false + } + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } + + wasEmpty := e.rcvBufSize == 0 + + // Push new packet into receive list and increment the buffer size. + packet := &rawPacket{ + senderAddr: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: srcAddr, + }, + packetInfo: tcpip.IPPacketInfo{ + // TODO(gvisor.dev/issue/3556): dstAddr may be a multicast or broadcast + // address. LocalAddr should hold a unicast address that can be + // used to respond to the incoming packet. + LocalAddr: dstAddr, + DestinationAddr: dstAddr, + NIC: pkt.NICID, + }, + } + + // Save any useful information from the network header to the packet. + packet.tosOrTClass, _ = pkt.Network().TOS() + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + packet.ttlOrHopLimit = header.IPv4(pkt.NetworkHeader().Slice()).TTL() + case header.IPv6ProtocolNumber: + packet.ttlOrHopLimit = header.IPv6(pkt.NetworkHeader().Slice()).HopLimit() + } + + // Raw IPv4 endpoints return the IP header, but IPv6 endpoints do not. + // We copy headers' underlying bytes because pkt.*Header may point to + // the middle of a slice, and another struct may point to the "outer" + // slice. Save/restore doesn't support overlapping slices and will fail. + // + // TODO(https://gvisor.dev/issue/6517): Avoid the copy once S/R supports + // overlapping slices. + transportHeader := pkt.TransportHeader().Slice() + var combinedBuf buffer.Buffer + defer combinedBuf.Release() + switch info.NetProto { + case header.IPv4ProtocolNumber: + networkHeader := pkt.NetworkHeader().Slice() + headers := buffer.NewView(len(networkHeader) + len(transportHeader)) + headers.Write(networkHeader) + headers.Write(transportHeader) + combinedBuf = buffer.MakeWithView(headers) + pktBuf := pkt.Data().ToBuffer() + combinedBuf.Merge(&pktBuf) + case header.IPv6ProtocolNumber: + /*if e.transProto == header.ICMPv6ProtocolNumber { + if len(transportHeader) < header.ICMPv6MinimumSize { + return false + } + + if e.icmpv6Filter.ShouldDeny(uint8(header.ICMPv6(transportHeader).Type())) { + return false + } + } + + combinedBuf = buffer.MakeWithView(pkt.TransportHeader().View()) + pktBuf := pkt.Data().ToBuffer() + combinedBuf.Merge(&pktBuf) + + if checksumOffset := e.ipv6ChecksumOffset; checksumOffset >= 0 { + bufSize := int(combinedBuf.Size()) + if bufSize < checksumOffset+checksum.Size { + // Message too small to fit checksum. + return false + } + + xsum := header.PseudoHeaderChecksum(e.transProto, srcAddr, dstAddr, uint16(bufSize)) + xsum = checksum.Combine(combinedBuf.Checksum(0), xsum) + if xsum != 0xFFFF { + // Invalid checksum. + return false + } + }*/ + networkHeader := pkt.NetworkHeader().Slice() + headers := buffer.NewView(len(networkHeader) + len(transportHeader)) + headers.Write(networkHeader) + headers.Write(transportHeader) + combinedBuf = buffer.MakeWithView(headers) + pktBuf := pkt.Data().ToBuffer() + combinedBuf.Merge(&pktBuf) + default: + panic(fmt.Sprintf("unrecognized protocol number = %d", info.NetProto)) + } + + packet.data = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: combinedBuf.Clone()}) + packet.receivedAt = e.stack.Clock().Now() + + e.rcvList.PushBack(packet) + e.rcvBufSize += packet.data.Data().Size() + e.stats.PacketsReceived.Increment() + + // Notify waiters that there is data to be read now. + return wasEmpty + }() + + if notifyReadableEvents { + e.waiterQueue.Notify(waiter.ReadableEvents) + } +} + +// State implements socket.Socket.State. +func (e *endpoint) State() uint32 { + return uint32(e.net.State()) +} + +// Info returns a copy of the endpoint info. +func (e *endpoint) Info() tcpip.EndpointInfo { + ret := e.net.Info() + return &ret +} + +// Stats returns a pointer to the endpoint stats. +func (e *endpoint) Stats() tcpip.EndpointStats { + return &e.stats +} + +// Wait implements stack.TransportEndpoint.Wait. +func (*endpoint) Wait() {} + +// LastError implements tcpip.Endpoint.LastError. +func (*endpoint) LastError() tcpip.Error { + return nil +} + +// SocketOptions implements tcpip.Endpoint.SocketOptions. +func (e *endpoint) SocketOptions() *tcpip.SocketOptions { + return &e.ops +} + +func (e *endpoint) setReceiveDisabled(v bool) { + e.rcvMu.Lock() + defer e.rcvMu.Unlock() + e.rcvDisabled = v +} diff --git a/pkg/tcpip/transport/raw/endpoint_state.go b/pkg/tcpip/transport/raw/endpoint_state.go new file mode 100644 index 0000000..6ba6003 --- /dev/null +++ b/pkg/tcpip/transport/raw/endpoint_state.go @@ -0,0 +1,78 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raw + +import ( + "context" + "time" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// saveReceivedAt is invoked by stateify. +func (p *rawPacket) saveReceivedAt() int64 { + return p.receivedAt.UnixNano() +} + +// loadReceivedAt is invoked by stateify. +func (p *rawPacket) loadReceivedAt(_ context.Context, nsec int64) { + p.receivedAt = time.Unix(0, nsec) +} + +// afterLoad is invoked by stateify. +func (e *endpoint) afterLoad(ctx context.Context) { + if e.stack.IsSaveRestoreEnabled() { + e.stack.RegisterRestoredEndpoint(e) + } else { + stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) + } +} + +// beforeSave is invoked by stateify. +func (e *endpoint) beforeSave() { + e.setReceiveDisabled(true) + e.stack.RegisterResumableEndpoint(e) +} + +// Restore implements tcpip.RestoredEndpoint.Restore. +func (e *endpoint) Restore(s *stack.Stack) { + if err := e.net.Resume(s); err != nil { + log.Warningf("Closing the raw endpoint as it cannot be restored, err: %v", err) + e.Close() + return + } + e.setReceiveDisabled(false) + if e.stack.IsSaveRestoreEnabled() { + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + return + } + + e.stack = s + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + + if e.associated { + netProto := e.net.NetProto() + if err := e.stack.RegisterRawTransportEndpoint(netProto, e.transProto, e); err != nil { + panic("RegisterRawTransportEndpoint failed during restore") + } + } +} + +// Resume implements tcpip.ResumableEndpoint.Resume. +func (e *endpoint) Resume() { + e.setReceiveDisabled(false) +} diff --git a/pkg/tcpip/transport/raw/protocol.go b/pkg/tcpip/transport/raw/protocol.go new file mode 100644 index 0000000..4085eba --- /dev/null +++ b/pkg/tcpip/transport/raw/protocol.go @@ -0,0 +1,55 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raw + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport/internal/noop" + "github.com/sagernet/gvisor/pkg/tcpip/transport/packet" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// EndpointFactory implements stack.RawFactory. +// +// +stateify savable +type EndpointFactory struct{} + +// NewUnassociatedEndpoint implements stack.RawFactory.NewUnassociatedEndpoint. +func (EndpointFactory) NewUnassociatedEndpoint(stack *stack.Stack, netProto tcpip.NetworkProtocolNumber, transProto tcpip.TransportProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return newEndpoint(stack, netProto, transProto, waiterQueue, false /* associated */) +} + +// NewPacketEndpoint implements stack.RawFactory.NewPacketEndpoint. +func (EndpointFactory) NewPacketEndpoint(stack *stack.Stack, cooked bool, netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return packet.NewEndpoint(stack, cooked, netProto, waiterQueue), nil +} + +// CreateOnlyFactory implements stack.RawFactory. It allows creation of raw +// endpoints that do not support reading, writing, binding, etc. +// +// +stateify savable +type CreateOnlyFactory struct{} + +// NewUnassociatedEndpoint implements stack.RawFactory.NewUnassociatedEndpoint. +func (CreateOnlyFactory) NewUnassociatedEndpoint(stk *stack.Stack, _ tcpip.NetworkProtocolNumber, _ tcpip.TransportProtocolNumber, _ *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return noop.New(stk), nil +} + +// NewPacketEndpoint implements stack.RawFactory.NewPacketEndpoint. +func (CreateOnlyFactory) NewPacketEndpoint(*stack.Stack, bool, tcpip.NetworkProtocolNumber, *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + // This isn't needed by anything, so it isn't implemented. + return nil, &tcpip.ErrNotPermitted{} +} diff --git a/pkg/tcpip/transport/raw/raw_packet_list.go b/pkg/tcpip/transport/raw/raw_packet_list.go new file mode 100644 index 0000000..fda4702 --- /dev/null +++ b/pkg/tcpip/transport/raw/raw_packet_list.go @@ -0,0 +1,239 @@ +package raw + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type rawPacketElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (rawPacketElementMapper) linkerFor(elem *rawPacket) *rawPacket { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type rawPacketList struct { + head *rawPacket + tail *rawPacket +} + +// Reset resets list l to the empty state. +func (l *rawPacketList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *rawPacketList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *rawPacketList) Front() *rawPacket { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *rawPacketList) Back() *rawPacket { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *rawPacketList) Len() (count int) { + for e := l.Front(); e != nil; e = (rawPacketElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *rawPacketList) PushFront(e *rawPacket) { + linker := rawPacketElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + rawPacketElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *rawPacketList) PushFrontList(m *rawPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + rawPacketElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + rawPacketElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *rawPacketList) PushBack(e *rawPacket) { + linker := rawPacketElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + rawPacketElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *rawPacketList) PushBackList(m *rawPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + rawPacketElementMapper{}.linkerFor(l.tail).SetNext(m.head) + rawPacketElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *rawPacketList) InsertAfter(b, e *rawPacket) { + bLinker := rawPacketElementMapper{}.linkerFor(b) + eLinker := rawPacketElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + rawPacketElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *rawPacketList) InsertBefore(a, e *rawPacket) { + aLinker := rawPacketElementMapper{}.linkerFor(a) + eLinker := rawPacketElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + rawPacketElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *rawPacketList) Remove(e *rawPacket) { + linker := rawPacketElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + rawPacketElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + rawPacketElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type rawPacketEntry struct { + next *rawPacket + prev *rawPacket +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *rawPacketEntry) Next() *rawPacket { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *rawPacketEntry) Prev() *rawPacket { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *rawPacketEntry) SetNext(elem *rawPacket) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *rawPacketEntry) SetPrev(elem *rawPacket) { + e.prev = elem +} diff --git a/pkg/tcpip/transport/raw/raw_state_autogen.go b/pkg/tcpip/transport/raw/raw_state_autogen.go new file mode 100644 index 0000000..7aee9f1 --- /dev/null +++ b/pkg/tcpip/transport/raw/raw_state_autogen.go @@ -0,0 +1,222 @@ +// automatically generated by stateify. + +package raw + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *rawPacket) StateTypeName() string { + return "pkg/tcpip/transport/raw.rawPacket" +} + +func (p *rawPacket) StateFields() []string { + return []string{ + "rawPacketEntry", + "data", + "receivedAt", + "senderAddr", + "packetInfo", + "tosOrTClass", + "ttlOrHopLimit", + } +} + +func (p *rawPacket) beforeSave() {} + +// +checklocksignore +func (p *rawPacket) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + var receivedAtValue int64 + receivedAtValue = p.saveReceivedAt() + stateSinkObject.SaveValue(2, receivedAtValue) + stateSinkObject.Save(0, &p.rawPacketEntry) + stateSinkObject.Save(1, &p.data) + stateSinkObject.Save(3, &p.senderAddr) + stateSinkObject.Save(4, &p.packetInfo) + stateSinkObject.Save(5, &p.tosOrTClass) + stateSinkObject.Save(6, &p.ttlOrHopLimit) +} + +func (p *rawPacket) afterLoad(context.Context) {} + +// +checklocksignore +func (p *rawPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.rawPacketEntry) + stateSourceObject.Load(1, &p.data) + stateSourceObject.Load(3, &p.senderAddr) + stateSourceObject.Load(4, &p.packetInfo) + stateSourceObject.Load(5, &p.tosOrTClass) + stateSourceObject.Load(6, &p.ttlOrHopLimit) + stateSourceObject.LoadValue(2, new(int64), func(y any) { p.loadReceivedAt(ctx, y.(int64)) }) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/transport/raw.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "DefaultSocketOptionsHandler", + "stack", + "transProto", + "waiterQueue", + "associated", + "net", + "stats", + "ops", + "rcvList", + "rcvBufSize", + "rcvClosed", + "rcvDisabled", + "ipv6ChecksumOffset", + "icmpv6Filter", + } +} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.DefaultSocketOptionsHandler) + stateSinkObject.Save(1, &e.stack) + stateSinkObject.Save(2, &e.transProto) + stateSinkObject.Save(3, &e.waiterQueue) + stateSinkObject.Save(4, &e.associated) + stateSinkObject.Save(5, &e.net) + stateSinkObject.Save(6, &e.stats) + stateSinkObject.Save(7, &e.ops) + stateSinkObject.Save(8, &e.rcvList) + stateSinkObject.Save(9, &e.rcvBufSize) + stateSinkObject.Save(10, &e.rcvClosed) + stateSinkObject.Save(11, &e.rcvDisabled) + stateSinkObject.Save(12, &e.ipv6ChecksumOffset) + stateSinkObject.Save(13, &e.icmpv6Filter) +} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.DefaultSocketOptionsHandler) + stateSourceObject.Load(1, &e.stack) + stateSourceObject.Load(2, &e.transProto) + stateSourceObject.Load(3, &e.waiterQueue) + stateSourceObject.Load(4, &e.associated) + stateSourceObject.Load(5, &e.net) + stateSourceObject.Load(6, &e.stats) + stateSourceObject.Load(7, &e.ops) + stateSourceObject.Load(8, &e.rcvList) + stateSourceObject.Load(9, &e.rcvBufSize) + stateSourceObject.Load(10, &e.rcvClosed) + stateSourceObject.Load(11, &e.rcvDisabled) + stateSourceObject.Load(12, &e.ipv6ChecksumOffset) + stateSourceObject.Load(13, &e.icmpv6Filter) + stateSourceObject.AfterLoad(func() { e.afterLoad(ctx) }) +} + +func (e *EndpointFactory) StateTypeName() string { + return "pkg/tcpip/transport/raw.EndpointFactory" +} + +func (e *EndpointFactory) StateFields() []string { + return []string{} +} + +func (e *EndpointFactory) beforeSave() {} + +// +checklocksignore +func (e *EndpointFactory) StateSave(stateSinkObject state.Sink) { + e.beforeSave() +} + +func (e *EndpointFactory) afterLoad(context.Context) {} + +// +checklocksignore +func (e *EndpointFactory) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (c *CreateOnlyFactory) StateTypeName() string { + return "pkg/tcpip/transport/raw.CreateOnlyFactory" +} + +func (c *CreateOnlyFactory) StateFields() []string { + return []string{} +} + +func (c *CreateOnlyFactory) beforeSave() {} + +// +checklocksignore +func (c *CreateOnlyFactory) StateSave(stateSinkObject state.Sink) { + c.beforeSave() +} + +func (c *CreateOnlyFactory) afterLoad(context.Context) {} + +// +checklocksignore +func (c *CreateOnlyFactory) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + +func (l *rawPacketList) StateTypeName() string { + return "pkg/tcpip/transport/raw.rawPacketList" +} + +func (l *rawPacketList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *rawPacketList) beforeSave() {} + +// +checklocksignore +func (l *rawPacketList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *rawPacketList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *rawPacketList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *rawPacketEntry) StateTypeName() string { + return "pkg/tcpip/transport/raw.rawPacketEntry" +} + +func (e *rawPacketEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *rawPacketEntry) beforeSave() {} + +// +checklocksignore +func (e *rawPacketEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *rawPacketEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *rawPacketEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*rawPacket)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*EndpointFactory)(nil)) + state.Register((*CreateOnlyFactory)(nil)) + state.Register((*rawPacketList)(nil)) + state.Register((*rawPacketEntry)(nil)) +} diff --git a/pkg/tcpip/transport/tcp/accept.go b/pkg/tcpip/transport/tcp/accept.go new file mode 100644 index 0000000..376665d --- /dev/null +++ b/pkg/tcpip/transport/tcp/accept.go @@ -0,0 +1,724 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "container/list" + "crypto/sha1" + "encoding/binary" + "fmt" + "hash" + "io" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +const ( + // tsLen is the length, in bits, of the timestamp in the SYN cookie. + tsLen = 8 + + // tsMask is a mask for timestamp values (i.e., tsLen bits). + tsMask = (1 << tsLen) - 1 + + // tsOffset is the offset, in bits, of the timestamp in the SYN cookie. + tsOffset = 24 + + // hashMask is the mask for hash values (i.e., tsOffset bits). + hashMask = (1 << tsOffset) - 1 + + // maxTSDiff is the maximum allowed difference between a received cookie + // timestamp and the current timestamp. If the difference is greater + // than maxTSDiff, the cookie is expired. + maxTSDiff = 2 +) + +// mssTable is a slice containing the possible MSS values that we +// encode in the SYN cookie with two bits. +var mssTable = []uint16{536, 1300, 1440, 1460} + +func encodeMSS(mss uint16) uint32 { + for i := len(mssTable) - 1; i > 0; i-- { + if mss >= mssTable[i] { + return uint32(i) + } + } + return 0 +} + +// listenContext is used by a listening endpoint to store state used while +// listening for connections. This struct is allocated by the listen goroutine +// and must not be accessed or have its methods called concurrently as they +// may mutate the stored objects. +type listenContext struct { + stack *stack.Stack + protocol *protocol + + // rcvWnd is the receive window that is sent by this listening context + // in the initial SYN-ACK. + rcvWnd seqnum.Size + + // nonce are random bytes that are initialized once when the context + // is created and used to seed the hash function when generating + // the SYN cookie. + nonce [2][sha1.BlockSize]byte + + // listenEP is a reference to the listening endpoint associated with + // this context. Can be nil if the context is created by the forwarder. + listenEP *Endpoint + + // hasherMu protects hasher. + hasherMu hasherMutex + // hasher is the hash function used to generate a SYN cookie. + hasher hash.Hash + + // v6Only is true if listenEP is a dual stack socket and has the + // IPV6_V6ONLY option set. + v6Only bool + + // netProto indicates the network protocol(IPv4/v6) for the listening + // endpoint. + netProto tcpip.NetworkProtocolNumber +} + +// timeStamp returns an 8-bit timestamp with a granularity of 64 seconds. +func timeStamp(clock tcpip.Clock) uint32 { + return uint32(clock.NowMonotonic().Sub(tcpip.MonotonicTime{}).Seconds()) >> 6 & tsMask +} + +// newListenContext creates a new listen context. +func newListenContext(stk *stack.Stack, protocol *protocol, listenEP *Endpoint, rcvWnd seqnum.Size, v6Only bool, netProto tcpip.NetworkProtocolNumber) *listenContext { + l := &listenContext{ + stack: stk, + protocol: protocol, + rcvWnd: rcvWnd, + hasher: sha1.New(), + v6Only: v6Only, + netProto: netProto, + listenEP: listenEP, + } + + for i := range l.nonce { + if _, err := io.ReadFull(stk.SecureRNG().Reader, l.nonce[i][:]); err != nil { + panic(err) + } + } + + return l +} + +// cookieHash calculates the cookieHash for the given id, timestamp and nonce +// index. The hash is used to create and validate cookies. +func (l *listenContext) cookieHash(id stack.TransportEndpointID, ts uint32, nonceIndex int) uint32 { + // Initialize block with fixed-size data: local ports and v. + var payload [8]byte + binary.BigEndian.PutUint16(payload[0:], id.LocalPort) + binary.BigEndian.PutUint16(payload[2:], id.RemotePort) + binary.BigEndian.PutUint32(payload[4:], ts) + + // Feed everything to the hasher. + l.hasherMu.Lock() + l.hasher.Reset() + + // Per hash.Hash.Writer: + // + // It never returns an error. + l.hasher.Write(payload[:]) + l.hasher.Write(l.nonce[nonceIndex][:]) + l.hasher.Write(id.LocalAddress.AsSlice()) + l.hasher.Write(id.RemoteAddress.AsSlice()) + + // Finalize the calculation of the hash and return the first 4 bytes. + h := l.hasher.Sum(nil) + l.hasherMu.Unlock() + + return binary.BigEndian.Uint32(h[:]) +} + +// createCookie creates a SYN cookie for the given id and incoming sequence +// number. +func (l *listenContext) createCookie(id stack.TransportEndpointID, seq seqnum.Value, data uint32) seqnum.Value { + ts := timeStamp(l.stack.Clock()) + v := l.cookieHash(id, 0, 0) + uint32(seq) + (ts << tsOffset) + v += (l.cookieHash(id, ts, 1) + data) & hashMask + return seqnum.Value(v) +} + +// isCookieValid checks if the supplied cookie is valid for the given id and +// sequence number. If it is, it also returns the data originally encoded in the +// cookie when createCookie was called. +func (l *listenContext) isCookieValid(id stack.TransportEndpointID, cookie seqnum.Value, seq seqnum.Value) (uint32, bool) { + ts := timeStamp(l.stack.Clock()) + v := uint32(cookie) - l.cookieHash(id, 0, 0) - uint32(seq) + cookieTS := v >> tsOffset + if ((ts - cookieTS) & tsMask) > maxTSDiff { + return 0, false + } + + return (v - l.cookieHash(id, cookieTS, 1)) & hashMask, true +} + +// createConnectingEndpoint creates a new endpoint in a connecting state, with +// the connection parameters given by the arguments. The newly created endpoint +// will be locked. +// +checklocksacquire:n.mu +func (l *listenContext) createConnectingEndpoint(s *segment, rcvdSynOpts header.TCPSynOptions, queue *waiter.Queue) (n *Endpoint, _ tcpip.Error) { + // Create a new endpoint. + netProto := l.netProto + if netProto == 0 { + netProto = s.pkt.NetworkProtocolNumber + } + + route, err := l.stack.FindRoute(s.pkt.NICID, s.pkt.Network().DestinationAddress(), s.pkt.Network().SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */) + if err != nil { + return nil, err // +checklocksignore + } + + n = newEndpoint(l.stack, l.protocol, netProto, queue) + n.mu.Lock() + n.ops.SetV6Only(l.v6Only) + n.TransportEndpointInfo.ID = s.id + n.boundNICID = s.pkt.NICID + n.route = route + n.effectiveNetProtos = []tcpip.NetworkProtocolNumber{s.pkt.NetworkProtocolNumber} + n.ops.SetReceiveBufferSize(int64(l.rcvWnd), false /* notify */) + n.amss = calculateAdvertisedMSS(n.userMSS, n.route) + n.setEndpointState(StateConnecting) + + n.maybeEnableTimestamp(rcvdSynOpts) + n.maybeEnableSACKPermitted(rcvdSynOpts) + + n.initGSO() + + // Bootstrap the auto tuning algorithm. Starting at zero will result in + // a large step function on the first window adjustment causing the + // window to grow to a really large value. + initWnd := n.initialReceiveWindow() + n.rcvQueueMu.Lock() + n.RcvAutoParams.PrevCopiedBytes = initWnd + n.rcvQueueMu.Unlock() + + return n, nil +} + +// startHandshake creates a new endpoint in connecting state and then sends +// the SYN-ACK for the TCP 3-way handshake. It returns the state of the +// handshake in progress, which includes the new endpoint in the SYN-RCVD +// state. +// +// On success, a handshake h is returned. +// +// NOTE: h.ep.mu is not held and must be acquired if any state needs to be +// modified. +// +// Precondition: if l.listenEP != nil, l.listenEP.mu must be locked. +func (l *listenContext) startHandshake(s *segment, opts header.TCPSynOptions, queue *waiter.Queue, owner tcpip.PacketOwner) (h *handshake, _ tcpip.Error) { + // Create new endpoint. + irs := s.sequenceNumber + isn := generateSecureISN(s.id, l.stack.Clock(), l.protocol.seqnumSecret) + ep, err := l.createConnectingEndpoint(s, opts, queue) + if err != nil { + return nil, err // +checklocksignore + } + + ep.owner = owner + + // listenEP is nil when listenContext is used by tcp.Forwarder. + deferAccept := time.Duration(0) + if l.listenEP != nil { + if l.listenEP.EndpointState() != StateListen { + + // Ensure we release any registrations done by the newly + // created endpoint. + ep.mu.Unlock() + ep.Close() + + return nil, &tcpip.ErrConnectionAborted{} // +checklocksignore + } + + // Propagate any inheritable options from the listening endpoint + // to the newly created endpoint. + l.listenEP.propagateInheritableOptionsLocked(ep) // +checklocksforce + + if !ep.reserveTupleLocked() { + ep.mu.Unlock() + ep.Close() + + return nil, &tcpip.ErrConnectionAborted{} // +checklocksignore + } + + deferAccept = l.listenEP.deferAccept + } + + // Register new endpoint so that packets are routed to it. + if err := ep.stack.RegisterTransportEndpoint( + ep.effectiveNetProtos, + ProtocolNumber, + ep.TransportEndpointInfo.ID, + ep, + ep.boundPortFlags, + ep.boundBindToDevice, + ); err != nil { + ep.mu.Unlock() + ep.Close() + + ep.drainClosingSegmentQueue() + + return nil, err // +checklocksignore + } + + ep.isRegistered = true + + // Initialize and start the handshake. + h = ep.newPassiveHandshake(isn, irs, opts, deferAccept) + h.listenEP = l.listenEP + h.start() + h.ep.mu.Unlock() + return h, nil +} + +// performHandshake performs a TCP 3-way handshake. On success, the new +// established endpoint is returned. +// +// Precondition: if l.listenEP != nil, l.listenEP.mu must be locked. +func (l *listenContext) performHandshake(s *segment, opts header.TCPSynOptions, queue *waiter.Queue, owner tcpip.PacketOwner) (*Endpoint, tcpip.Error) { + waitEntry, notifyCh := waiter.NewChannelEntry(waiter.WritableEvents) + queue.EventRegister(&waitEntry) + defer queue.EventUnregister(&waitEntry) + + h, err := l.startHandshake(s, opts, queue, owner) + if err != nil { + return nil, err + } + + // performHandshake is used by the Forwarder which will block till the + // handshake either succeeds or fails. We do this by registering for + // events above and block on the notification channel. + <-notifyCh + + ep := h.ep + ep.mu.Lock() + if !ep.EndpointState().connected() { + ep.stack.Stats().TCP.FailedConnectionAttempts.Increment() + ep.stats.FailedConnectionAttempts.Increment() + ep.h = nil + ep.mu.Unlock() + ep.Close() + ep.notifyAborted() + ep.drainClosingSegmentQueue() + err := ep.LastError() + if err == nil { + // If err was nil then return the best error we can to indicate + // a connection failure. + err = &tcpip.ErrConnectionAborted{} + } + return nil, err + } + + ep.isConnectNotified = true + + // Transfer any state from the completed handshake to the endpoint. + // + // Update the receive window scaling. We can't do it before the + // handshake because it's possible that the peer doesn't support window + // scaling. + ep.rcv.RcvWndScale = ep.h.effectiveRcvWndScale() + + // Clean up handshake state stored in the endpoint so that it can be + // GCed. + ep.h = nil + ep.mu.Unlock() + return ep, nil +} + +// propagateInheritableOptionsLocked propagates any options set on the listening +// endpoint to the newly created endpoint. +// +// +checklocks:e.mu +// +checklocks:n.mu +func (e *Endpoint) propagateInheritableOptionsLocked(n *Endpoint) { + n.userTimeout = e.userTimeout + n.portFlags = e.portFlags + n.boundBindToDevice = e.boundBindToDevice + n.boundPortFlags = e.boundPortFlags + n.userMSS = e.userMSS +} + +// reserveTupleLocked reserves an accepted endpoint's tuple. +// +// Precondition: e.propagateInheritableOptionsLocked has been called. +// +// +checklocks:e.mu +func (e *Endpoint) reserveTupleLocked() bool { + dest := tcpip.FullAddress{ + Addr: e.TransportEndpointInfo.ID.RemoteAddress, + Port: e.TransportEndpointInfo.ID.RemotePort, + } + portRes := ports.Reservation{ + Networks: e.effectiveNetProtos, + Transport: ProtocolNumber, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: e.TransportEndpointInfo.ID.LocalPort, + Flags: e.boundPortFlags, + BindToDevice: e.boundBindToDevice, + Dest: dest, + } + if !e.stack.ReserveTuple(portRes) { + e.stack.Stats().TCP.FailedPortReservations.Increment() + return false + } + + e.isPortReserved = true + e.boundDest = dest + return true +} + +// notifyAborted wakes up any waiters on registered, but not accepted +// endpoints. +// +// This is strictly not required normally as a socket that was never accepted +// can't really have any registered waiters except when stack.Wait() is called +// which waits for all registered endpoints to stop and expects an EventHUp. +func (e *Endpoint) notifyAborted() { + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) +} + +func (e *Endpoint) acceptQueueIsFull() bool { + e.acceptMu.Lock() + full := e.acceptQueue.isFull() + e.acceptMu.Unlock() + return full +} + +// +stateify savable +type acceptQueue struct { + // NB: this could be an endpointList, but ilist only permits endpoints to + // belong to one list at a time, and endpoints are already stored in the + // dispatcher's list. + endpoints list.List `state:".([]*Endpoint)"` + + // pendingEndpoints is a set of all endpoints for which a handshake is + // in progress. + pendingEndpoints map[*Endpoint]struct{} + + // capacity is the maximum number of endpoints that can be in endpoints. + capacity int +} + +func (a *acceptQueue) isFull() bool { + return a.endpoints.Len() >= a.capacity +} + +// handleListenSegment is called when a listening endpoint receives a segment +// and needs to handle it. +// +// +checklocks:e.mu +func (e *Endpoint) handleListenSegment(ctx *listenContext, s *segment) tcpip.Error { + e.rcvQueueMu.Lock() + rcvClosed := e.RcvClosed + e.rcvQueueMu.Unlock() + if rcvClosed || s.flags.Contains(header.TCPFlagSyn|header.TCPFlagAck) { + // If the endpoint is shutdown, reply with reset. + // + // RFC 793 section 3.4 page 35 (figure 12) outlines that a RST + // must be sent in response to a SYN-ACK while in the listen + // state to prevent completing a handshake from an old SYN. + return replyWithReset(e.stack, s, e.sendTOS, e.ipv4TTL, e.ipv6HopLimit) + } + + switch { + case s.flags.Contains(header.TCPFlagRst): + e.stack.Stats().DroppedPackets.Increment() + return nil + + case s.flags.Contains(header.TCPFlagSyn): + if e.acceptQueueIsFull() { + e.stack.Stats().TCP.ListenOverflowSynDrop.Increment() + e.stats.ReceiveErrors.ListenOverflowSynDrop.Increment() + e.stack.Stats().DroppedPackets.Increment() + return nil + } + + opts := parseSynSegmentOptions(s) + + useSynCookies, err := func() (bool, tcpip.Error) { + var alwaysUseSynCookies tcpip.TCPAlwaysUseSynCookies + if err := e.stack.TransportProtocolOption(header.TCPProtocolNumber, &alwaysUseSynCookies); err != nil { + panic(fmt.Sprintf("TransportProtocolOption(%d, %T) = %s", header.TCPProtocolNumber, alwaysUseSynCookies, err)) + } + if alwaysUseSynCookies { + return true, nil + } + e.acceptMu.Lock() + defer e.acceptMu.Unlock() + + // The capacity of the accepted queue would always be one greater than the + // listen backlog. But, the SYNRCVD connections count is always checked + // against the listen backlog value for Linux parity reason. + // https://github.com/torvalds/linux/blob/7acac4b3196/include/net/inet_connection_sock.h#L280 + if len(e.acceptQueue.pendingEndpoints) == e.acceptQueue.capacity-1 { + return true, nil + } + + h, err := ctx.startHandshake(s, opts, &waiter.Queue{}, e.owner) + if err != nil { + e.stack.Stats().TCP.FailedConnectionAttempts.Increment() + e.stats.FailedConnectionAttempts.Increment() + return false, err + } + e.acceptQueue.pendingEndpoints[h.ep] = struct{}{} + + return false, nil + }() + if err != nil { + return err + } + if !useSynCookies { + return nil + } + + net := s.pkt.Network() + route, err := e.stack.FindRoute(s.pkt.NICID, net.DestinationAddress(), net.SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */) + if err != nil { + return err + } + defer route.Release() + + // Send SYN without window scaling because we currently + // don't encode this information in the cookie. + // + // Enable Timestamp option if the original syn did have + // the timestamp option specified. + // + // Use the user supplied MSS on the listening socket for + // new connections, if available. + synOpts := header.TCPSynOptions{ + WS: -1, + TS: opts.TS, + TSEcr: opts.TSVal, + MSS: calculateAdvertisedMSS(e.userMSS, route), + } + if opts.TS { + offset := e.protocol.tsOffset(net.DestinationAddress(), net.SourceAddress()) + now := e.stack.Clock().NowMonotonic() + synOpts.TSVal = offset.TSVal(now) + } + cookie := ctx.createCookie(s.id, s.sequenceNumber, encodeMSS(opts.MSS)) + fields := tcpFields{ + id: s.id, + ttl: calculateTTL(route, e.ipv4TTL, e.ipv6HopLimit), + tos: e.sendTOS, + flags: header.TCPFlagSyn | header.TCPFlagAck, + seq: cookie, + ack: s.sequenceNumber + 1, + rcvWnd: ctx.rcvWnd, + expOptVal: e.getExperimentOptionValue(route), + } + if err := e.sendSynTCP(route, fields, synOpts); err != nil { + return err + } + e.stack.Stats().TCP.ListenOverflowSynCookieSent.Increment() + return nil + + case s.flags.Contains(header.TCPFlagAck): + iss := s.ackNumber - 1 + irs := s.sequenceNumber - 1 + + // As an edge case when SYN-COOKIES are in use and we receive a + // segment that has data and is valid we should check if it + // already matches a created endpoint and redirect the segment + // rather than try and create a new endpoint. This can happen + // where the final ACK for the handshake and other data packets + // arrive at the same time and are queued to the listening + // endpoint before the listening endpoint has had time to + // process the first ACK and create the endpoint that matches + // the incoming packet's full 5 tuple. + netProtos := []tcpip.NetworkProtocolNumber{s.pkt.NetworkProtocolNumber} + // If the local address is an IPv4 Address then also look for IPv6 + // dual stack endpoints. + if s.id.LocalAddress.To4() != (tcpip.Address{}) { + netProtos = []tcpip.NetworkProtocolNumber{header.IPv4ProtocolNumber, header.IPv6ProtocolNumber} + } + for _, netProto := range netProtos { + if newEP := e.stack.FindTransportEndpoint(netProto, ProtocolNumber, s.id, s.pkt.NICID); newEP != nil && newEP != e { + tcpEP := newEP.(*Endpoint) + if !tcpEP.EndpointState().connected() { + continue + } + if !tcpEP.enqueueSegment(s) { + // Just silently drop the segment as we failed + // to queue, we don't want to generate a RST + // further below or try and create a new + // endpoint etc. + return nil + } + tcpEP.notifyProcessor() + return nil + } + } + + // Since SYN cookies are in use this is potentially an ACK to a + // SYN-ACK we sent but don't have a half open connection state + // as cookies are being used to protect against a potential SYN + // flood. In such cases validate the cookie and if valid create + // a fully connected endpoint and deliver to the accept queue. + // + // If not, silently drop the ACK to avoid leaking information + // when under a potential syn flood attack. + // + // Validate the cookie. + data, ok := ctx.isCookieValid(s.id, iss, irs) + if !ok || int(data) >= len(mssTable) { + e.stack.Stats().TCP.ListenOverflowInvalidSynCookieRcvd.Increment() + e.stack.Stats().DroppedPackets.Increment() + + // When not using SYN cookies, as per RFC 793, section 3.9, page 64: + // Any acknowledgment is bad if it arrives on a connection still in + // the LISTEN state. An acceptable reset segment should be formed + // for any arriving ACK-bearing segment. The RST should be + // formatted as follows: + // + // + // + // Send a reset as this is an ACK for which there is no + // half open connections and we are not using cookies + // yet. + // + // The only time we should reach here when a connection + // was opened and closed really quickly and a delayed + // ACK was received from the sender. + return replyWithReset(e.stack, s, e.sendTOS, e.ipv4TTL, e.ipv6HopLimit) + } + + // Keep hold of acceptMu until the new endpoint is in the accept queue (or + // if there is an error), to guarantee that we will keep our spot in the + // queue even if another handshake from the syn queue completes. + e.acceptMu.Lock() + if e.acceptQueue.isFull() { + // Silently drop the ack as the application can't accept + // the connection at this point. The ack will be + // retransmitted by the sender anyway and we can + // complete the connection at the time of retransmit if + // the backlog has space. + e.acceptMu.Unlock() + e.stack.Stats().TCP.ListenOverflowAckDrop.Increment() + e.stats.ReceiveErrors.ListenOverflowAckDrop.Increment() + e.stack.Stats().DroppedPackets.Increment() + return nil + } + + e.stack.Stats().TCP.ListenOverflowSynCookieRcvd.Increment() + // Create newly accepted endpoint and deliver it. + rcvdSynOptions := header.TCPSynOptions{ + MSS: mssTable[data], + // Disable Window scaling as original SYN is + // lost. + WS: -1, + } + + // When syn cookies are in use we enable timestamp only + // if the ack specifies the timestamp option assuming + // that the other end did in fact negotiate the + // timestamp option in the original SYN. + if s.parsedOptions.TS { + rcvdSynOptions.TS = true + rcvdSynOptions.TSVal = s.parsedOptions.TSVal + rcvdSynOptions.TSEcr = s.parsedOptions.TSEcr + } + + n, err := ctx.createConnectingEndpoint(s, rcvdSynOptions, &waiter.Queue{}) + if err != nil { + e.acceptMu.Unlock() + return err + } + + // Propagate any inheritable options from the listening endpoint + // to the newly created endpoint. + e.propagateInheritableOptionsLocked(n) + + if !n.reserveTupleLocked() { + n.mu.Unlock() + e.acceptMu.Unlock() + n.Close() + + e.stack.Stats().TCP.FailedConnectionAttempts.Increment() + e.stats.FailedConnectionAttempts.Increment() + return nil + } + + // Register new endpoint so that packets are routed to it. + if err := n.stack.RegisterTransportEndpoint( + n.effectiveNetProtos, + ProtocolNumber, + n.TransportEndpointInfo.ID, + n, + n.boundPortFlags, + n.boundBindToDevice, + ); err != nil { + n.mu.Unlock() + e.acceptMu.Unlock() + n.Close() + + e.stack.Stats().TCP.FailedConnectionAttempts.Increment() + e.stats.FailedConnectionAttempts.Increment() + return err + } + + n.isRegistered = true + net := s.pkt.Network() + n.TSOffset = n.protocol.tsOffset(net.DestinationAddress(), net.SourceAddress()) + + // Switch state to connected. + n.isConnectNotified = true + h := handshake{ + ep: n, + iss: iss, + ackNum: irs + 1, + rcvWnd: seqnum.Size(n.initialReceiveWindow()), + sndWnd: s.window, + rcvWndScale: e.rcvWndScaleForHandshake(), + sndWndScale: rcvdSynOptions.WS, + mss: rcvdSynOptions.MSS, + sampleRTTWithTSOnly: true, + } + h.ep.AssertLockHeld(n) + h.transitionToStateEstablishedLocked(s) + n.mu.Unlock() + + // Requeue the segment if the ACK completing the handshake has more info + // to be processed by the newly established endpoint. + if (s.flags.Contains(header.TCPFlagFin) || s.payloadSize() > 0) && n.enqueueSegment(s) { + n.notifyProcessor() + } + + e.stack.Stats().TCP.PassiveConnectionOpenings.Increment() + + // Deliver the endpoint to the accept queue. + e.acceptQueue.endpoints.PushBack(n) + e.acceptMu.Unlock() + + e.waiterQueue.Notify(waiter.ReadableEvents) + return nil + + default: + e.stack.Stats().DroppedPackets.Increment() + return nil + } +} diff --git a/pkg/tcpip/transport/tcp/accept_mutex.go b/pkg/tcpip/transport/tcp/accept_mutex.go new file mode 100644 index 0000000..a8147bb --- /dev/null +++ b/pkg/tcpip/transport/tcp/accept_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type acceptMutex struct { + mu sync.Mutex +} + +var acceptprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var acceptlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type acceptlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *acceptMutex) Lock() { + locking.AddGLock(acceptprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *acceptMutex) NestedLock(i acceptlockNameIndex) { + locking.AddGLock(acceptprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *acceptMutex) Unlock() { + locking.DelGLock(acceptprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *acceptMutex) NestedUnlock(i acceptlockNameIndex) { + locking.DelGLock(acceptprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func acceptinitLockNames() {} + +func init() { + acceptinitLockNames() + acceptprefixIndex = locking.NewMutexClass(reflect.TypeOf(acceptMutex{}), acceptlockNames) +} diff --git a/pkg/tcpip/transport/tcp/connect.go b/pkg/tcpip/transport/tcp/connect.go new file mode 100644 index 0000000..b6bbd49 --- /dev/null +++ b/pkg/tcpip/transport/tcp/connect.go @@ -0,0 +1,1532 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "math" + "time" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +const ( + // tcpMinTimeout is the minimum timeout for a SYN retransmit. + // This mirrors the TCP_TIMEOUT_MIN variable in Linux. + // See: https://github.com/torvalds/linux/blob/249aca0d3d631660aa3583c6a3559b75b6e971b4/include/net/tcp.h#L143 + tcpMinTimeout = 2 * time.Microsecond + + // InitialRTO is the initial retransmission timeout. + // https://github.com/torvalds/linux/blob/7c636d4d20f/include/net/tcp.h#L142 + InitialRTO = time.Second + + // maxSegmentsPerWake is the maximum number of segments to process per + // wake-up. Yielding [after this number of segments are processed] + // allows other events to be processed as well (e.g., timeouts, resets, + // etc.). + maxSegmentsPerWake = 100 +) + +type handshakeState int + +// The following are the possible states of the TCP connection during a 3-way +// handshake. A depiction of the states and transitions can be found in RFC 793, +// page 23. +const ( + handshakeSynSent handshakeState = iota + handshakeSynRcvd + handshakeCompleted +) + +const ( + // Maximum space available for options. + maxOptionSize = 40 +) + +// handshake holds the state used during a TCP 3-way handshake. +// +// NOTE: handshake.ep.mu is held during handshake processing. It is released if +// we are going to block and reacquired when we start processing an event. +// +// +stateify savable +type handshake struct { + ep *Endpoint + listenEP *Endpoint + state handshakeState + active bool + flags header.TCPFlags + ackNum seqnum.Value + + // iss is the initial send sequence number, as defined in RFC 793. + iss seqnum.Value + + // rcvWnd is the receive window, as defined in RFC 793. + rcvWnd seqnum.Size + + // sndWnd is the send window, as defined in RFC 793. + sndWnd seqnum.Size + + // mss is the maximum segment size received from the peer. + mss uint16 + + // sndWndScale is the send window scale, as defined in RFC 1323. A + // negative value means no scaling is supported by the peer. + sndWndScale int + + // rcvWndScale is the receive window scale, as defined in RFC 1323. + rcvWndScale int + + // startTime is the time at which the first SYN/SYN-ACK was sent. + startTime tcpip.MonotonicTime + + // deferAccept if non-zero will drop the final ACK for a passive + // handshake till an ACK segment with data is received or the timeout is + // hit. + deferAccept time.Duration + + // acked is true if the final ACK for a 3-way handshake has + // been received. This is required to stop retransmitting the + // original SYN-ACK when deferAccept is enabled. + acked bool + + // sendSYNOpts is the cached values for the SYN options to be sent. + sendSYNOpts header.TCPSynOptions + + // sampleRTTWithTSOnly is true when the segment was retransmitted or we can't + // tell; then RTT can only be sampled when the incoming segment has timestamp + // options enabled. + sampleRTTWithTSOnly bool + + // retransmitTimer is used to retransmit SYN/SYN-ACK with exponential backoff + // till handshake is either completed or timesout. + retransmitTimer *backoffTimer `state:"nosave"` +} + +// timerHandler takes a handler function for a timer and returns a function that +// will invoke the provided handler with the endpoint mutex held. In addition +// the returned function will perform any cleanup that may be required if the +// timer handler returns an error. In the case of no errors it will notify the +// processor if there are pending segments that need to be processed. +// +// NOTE: e.mu is held for the duration of the call to f(). +func timerHandler(e *Endpoint, f func() tcpip.Error) func() { + return func() { + e.mu.Lock() + if err := f(); err != nil { + e.lastErrorMu.Lock() + // If the handler timed out and we have a lastError recorded (maybe due + // to an ICMP message received), promote it to be the hard error. + if _, isTimeout := err.(*tcpip.ErrTimeout); e.lastError != nil && isTimeout { + e.hardError = e.lastError + } else { + e.hardError = err + } + e.lastError = err + e.lastErrorMu.Unlock() + e.cleanupLocked() + e.setEndpointState(StateError) + e.mu.Unlock() + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + return + } + processor := e.protocol.dispatcher.selectProcessor(e.ID) + e.mu.Unlock() + + // notify processor if there are pending segments to be + // processed. + if !e.segmentQueue.empty() { + processor.queueEndpoint(e) + } + } +} + +// +checklocks:e.mu +// +checklocksacquire:h.ep.mu +func (e *Endpoint) newHandshake() (h *handshake) { + h = &handshake{ + ep: e, + active: true, + rcvWnd: seqnum.Size(e.initialReceiveWindow()), + rcvWndScale: e.rcvWndScaleForHandshake(), + } + h.ep.AssertLockHeld(e) + h.resetState() + // Store reference to handshake state in endpoint. + e.h = h + // By the time handshake is created, e.ID is already initialized. + e.TSOffset = e.protocol.tsOffset(e.ID.LocalAddress, e.ID.RemoteAddress) + timer, err := newBackoffTimer(h.ep.stack.Clock(), InitialRTO, MaxRTO, timerHandler(e, h.retransmitHandlerLocked)) + if err != nil { + panic(fmt.Sprintf("newBackOffTimer(_, %s, %s, _) failed: %s", InitialRTO, MaxRTO, err)) + } + h.retransmitTimer = timer + return h +} + +// +checklocks:e.mu +// +checklocksacquire:h.ep.mu +func (e *Endpoint) newPassiveHandshake(isn, irs seqnum.Value, opts header.TCPSynOptions, deferAccept time.Duration) (h *handshake) { + h = e.newHandshake() + h.resetToSynRcvd(isn, irs, opts, deferAccept) + return h +} + +// FindWndScale determines the window scale to use for the given maximum window +// size. +func FindWndScale(wnd seqnum.Size) int { + if wnd < 0x10000 { + return 0 + } + + max := seqnum.Size(math.MaxUint16) + s := 0 + for wnd > max && s < header.MaxWndScale { + s++ + max <<= 1 + } + + return s +} + +// resetState resets the state of the handshake object such that it becomes +// ready for a new 3-way handshake. +func (h *handshake) resetState() { + h.state = handshakeSynSent + h.flags = header.TCPFlagSyn + h.ackNum = 0 + h.mss = 0 + h.iss = generateSecureISN(h.ep.TransportEndpointInfo.ID, h.ep.stack.Clock(), h.ep.protocol.seqnumSecret) +} + +// generateSecureISN generates a secure Initial Sequence number based on the +// recommendation here https://tools.ietf.org/html/rfc6528#page-3. +func generateSecureISN(id stack.TransportEndpointID, clock tcpip.Clock, seed [16]byte) seqnum.Value { + isnHasher := sha256.New() + + // Per hash.Hash.Writer: + // + // It never returns an error. + _, _ = isnHasher.Write(seed[:]) + _, _ = isnHasher.Write(id.LocalAddress.AsSlice()) + _, _ = isnHasher.Write(id.RemoteAddress.AsSlice()) + portBuf := make([]byte, 2) + binary.LittleEndian.PutUint16(portBuf, id.LocalPort) + _, _ = isnHasher.Write(portBuf) + binary.LittleEndian.PutUint16(portBuf, id.RemotePort) + _, _ = isnHasher.Write(portBuf) + // The time period here is 64ns. This is similar to what linux uses + // generate a sequence number that overlaps less than one + // time per MSL (2 minutes). + // + // A 64ns clock ticks 10^9/64 = 15625000) times in a second. + // To wrap the whole 32 bit space would require + // 2^32/1562500 ~ 274 seconds. + // + // Which sort of guarantees that we won't reuse the ISN for a new + // connection for the same tuple for at least 274s. + hash := binary.LittleEndian.Uint32(isnHasher.Sum(nil)[:4]) + isn := hash + uint32(clock.NowMonotonic().Sub(tcpip.MonotonicTime{}).Nanoseconds()>>6) + return seqnum.Value(isn) +} + +// effectiveRcvWndScale returns the effective receive window scale to be used. +// If the peer doesn't support window scaling, the effective rcv wnd scale is +// zero; otherwise it's the value calculated based on the initial rcv wnd. +func (h *handshake) effectiveRcvWndScale() uint8 { + if h.sndWndScale < 0 { + return 0 + } + return uint8(h.rcvWndScale) +} + +// resetToSynRcvd resets the state of the handshake object to the SYN-RCVD +// state. +// +checklocks:h.ep.mu +func (h *handshake) resetToSynRcvd(iss seqnum.Value, irs seqnum.Value, opts header.TCPSynOptions, deferAccept time.Duration) { + h.active = false + h.state = handshakeSynRcvd + h.flags = header.TCPFlagSyn | header.TCPFlagAck + h.iss = iss + h.ackNum = irs + 1 + h.mss = opts.MSS + h.sndWndScale = opts.WS + h.deferAccept = deferAccept + h.ep.setEndpointState(StateSynRecv) +} + +// checkAck checks if the ACK number, if present, of a segment received during +// a TCP 3-way handshake is valid. +func (h *handshake) checkAck(s *segment) bool { + return !(s.flags.Contains(header.TCPFlagAck) && s.ackNumber != h.iss+1) +} + +// synSentState handles a segment received when the TCP 3-way handshake is in +// the SYN-SENT state. +// +checklocks:h.ep.mu +func (h *handshake) synSentState(s *segment) tcpip.Error { + // RFC 793, page 37, states that in the SYN-SENT state, a reset is + // acceptable if the ack field acknowledges the SYN. + if s.flags.Contains(header.TCPFlagRst) { + if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == h.iss+1 { + // RFC 793, page 67, states that "If the RST bit is set [and] If the ACK + // was acceptable then signal the user "error: connection reset", drop + // the segment, enter CLOSED state, delete TCB, and return." + // Although the RFC above calls out ECONNRESET, Linux actually returns + // ECONNREFUSED here so we do as well. + return &tcpip.ErrConnectionRefused{} + } + return nil + } + + if !h.checkAck(s) { + // RFC 793, page 72 (https://datatracker.ietf.org/doc/html/rfc793#page-72): + // If the segment acknowledgment is not acceptable, form a reset segment, + // + // and send it. + h.ep.sendEmptyRaw(header.TCPFlagRst, s.ackNumber, 0, 0) + // Since this was a challenge ACK reschedule the retransmit timer to fire + // soon so that the SYN is retransmitted quickly. + h.retransmitTimer.reinit(tcpMinTimeout) + return nil + } + + // We are in the SYN-SENT state. We only care about segments that have + // the SYN flag. + if !s.flags.Contains(header.TCPFlagSyn) { + return nil + } + + // Parse the SYN options. + rcvSynOpts := parseSynSegmentOptions(s) + + // Remember if the Timestamp option was negotiated. + h.ep.maybeEnableTimestamp(rcvSynOpts) + + // Remember if the SACKPermitted option was negotiated. + h.ep.maybeEnableSACKPermitted(rcvSynOpts) + + // Remember the sequence we'll ack from now on. + h.ackNum = s.sequenceNumber + 1 + h.flags |= header.TCPFlagAck + h.mss = rcvSynOpts.MSS + h.sndWndScale = rcvSynOpts.WS + + // If this is a SYN ACK response, we only need to acknowledge the SYN + // and the handshake is completed. + if s.flags.Contains(header.TCPFlagAck) { + h.state = handshakeCompleted + h.transitionToStateEstablishedLocked(s) + + h.ep.sendEmptyRaw(header.TCPFlagAck, h.iss+1, h.ackNum, h.rcvWnd>>h.effectiveRcvWndScale()) + return nil + } + + // A SYN segment was received, but no ACK in it. We acknowledge the SYN + // but resend our own SYN and wait for it to be acknowledged in the + // SYN-RCVD state. + h.state = handshakeSynRcvd + ttl := calculateTTL(h.ep.route, h.ep.ipv4TTL, h.ep.ipv6HopLimit) + amss := h.ep.amss + h.ep.setEndpointState(StateSynRecv) + synOpts := header.TCPSynOptions{ + WS: int(h.effectiveRcvWndScale()), + TS: rcvSynOpts.TS, + TSVal: h.ep.tsValNow(), + TSEcr: h.ep.recentTimestamp(), + + // We only send SACKPermitted if the other side indicated it + // permits SACK. This is not explicitly defined in the RFC but + // this is the behaviour implemented by Linux. + SACKPermitted: rcvSynOpts.SACKPermitted, + MSS: amss, + } + if ttl == 0 { + ttl = h.ep.route.DefaultTTL() + } + h.ep.sendSynTCP(h.ep.route, tcpFields{ + id: h.ep.TransportEndpointInfo.ID, + ttl: ttl, + tos: h.ep.sendTOS, + flags: h.flags, + seq: h.iss, + ack: h.ackNum, + rcvWnd: h.rcvWnd, + expOptVal: h.ep.getExperimentOptionValue(h.ep.route), + }, synOpts) + return nil +} + +// synRcvdState handles a segment received when the TCP 3-way handshake is in +// the SYN-RCVD state. +// +checklocks:h.ep.mu +func (h *handshake) synRcvdState(s *segment) tcpip.Error { + if s.flags.Contains(header.TCPFlagRst) { + // RFC 793, page 37, states that in the SYN-RCVD state, a reset + // is acceptable if the sequence number is in the window. + if s.sequenceNumber.InWindow(h.ackNum, h.rcvWnd) { + return &tcpip.ErrConnectionRefused{} + } + return nil + } + + // It's possible that s is an ACK of a SYN cookie. This can happen if: + // + // - We receive a SYN while under load and issue a SYN/ACK with + // cookie S. + // - We receive a retransmitted SYN while space exists in the SYN + // queue, and issue a SYN/ACK with seqnum S'. + // - We receive the ACK based on S. + // + // If we receive a SYN cookie ACK, just use the cookie seqnum. + if !h.checkAck(s) && h.listenEP != nil { + iss := s.ackNumber - 1 + data, ok := h.listenEP.listenCtx.isCookieValid(s.id, iss, s.sequenceNumber-1) + if !ok || int(data) >= len(mssTable) { + // This isn't a valid cookie. + // RFC 793, page 72 (https://datatracker.ietf.org/doc/html/rfc793#page-72): + // If the segment acknowledgment is not acceptable, form a reset segment, + // + // and send it. + h.ep.sendEmptyRaw(header.TCPFlagRst, s.ackNumber, 0, 0) + return nil + } + // This is a cookie that snuck its way in after we stopped using them. + h.mss = mssTable[data] + h.iss = iss + } + + // RFC 793, Section 3.9, page 69, states that in the SYN-RCVD state, a + // sequence number outside of the window causes an ACK with the proper seq + // number and "After sending the acknowledgment, drop the unacceptable + // segment and return." + if !s.sequenceNumber.InWindow(h.ackNum, h.rcvWnd) { + if h.ep.allowOutOfWindowAck() { + h.ep.sendEmptyRaw(header.TCPFlagAck, h.iss+1, h.ackNum, h.rcvWnd) + } + return nil + } + + if s.flags.Contains(header.TCPFlagSyn) && s.sequenceNumber != h.ackNum-1 { + // We received two SYN segments with different sequence + // numbers, so we reset this and restart the whole + // process, except that we don't reset the timer. + ack := s.sequenceNumber.Add(s.logicalLen()) + seq := seqnum.Value(0) + if s.flags.Contains(header.TCPFlagAck) { + seq = s.ackNumber + } + h.ep.sendEmptyRaw(header.TCPFlagRst|header.TCPFlagAck, seq, ack, 0) + + if !h.active { + return &tcpip.ErrInvalidEndpointState{} + } + + h.resetState() + synOpts := header.TCPSynOptions{ + WS: h.rcvWndScale, + TS: h.ep.SendTSOk, + TSVal: h.ep.tsValNow(), + TSEcr: h.ep.recentTimestamp(), + SACKPermitted: h.ep.SACKPermitted, + MSS: h.ep.amss, + } + h.ep.sendSynTCP(h.ep.route, tcpFields{ + id: h.ep.TransportEndpointInfo.ID, + ttl: calculateTTL(h.ep.route, h.ep.ipv4TTL, h.ep.ipv6HopLimit), + tos: h.ep.sendTOS, + flags: h.flags, + seq: h.iss, + ack: h.ackNum, + rcvWnd: h.rcvWnd, + expOptVal: h.ep.getExperimentOptionValue(h.ep.route), + }, synOpts) + return nil + } + + // We have previously received (and acknowledged) the peer's SYN. If the + // peer acknowledges our SYN, the handshake is completed. + if s.flags.Contains(header.TCPFlagAck) { + // If deferAccept is not zero and this is a bare ACK and the + // timeout is not hit then drop the ACK. + if h.deferAccept != 0 && s.payloadSize() == 0 && h.ep.stack.Clock().NowMonotonic().Sub(h.startTime) < h.deferAccept { + h.acked = true + h.ep.stack.Stats().DroppedPackets.Increment() + return nil + } + + // If the timestamp option is negotiated and the segment does + // not carry a timestamp option then the segment must be dropped + // as per https://tools.ietf.org/html/rfc7323#section-3.2. + if h.ep.SendTSOk && !s.parsedOptions.TS { + h.ep.stack.Stats().DroppedPackets.Increment() + return nil + } + + // Drop the ACK if the accept queue is full. + // https://github.com/torvalds/linux/blob/7acac4b3196/net/ipv4/tcp_ipv4.c#L1523 + // We could abort the connection as well with a tunable as in + // https://github.com/torvalds/linux/blob/7acac4b3196/net/ipv4/tcp_minisocks.c#L788 + if listenEP := h.listenEP; listenEP != nil && listenEP.acceptQueueIsFull() { + listenEP.stack.Stats().DroppedPackets.Increment() + return nil + } + + // Update timestamp if required. See RFC7323, section-4.3. + if h.ep.SendTSOk && s.parsedOptions.TS { + h.ep.updateRecentTimestamp(s.parsedOptions.TSVal, h.ackNum, s.sequenceNumber) + } + + h.state = handshakeCompleted + h.transitionToStateEstablishedLocked(s) + + // Requeue the segment if the ACK completing the handshake has more info + // to be processed by the newly established endpoint. + if (s.flags.Contains(header.TCPFlagFin) || s.payloadSize() > 0) && h.ep.enqueueSegment(s) { + h.ep.protocol.dispatcher.selectProcessor(h.ep.ID).queueEndpoint(h.ep) + } + return nil + } + + return nil +} + +// +checklocks:h.ep.mu +func (h *handshake) handleSegment(s *segment) tcpip.Error { + h.sndWnd = s.window + if !s.flags.Contains(header.TCPFlagSyn) && h.sndWndScale > 0 { + h.sndWnd <<= uint8(h.sndWndScale) + } + + switch h.state { + case handshakeSynRcvd: + return h.synRcvdState(s) + case handshakeSynSent: + return h.synSentState(s) + } + return nil +} + +// processSegments goes through the segment queue and processes up to +// maxSegmentsPerWake (if they're available). +// +checklocks:h.ep.mu +func (h *handshake) processSegments() tcpip.Error { + for i := 0; i < maxSegmentsPerWake; i++ { + s := h.ep.segmentQueue.dequeue() + if s == nil { + return nil + } + + err := h.handleSegment(s) + s.DecRef() + if err != nil { + return err + } + + // We stop processing packets once the handshake is completed, + // otherwise we may process packets meant to be processed by + // the TCP processor goroutine. + if h.state == handshakeCompleted { + break + } + } + + return nil +} + +// start sends the first SYN/SYN-ACK. It does not block, even if link address +// resolution is required. +func (h *handshake) start() { + h.startTime = h.ep.stack.Clock().NowMonotonic() + h.ep.amss = calculateAdvertisedMSS(h.ep.userMSS, h.ep.route) + var sackEnabled tcpip.TCPSACKEnabled + if err := h.ep.stack.TransportProtocolOption(ProtocolNumber, &sackEnabled); err != nil { + // If stack returned an error when checking for SACKEnabled + // status then just default to switching off SACK negotiation. + sackEnabled = false + } + + synOpts := header.TCPSynOptions{ + WS: h.rcvWndScale, + TS: true, + TSVal: h.ep.tsValNow(), + TSEcr: h.ep.recentTimestamp(), + SACKPermitted: bool(sackEnabled), + MSS: h.ep.amss, + } + + // start() is also called in a listen context so we want to make sure we only + // send the TS/SACK option when we received the TS/SACK in the initial SYN. + if h.state == handshakeSynRcvd { + synOpts.TS = h.ep.SendTSOk + synOpts.SACKPermitted = h.ep.SACKPermitted && bool(sackEnabled) + if h.sndWndScale < 0 { + // Disable window scaling if the peer did not send us + // the window scaling option. + synOpts.WS = -1 + } + } + + h.sendSYNOpts = synOpts + h.ep.sendSynTCP(h.ep.route, tcpFields{ + id: h.ep.TransportEndpointInfo.ID, + ttl: calculateTTL(h.ep.route, h.ep.ipv4TTL, h.ep.ipv6HopLimit), + tos: h.ep.sendTOS, + flags: h.flags, + seq: h.iss, + ack: h.ackNum, + rcvWnd: h.rcvWnd, + expOptVal: h.ep.getExperimentOptionValue(h.ep.route), + }, synOpts) +} + +// retransmitHandler handles retransmissions of un-acked SYNs. +// +checklocks:h.ep.mu +func (h *handshake) retransmitHandlerLocked() tcpip.Error { + e := h.ep + // If the endpoint has already transition out of a connecting state due + // to say an error (e.g) peer send RST or an ICMP error. Then just + // return. Any required cleanup should have been done when the RST/error + // was handled. + if !e.EndpointState().connecting() { + return nil + } + + if err := h.retransmitTimer.reset(); err != nil { + return err + } + + // Resend the SYN/SYN-ACK only if the following conditions hold. + // - It's an active handshake (deferAccept does not apply) + // - It's a passive handshake and we have not yet got the final-ACK. + // - It's a passive handshake and we got an ACK but deferAccept is + // enabled and we are now past the deferAccept duration. + // The last is required to provide a way for the peer to complete + // the connection with another ACK or data (as ACKs are never + // retransmitted on their own). + if h.active || !h.acked || h.deferAccept != 0 && e.stack.Clock().NowMonotonic().Sub(h.startTime) > h.deferAccept { + e.sendSynTCP(e.route, tcpFields{ + id: e.TransportEndpointInfo.ID, + ttl: calculateTTL(e.route, e.ipv4TTL, e.ipv6HopLimit), + tos: e.sendTOS, + flags: h.flags, + seq: h.iss, + ack: h.ackNum, + rcvWnd: h.rcvWnd, + expOptVal: e.getExperimentOptionValue(e.route), + }, h.sendSYNOpts) + // If we have ever retransmitted the SYN-ACK or + // SYN segment, we should only measure RTT if + // TS option is present. + h.sampleRTTWithTSOnly = true + } + return nil +} + +// transitionToStateEstablisedLocked transitions the endpoint of the handshake +// to an established state given the last segment received from peer. It also +// initializes sender/receiver. +// +checklocks:h.ep.mu +// +checklocksalias:h.ep.snd.ep.mu=h.ep.mu +func (h *handshake) transitionToStateEstablishedLocked(s *segment) { + // Stop the SYN retransmissions now that handshake is complete. + if h.retransmitTimer != nil { + h.retransmitTimer.stop() + } + + // Transfer handshake state to TCP connection. We disable + // receive window scaling if the peer doesn't support it + // (indicated by a negative send window scale). + h.ep.snd = newSender(h.ep, h.iss, h.ackNum-1, h.sndWnd, h.mss, h.sndWndScale) + + now := h.ep.stack.Clock().NowMonotonic() + + var rtt time.Duration + if h.ep.SendTSOk && s.parsedOptions.TSEcr != 0 { + rtt = h.ep.elapsed(now, s.parsedOptions.TSEcr) + } + if !h.sampleRTTWithTSOnly && rtt == 0 { + rtt = now.Sub(h.startTime) + } + + if rtt > 0 { + h.ep.snd.updateRTO(rtt) + } + + h.ep.rcvQueueMu.Lock() + h.ep.rcv = newReceiver(h.ep, h.ackNum-1, h.rcvWnd, h.effectiveRcvWndScale()) + // Bootstrap the auto tuning algorithm. Starting at zero will + // result in a really large receive window after the first auto + // tuning adjustment. + h.ep.RcvAutoParams.PrevCopiedBytes = int(h.rcvWnd) + h.ep.rcvQueueMu.Unlock() + + h.ep.setEndpointState(StateEstablished) + + // Completing the 3-way handshake is an indication that the route is valid + // and the remote is reachable as the only way we can complete a handshake + // is if our SYN reached the remote and their ACK reached us. + h.ep.route.ConfirmReachable() + + // Tell waiters that the endpoint is connected and writable. + h.ep.waiterQueue.Notify(waiter.WritableEvents) +} + +type backoffTimer struct { + timeout time.Duration + maxTimeout time.Duration + t tcpip.Timer +} + +func newBackoffTimer(clock tcpip.Clock, timeout, maxTimeout time.Duration, f func()) (*backoffTimer, tcpip.Error) { + if timeout > maxTimeout { + return nil, &tcpip.ErrTimeout{} + } + bt := &backoffTimer{timeout: timeout, maxTimeout: maxTimeout} + bt.t = clock.AfterFunc(timeout, f) + return bt, nil +} + +func (bt *backoffTimer) reset() tcpip.Error { + bt.timeout *= 2 + if bt.timeout > bt.maxTimeout { + return &tcpip.ErrTimeout{} + } + bt.t.Reset(bt.timeout) + return nil +} + +func (bt *backoffTimer) reinit(timeout time.Duration) { + bt.timeout = timeout + bt.t.Reset(bt.timeout) +} + +func (bt *backoffTimer) stop() { + bt.t.Stop() +} + +func parseSynSegmentOptions(s *segment) header.TCPSynOptions { + synOpts := header.ParseSynOptions(s.options, s.flags.Contains(header.TCPFlagAck)) + if synOpts.TS { + s.parsedOptions.TSVal = synOpts.TSVal + s.parsedOptions.TSEcr = synOpts.TSEcr + } + return synOpts +} + +var optionPool = sync.Pool{ + New: func() any { + return &[maxOptionSize]byte{} + }, +} + +func getOptions() []byte { + return (*optionPool.Get().(*[maxOptionSize]byte))[:] +} + +func putOptions(options []byte) { + // Reslice to full capacity. + optionPool.Put(optionsToArray(options)) +} + +func makeSynOptions(opts header.TCPSynOptions) []byte { + // Emulate linux option order. This is as follows: + // + // if md5: NOP NOP MD5SIG 18 md5sig(16) + // if mss: MSS 4 mss(2) + // if ts and sack_advertise: + // SACK 2 TIMESTAMP 2 timestamp(8) + // elif ts: NOP NOP TIMESTAMP 10 timestamp(8) + // elif sack: NOP NOP SACK 2 + // if wscale: NOP WINDOW 3 ws(1) + // if sack_blocks: NOP NOP SACK ((2 + (#blocks * 8)) + // [for each block] start_seq(4) end_seq(4) + // if fastopen_cookie: + // if exp: EXP (4 + len(cookie)) FASTOPEN_MAGIC(2) + // else: FASTOPEN (2 + len(cookie)) + // cookie(variable) [padding to four bytes] + // + options := getOptions() + + // Always encode the mss. + offset := header.EncodeMSSOption(uint32(opts.MSS), options) + + // Special ordering is required here. If both TS and SACK are enabled, + // then the SACK option precedes TS, with no padding. If they are + // enabled individually, then we see padding before the option. + if opts.TS && opts.SACKPermitted { + offset += header.EncodeSACKPermittedOption(options[offset:]) + offset += header.EncodeTSOption(opts.TSVal, opts.TSEcr, options[offset:]) + } else if opts.TS { + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeTSOption(opts.TSVal, opts.TSEcr, options[offset:]) + } else if opts.SACKPermitted { + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeSACKPermittedOption(options[offset:]) + } + + // Initialize the WS option. + if opts.WS >= 0 { + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeWSOption(opts.WS, options[offset:]) + } + + // Padding to the end; note that this never apply unless we add a + // fastopen option, we always expect the offset to remain the same. + if delta := header.AddTCPOptionPadding(options, offset); delta != 0 { + panic("unexpected option encoding") + } + + return options[:offset] +} + +// tcpFields is a struct to carry different parameters required by the +// send*TCP variant functions below. +type tcpFields struct { + id stack.TransportEndpointID + ttl uint8 + tos uint8 + flags header.TCPFlags + seq seqnum.Value + ack seqnum.Value + rcvWnd seqnum.Size + opts []byte + txHash uint32 + df bool + expOptVal uint16 +} + +func (e *Endpoint) sendSynTCP(r *stack.Route, tf tcpFields, opts header.TCPSynOptions) tcpip.Error { + tf.opts = makeSynOptions(opts) + // We ignore SYN send errors and let the callers re-attempt send. + hdrSize := header.TCPMinimumSize + int(r.MaxHeaderLength()) + len(tf.opts) + if r.NetProto() == header.IPv6ProtocolNumber && tf.expOptVal != 0 { + hdrSize += header.IPv6ExperimentHdrLength + } + p := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize}) + defer p.DecRef() + if err := e.sendTCP(r, tf, p, stack.GSO{}); err != nil { + e.stats.SendErrors.SynSendToNetworkFailed.Increment() + } + putOptions(tf.opts) + return nil +} + +// This method takes ownership of pkt. +func (e *Endpoint) sendTCP(r *stack.Route, tf tcpFields, pkt *stack.PacketBuffer, gso stack.GSO) tcpip.Error { + tf.txHash = e.txHash + if err := sendTCP(r, tf, pkt, gso, e.owner); err != nil { + e.stats.SendErrors.SegmentSendToNetworkFailed.Increment() + return err + } + e.stats.SegmentsSent.Increment() + return nil +} + +func buildTCPHdr(r *stack.Route, tf tcpFields, pkt *stack.PacketBuffer, gso stack.GSO) { + optLen := len(tf.opts) + tcp := header.TCP(pkt.TransportHeader().Push(header.TCPMinimumSize + optLen)) + pkt.TransportProtocolNumber = header.TCPProtocolNumber + tcp.Encode(&header.TCPFields{ + SrcPort: tf.id.LocalPort, + DstPort: tf.id.RemotePort, + SeqNum: uint32(tf.seq), + AckNum: uint32(tf.ack), + DataOffset: uint8(header.TCPMinimumSize + optLen), + Flags: tf.flags, + WindowSize: uint16(tf.rcvWnd), + }) + copy(tcp[header.TCPMinimumSize:], tf.opts) + + xsum := r.PseudoHeaderChecksum(ProtocolNumber, uint16(pkt.Size())) + // Only calculate the checksum if offloading isn't supported. + if gso.Type != stack.GSONone && gso.NeedsCsum { + // This is called CHECKSUM_PARTIAL in the Linux kernel. We + // calculate a checksum of the pseudo-header and save it in the + // TCP header, then the kernel calculate a checksum of the + // header and data and get the right sum of the TCP packet. + tcp.SetChecksum(xsum) + } else if r.RequiresTXTransportChecksum() { + xsum = checksum.Combine(xsum, pkt.Data().Checksum()) + tcp.SetChecksum(^tcp.CalculateChecksum(xsum)) + } +} + +func sendTCPBatch(r *stack.Route, tf tcpFields, pkt *stack.PacketBuffer, gso stack.GSO, owner tcpip.PacketOwner) tcpip.Error { + optLen := len(tf.opts) + if tf.rcvWnd > math.MaxUint16 { + tf.rcvWnd = math.MaxUint16 + } + + mss := int(gso.MSS) + n := (pkt.Data().Size() + mss - 1) / mss + + size := pkt.Data().Size() + hdrSize := header.TCPMinimumSize + int(r.MaxHeaderLength()) + optLen + for i := 0; i < n; i++ { + packetSize := mss + if packetSize > size { + packetSize = size + } + size -= packetSize + + pkt := pkt + // No need to split the packet in the final iteration. The original + // packet already has the truncated data. + shouldSplitPacket := i != n-1 + if shouldSplitPacket { + if r.NetProto() == header.IPv6ProtocolNumber && tf.expOptVal != 0 { + // Reserve extra bytes for the experiment option. + hdrSize += header.IPv6ExperimentHdrLength + } + splitPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize}) + splitPkt.Data().ReadFromPacketData(pkt.Data(), packetSize) + pkt = splitPkt + } + pkt.Hash = tf.txHash + pkt.Owner = owner + + buildTCPHdr(r, tf, pkt, gso) + tf.seq = tf.seq.Add(seqnum.Size(packetSize)) + pkt.GSOOptions = gso + if err := r.WritePacket(stack.NetworkHeaderParams{ + Protocol: ProtocolNumber, + TTL: tf.ttl, + TOS: tf.tos, + DF: tf.df, + ExperimentOptionValue: tf.expOptVal, + }, pkt); err != nil { + r.Stats().TCP.SegmentSendErrors.Increment() + if shouldSplitPacket { + pkt.DecRef() + } + return err + } + r.Stats().TCP.SegmentsSent.Increment() + if shouldSplitPacket { + pkt.DecRef() + } + } + return nil +} + +// sendTCP sends a TCP segment with the provided options via the provided +// network endpoint and under the provided identity. This method takes +// ownership of pkt. +func sendTCP(r *stack.Route, tf tcpFields, pkt *stack.PacketBuffer, gso stack.GSO, owner tcpip.PacketOwner) tcpip.Error { + if tf.rcvWnd > math.MaxUint16 { + tf.rcvWnd = math.MaxUint16 + } + + if r.Loop()&stack.PacketLoop == 0 && gso.Type == stack.GSOGvisor && int(gso.MSS) < pkt.Data().Size() { + return sendTCPBatch(r, tf, pkt, gso, owner) + } + + pkt.GSOOptions = gso + pkt.Hash = tf.txHash + pkt.Owner = owner + buildTCPHdr(r, tf, pkt, gso) + + if err := r.WritePacket(stack.NetworkHeaderParams{ + Protocol: ProtocolNumber, + TTL: tf.ttl, + TOS: tf.tos, + DF: tf.df, + ExperimentOptionValue: tf.expOptVal, + }, pkt); err != nil { + r.Stats().TCP.SegmentSendErrors.Increment() + return err + } + r.Stats().TCP.SegmentsSent.Increment() + if (tf.flags & header.TCPFlagRst) != 0 { + r.Stats().TCP.ResetsSent.Increment() + } + return nil +} + +// makeOptions makes an options slice. +func (e *Endpoint) makeOptions(sackBlocks []header.SACKBlock) []byte { + options := getOptions() + offset := 0 + + // N.B. the ordering here matches the ordering used by Linux internally + // and described in the raw makeOptions function. We don't include + // unnecessary cases here (post connection.) + if e.SendTSOk { + // Embed the timestamp if timestamp has been enabled. + // + // We only use the lower 32 bits of the unix time in + // milliseconds. This is similar to what Linux does where it + // uses the lower 32 bits of the jiffies value in the tsVal + // field of the timestamp option. + // + // Further, RFC7323 section-5.4 recommends millisecond + // resolution as the lowest recommended resolution for the + // timestamp clock. + // + // Ref: https://tools.ietf.org/html/rfc7323#section-5.4. + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeTSOption(e.tsValNow(), e.recentTimestamp(), options[offset:]) + } + if e.SACKPermitted && len(sackBlocks) > 0 { + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeNOP(options[offset:]) + offset += header.EncodeSACKBlocks(sackBlocks, options[offset:]) + } + + // We expect the above to produce an aligned offset. + if delta := header.AddTCPOptionPadding(options, offset); delta != 0 { + panic("unexpected option encoding") + } + + return options[:offset] +} + +// sendEmptyRaw sends a TCP segment with no payload to the endpoint's peer. +// +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) sendEmptyRaw(flags header.TCPFlags, seq, ack seqnum.Value, rcvWnd seqnum.Size) tcpip.Error { + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{}) + defer pkt.DecRef() + return e.sendRaw(pkt, flags, seq, ack, rcvWnd) +} + +// sendRaw sends a TCP segment to the endpoint's peer. This method takes +// ownership of pkt. pkt must not have any headers set. +// +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) sendRaw(pkt *stack.PacketBuffer, flags header.TCPFlags, seq, ack seqnum.Value, rcvWnd seqnum.Size) tcpip.Error { + var sackBlocks []header.SACKBlock + if e.EndpointState() == StateEstablished && e.rcv.pendingRcvdSegments.Len() > 0 && (flags&header.TCPFlagAck != 0) { + sackBlocks = e.sack.Blocks[:e.sack.NumBlocks] + } + options := e.makeOptions(sackBlocks) + defer putOptions(options) + hdrSize := header.TCPMinimumSize + int(e.route.MaxHeaderLength()) + len(options) + expOptVal := e.getExperimentOptionValue(e.route) + if e.route.NetProto() == header.IPv6ProtocolNumber && expOptVal != 0 { + hdrSize += header.IPv6ExperimentHdrLength + } + pkt.ReserveHeaderBytes(hdrSize) + return e.sendTCP(e.route, tcpFields{ + id: e.TransportEndpointInfo.ID, + ttl: calculateTTL(e.route, e.ipv4TTL, e.ipv6HopLimit), + tos: e.sendTOS, + flags: flags, + seq: seq, + ack: ack, + rcvWnd: rcvWnd, + opts: options, + df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo, + expOptVal: expOptVal, + }, pkt, e.gso) +} + +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) sendData(next *segment) { + // Initialize the next segment to write if it's currently nil. + if e.snd.writeNext == nil { + if next == nil { + return + } + e.snd.updateWriteNext(next) + } + + // Push out any new packets. + e.snd.sendData() +} + +// resetConnectionLocked puts the endpoint in an error state with the given +// error code and sends a RST if and only if the error is not ErrConnectionReset +// indicating that the connection is being reset due to receiving a RST. +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) resetConnectionLocked(err tcpip.Error) { + // Only send a reset if the connection is being aborted for a reason + // other than receiving a reset. + e.hardError = err + switch err.(type) { + case *tcpip.ErrConnectionReset, *tcpip.ErrTimeout: + default: + // The exact sequence number to be used for the RST is the same as the + // one used by Linux. We need to handle the case of window being shrunk + // which can cause sndNxt to be outside the acceptable window on the + // receiver. + // + // See: https://www.snellman.net/blog/archive/2016-02-01-tcp-rst/ for more + // information. + sndWndEnd := e.snd.SndUna.Add(e.snd.SndWnd) + resetSeqNum := sndWndEnd + if !sndWndEnd.LessThan(e.snd.SndNxt) || e.snd.SndNxt.Size(sndWndEnd) < (1< + // + // After sending the acknowledgment, TCP MUST drop the unacceptable + // segment and stop processing further. + // + // By sending an ACK, the remote peer is challenged to confirm the loss + // of the previous connection and the request to start a new connection. + // A legitimate peer, after restart, would not have a TCB in the + // synchronized state. Thus, when the ACK arrives, the peer should send + // a RST segment back with the sequence number derived from the ACK + // field that caused the RST. + + // This RST will confirm that the remote peer has indeed closed the + // previous connection. Upon receipt of a valid RST, the local TCP + // endpoint MUST terminate its connection. The local TCP endpoint + // should then rely on SYN retransmission from the remote end to + // re-establish the connection. + e.snd.maybeSendOutOfWindowAck(s) + } else if s.flags.Contains(header.TCPFlagAck) { + // Patch the window size in the segment according to the + // send window scale. + s.window <<= e.snd.SndWndScale + + // RFC 793, page 41 states that "once in the ESTABLISHED + // state all segments must carry current acknowledgment + // information." + drop, err := e.rcv.handleRcvdSegment(s) + if err != nil { + return false, err + } + if drop { + return true, nil + } + + // Now check if the received segment has caused us to transition + // to a CLOSED state, if yes then terminate processing and do + // not invoke the sender. + // It is also possible that the sender has sent a RST before + // which got lost and didn't reach the other side. At that time, + // we can still receive ACKs after the sender has purged the + // write list. Do not process such ACKs and return immediately. + state := e.EndpointState() + if state == StateClose || state == StateError { + // When we get into StateClose while processing from the queue, + // return immediately and let the TCP processors handle it. + return false, nil + } + + e.snd.handleRcvdSegment(s) + } + + return true, nil +} + +// keepaliveTimerExpired is called when the keepaliveTimer fires. We send TCP +// keepalive packets periodically when the connection is idle. If we don't hear +// from the other side after a number of tries, we terminate the connection. +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) keepaliveTimerExpired() tcpip.Error { + userTimeout := e.userTimeout + + // If the route is not ready or already cleaned up, then we don't need to + // send keepalives. + if e.route == nil { + return nil + } + e.keepalive.Lock() + if !e.SocketOptions().GetKeepAlive() || e.keepalive.timer.isUninitialized() || !e.keepalive.timer.checkExpiration() { + e.keepalive.Unlock() + return nil + } + + // If a userTimeout is set then abort the connection if it is + // exceeded. + if userTimeout != 0 && e.stack.Clock().NowMonotonic().Sub(e.rcv.lastRcvdAckTime) >= userTimeout && e.keepalive.unacked > 0 { + e.keepalive.Unlock() + e.stack.Stats().TCP.EstablishedTimedout.Increment() + return &tcpip.ErrTimeout{} + } + + if e.keepalive.unacked >= e.keepalive.count { + e.keepalive.Unlock() + e.stack.Stats().TCP.EstablishedTimedout.Increment() + return &tcpip.ErrTimeout{} + } + + // RFC1122 4.2.3.6: TCP keepalive is a dataless ACK with + // seg.seq = snd.nxt-1. + e.keepalive.unacked++ + e.keepalive.Unlock() + e.snd.sendEmptySegment(header.TCPFlagAck, e.snd.SndNxt-1) + e.resetKeepaliveTimer(false) + return nil +} + +// resetKeepaliveTimer restarts or stops the keepalive timer, depending on +// whether it is enabled for this endpoint. +// +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) resetKeepaliveTimer(receivedData bool) { + e.keepalive.Lock() + defer e.keepalive.Unlock() + if e.keepalive.timer.isUninitialized() { + if state := e.EndpointState(); !state.closed() { + panic(fmt.Sprintf("Unexpected state when the keepalive time is cleaned up, got %s, want %s or %s", state, StateClose, StateError)) + } + return + } + if receivedData { + e.keepalive.unacked = 0 + } + // Start the keepalive timer IFF it's enabled and there is no pending + // data to send. + if !e.SocketOptions().GetKeepAlive() || e.snd == nil || e.snd.SndUna != e.snd.SndNxt { + e.keepalive.timer.disable() + return + } + if e.keepalive.unacked > 0 { + e.keepalive.timer.enable(e.keepalive.interval) + } else { + e.keepalive.timer.enable(e.keepalive.idle) + } +} + +// disableKeepaliveTimer stops the keepalive timer. +func (e *Endpoint) disableKeepaliveTimer() { + e.keepalive.Lock() + e.keepalive.timer.disable() + e.keepalive.Unlock() +} + +// finWait2TimerExpired is called when the FIN-WAIT-2 timeout is hit +// and the peer hasn't sent us a FIN. +func (e *Endpoint) finWait2TimerExpired() { + e.mu.Lock() + e.transitionToStateCloseLocked() + e.mu.Unlock() + e.drainClosingSegmentQueue() + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) +} + +// +checklocks:e.mu +func (e *Endpoint) handshakeFailed(err tcpip.Error) { + e.lastErrorMu.Lock() + e.lastError = err + e.lastErrorMu.Unlock() + // handshakeFailed is also called from startHandshake when a listener + // transitions out of Listen state by the time the SYN is processed. In + // such cases the handshake is never initialized and the newly created + // endpoint is closed right away. + if e.h != nil && e.h.retransmitTimer != nil { + e.h.retransmitTimer.stop() + } + e.hardError = err + e.cleanupLocked() + e.setEndpointState(StateError) +} + +// handleTimeWaitSegments processes segments received during TIME_WAIT +// state. +// +checklocks:e.mu +// +checklocksalias:e.rcv.ep.mu=e.mu +func (e *Endpoint) handleTimeWaitSegments() (extendTimeWait bool, reuseTW func()) { + for i := 0; i < maxSegmentsPerWake; i++ { + s := e.segmentQueue.dequeue() + if s == nil { + break + } + extTW, newSyn := e.rcv.handleTimeWaitSegment(s) + if newSyn { + info := e.TransportEndpointInfo + newID := info.ID + newID.RemoteAddress = tcpip.Address{} + newID.RemotePort = 0 + netProtos := []tcpip.NetworkProtocolNumber{info.NetProto} + // If the local address is an IPv4 address then also + // look for IPv6 dual stack endpoints that might be + // listening on the local address. + if newID.LocalAddress.To4() != (tcpip.Address{}) { + netProtos = []tcpip.NetworkProtocolNumber{header.IPv4ProtocolNumber, header.IPv6ProtocolNumber} + } + for _, netProto := range netProtos { + if listenEP := e.stack.FindTransportEndpoint(netProto, info.TransProto, newID, s.pkt.NICID); listenEP != nil { + tcpEP := listenEP.(*Endpoint) + if EndpointState(tcpEP.State()) == StateListen { + reuseTW = func() { + if !tcpEP.enqueueSegment(s) { + return + } + tcpEP.notifyProcessor() + s.DecRef() + } + // We explicitly do not DecRef the segment as it's still valid and + // being reflected to a listening endpoint. + return false, reuseTW + } + } + } + } + if extTW { + extendTimeWait = true + } + s.DecRef() + } + return extendTimeWait, nil +} + +// +checklocks:e.mu +func (e *Endpoint) getTimeWaitDuration() time.Duration { + timeWaitDuration := DefaultTCPTimeWaitTimeout + + // Get the stack wide configuration. + var tcpTW tcpip.TCPTimeWaitTimeoutOption + if err := e.stack.TransportProtocolOption(ProtocolNumber, &tcpTW); err == nil { + timeWaitDuration = time.Duration(tcpTW) + } + return timeWaitDuration +} + +// timeWaitTimerExpired is called when an endpoint completes the required time +// (typically 2 * MSL unless configured to something else at a stack level) in +// TIME-WAIT state. +func (e *Endpoint) timeWaitTimerExpired() { + e.mu.Lock() + if e.EndpointState() != StateTimeWait { + e.mu.Unlock() + return + } + e.transitionToStateCloseLocked() + e.mu.Unlock() + e.drainClosingSegmentQueue() + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) +} + +// notifyProcessor queues this endpoint for processing to its TCP processor. +func (e *Endpoint) notifyProcessor() { + // We use TryLock here to avoid deadlocks in cases where a listening endpoint that is being + // closed tries to abort half completed connections which in turn try to queue any segments + // queued to that endpoint back to the same listening endpoint (because it may have got + // segments that matched its id but were either a RST or a new SYN which must be handled + // by a listening endpoint). In such cases the Close() on the listening endpoint will handle + // any queued segments after it releases the lock. + if !e.mu.TryLock() { + return + } + processor := e.protocol.dispatcher.selectProcessor(e.ID) + e.mu.Unlock() + processor.queueEndpoint(e) +} diff --git a/pkg/tcpip/transport/tcp/connect_unsafe.go b/pkg/tcpip/transport/tcp/connect_unsafe.go new file mode 100644 index 0000000..cfc3046 --- /dev/null +++ b/pkg/tcpip/transport/tcp/connect_unsafe.go @@ -0,0 +1,30 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "reflect" + "unsafe" +) + +// optionsToArray converts a slice of capacity >-= maxOptionSize to an array. +// +// optionsToArray panics if the capacity of options is smaller than +// maxOptionSize. +func optionsToArray(options []byte) *[maxOptionSize]byte { + // Reslice to full capacity. + options = options[0:maxOptionSize] + return (*[maxOptionSize]byte)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&options)).Data)) +} diff --git a/pkg/tcpip/transport/tcp/cubic.go b/pkg/tcpip/transport/tcp/cubic.go new file mode 100644 index 0000000..a3f8db6 --- /dev/null +++ b/pkg/tcpip/transport/tcp/cubic.go @@ -0,0 +1,318 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "math" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +// effectivelyInfinity is an initialization value used for round-trip times +// that are then set using min. It is equal to approximately 100 years: large +// enough that it will always be greater than a real TCP round-trip time, and +// small enough that it fits in time.Duration. +const effectivelyInfinity = time.Duration(math.MaxInt64) + +const ( + // RTT = round-trip time. + + // The delay increase sensitivity is determined by minRTTThresh and + // maxRTTThresh. Smaller values of minRTTThresh may cause spurious exits + // from slow start. Larger values of maxRTTThresh may result in slow start + // not exiting until loss is encountered for connections on large RTT paths. + minRTTThresh = 4 * time.Millisecond + maxRTTThresh = 16 * time.Millisecond + + // minRTTDivisor is a fraction of RTT to compute the delay threshold. A + // smaller value would mean a larger threshold and thus less sensitivity to + // delay increase, and vice versa. + minRTTDivisor = 8 + + // nRTTSample is the minimum number of RTT samples in the round before + // considering whether to exit the round due to increased RTT. + nRTTSample = 8 + + // ackDelta is the maximum time between ACKs for them to be considered part + // of the same ACK Train during HyStart + ackDelta = 2 * time.Millisecond +) + +// cubicState stores the variables related to TCP CUBIC congestion +// control algorithm state. +// +// See: https://tools.ietf.org/html/rfc8312. +// +stateify savable +type cubicState struct { + TCPCubicState + + // numCongestionEvents tracks the number of congestion events since last + // RTO. + numCongestionEvents int + + s *sender +} + +// newCubicCC returns a partially initialized cubic state with the constants +// beta and c set and t set to current time. +// +// +checklocks:s.ep.mu +func newCubicCC(s *sender) *cubicState { + now := s.ep.stack.Clock().NowMonotonic() + return &cubicState{ + TCPCubicState: TCPCubicState{ + T: now, + Beta: 0.7, + C: 0.4, + // By this point, the sender has initialized it's initial sequence + // number. + EndSeq: s.SndNxt, + LastRTT: effectivelyInfinity, + CurrRTT: effectivelyInfinity, + LastAck: now, + RoundStart: now, + }, + s: s, + } +} + +// enterCongestionAvoidance is used to initialize cubic in cases where we exit +// SlowStart without a real congestion event taking place. This can happen when +// a connection goes back to slow start due to a retransmit and we exceed the +// previously lowered ssThresh without experiencing packet loss. +// +// Refer: https://tools.ietf.org/html/rfc8312#section-4.8 +// +// +checklocks:c.s.ep.mu +func (c *cubicState) enterCongestionAvoidance() { + // See: https://tools.ietf.org/html/rfc8312#section-4.7 & + // https://tools.ietf.org/html/rfc8312#section-4.8 + if c.numCongestionEvents == 0 { + c.K = 0 + c.T = c.s.ep.stack.Clock().NowMonotonic() + c.WLastMax = c.WMax + c.WMax = float64(c.s.SndCwnd) + } +} + +// updateHyStart tracks packet round-trip time (rtt) to find a safe threshold +// to exit slow start without triggering packet loss. It updates the SSThresh +// when it does. +// +// Implementation of HyStart follows the algorithm from the Linux kernel, rather +// than RFC 9406 (https://www.rfc-editor.org/rfc/rfc9406.html). Briefly, the +// Linux kernel algorithm is based directly on the original HyStart paper +// (https://doi.org/10.1016/j.comnet.2011.01.014), and differs from the RFC in +// that two detection algorithms run in parallel ('ACK train' and 'Delay +// increase'). The RFC version includes only the latter algorithm and adds an +// intermediate phase called Conservative Slow Start, which is not implemented +// here. +// +// +checklocks:c.s.ep.mu +func (c *cubicState) updateHyStart(rtt time.Duration) { + if rtt < 0 { + // negative indicates unknown + return + } + now := c.s.ep.stack.Clock().NowMonotonic() + if c.EndSeq.LessThan(c.s.SndUna) { + c.beginHyStartRound(now) + } + // ACK train + if now.Sub(c.LastAck) < ackDelta && // ensures acks are part of the same "train" + c.LastRTT < effectivelyInfinity { + c.LastAck = now + if thresh := c.LastRTT / 2; now.Sub(c.RoundStart) > thresh { + c.s.Ssthresh = c.s.SndCwnd + } + } + + // Delay increase + c.CurrRTT = min(c.CurrRTT, rtt) + c.SampleCount++ + + if c.SampleCount >= nRTTSample && c.LastRTT < effectivelyInfinity { + // i.e. LastRTT/minRTTDivisor, but clamped to minRTTThresh & maxRTTThresh + thresh := max( + minRTTThresh, + min(maxRTTThresh, c.LastRTT/minRTTDivisor), + ) + if c.CurrRTT >= (c.LastRTT + thresh) { + // Triggered HyStart safe exit threshold + c.s.Ssthresh = c.s.SndCwnd + } + } +} + +// +checklocks:c.s.ep.mu +func (c *cubicState) beginHyStartRound(now tcpip.MonotonicTime) { + c.EndSeq = c.s.SndNxt + c.SampleCount = 0 + c.LastRTT = c.CurrRTT + c.CurrRTT = effectivelyInfinity + c.LastAck = now + c.RoundStart = now +} + +// updateSlowStart will update the congestion window as per the slow-start +// algorithm used by NewReno. If after adjusting the congestion window we cross +// the ssThresh then it will return the number of packets that must be consumed +// in congestion avoidance mode. +// +// +checklocks:c.s.ep.mu +func (c *cubicState) updateSlowStart(packetsAcked int) int { + // Don't let the congestion window cross into the congestion + // avoidance range. + newcwnd := c.s.SndCwnd + packetsAcked + enterCA := false + if newcwnd >= c.s.Ssthresh { + newcwnd = c.s.Ssthresh + c.s.SndCAAckCount = 0 + enterCA = true + } + + packetsAcked -= newcwnd - c.s.SndCwnd + c.s.SndCwnd = newcwnd + if enterCA { + c.enterCongestionAvoidance() + } + return packetsAcked +} + +// Update updates cubic's internal state variables. It must be called on every +// ACK received. +// Refer: https://tools.ietf.org/html/rfc8312#section-4 +// +// +checklocks:c.s.ep.mu +func (c *cubicState) Update(packetsAcked int, rtt time.Duration) { + if c.s.Ssthresh == InitialSsthresh && c.s.SndCwnd < c.s.Ssthresh { + c.updateHyStart(rtt) + } + if c.s.SndCwnd < c.s.Ssthresh { + packetsAcked = c.updateSlowStart(packetsAcked) + if packetsAcked == 0 { + return + } + } else { + c.s.rtt.Lock() + srtt := c.s.rtt.TCPRTTState.SRTT + c.s.rtt.Unlock() + c.s.SndCwnd = c.getCwnd(packetsAcked, c.s.SndCwnd, srtt) + } +} + +// cubicCwnd computes the CUBIC congestion window after t seconds from last +// congestion event. +func (c *cubicState) cubicCwnd(t float64) float64 { + return c.C*math.Pow(t, 3.0) + c.WMax +} + +// getCwnd returns the current congestion window as computed by CUBIC. +// Refer: https://tools.ietf.org/html/rfc8312#section-4 +func (c *cubicState) getCwnd(packetsAcked, sndCwnd int, srtt time.Duration) int { + elapsed := c.s.ep.stack.Clock().NowMonotonic().Sub(c.T) + elapsedSeconds := elapsed.Seconds() + + // Compute the window as per Cubic after 'elapsed' time + // since last congestion event. + c.WC = c.cubicCwnd(elapsedSeconds - c.K) + + // Compute the TCP friendly estimate of the congestion window. + c.WEst = c.WMax*c.Beta + (3.0*((1.0-c.Beta)/(1.0+c.Beta)))*(elapsedSeconds/srtt.Seconds()) + + // Make sure in the TCP friendly region CUBIC performs at least + // as well as Reno. + if c.WC < c.WEst && float64(sndCwnd) < c.WEst { + // TCP Friendly region of cubic. + return int(c.WEst) + } + + // In Concave/Convex region of CUBIC, calculate what CUBIC window + // will be after 1 RTT and use that to grow congestion window + // for every ack. + tEst := (elapsed + srtt).Seconds() + wtRtt := c.cubicCwnd(tEst - c.K) + // As per 4.3 for each received ACK cwnd must be incremented + // by (w_cubic(t+RTT) - cwnd/cwnd. + cwnd := float64(sndCwnd) + for i := 0; i < packetsAcked; i++ { + // Concave/Convex regions of cubic have the same formulas. + // See: https://tools.ietf.org/html/rfc8312#section-4.3 + cwnd += (wtRtt - cwnd) / cwnd + } + return int(cwnd) +} + +// HandleLossDetected implements congestionControl.HandleLossDetected. +// +// +checklocks:c.s.ep.mu +func (c *cubicState) HandleLossDetected() { + // See: https://tools.ietf.org/html/rfc8312#section-4.5 + c.numCongestionEvents++ + c.T = c.s.ep.stack.Clock().NowMonotonic() + c.WLastMax = c.WMax + c.WMax = float64(c.s.SndCwnd) + + c.fastConvergence() + c.reduceSlowStartThreshold() +} + +// HandleRTOExpired implements congestionContrl.HandleRTOExpired. +// +// +checklocks:c.s.ep.mu +func (c *cubicState) HandleRTOExpired() { + // See: https://tools.ietf.org/html/rfc8312#section-4.6 + c.T = c.s.ep.stack.Clock().NowMonotonic() + c.numCongestionEvents = 0 + c.WLastMax = c.WMax + c.WMax = float64(c.s.SndCwnd) + + c.fastConvergence() + + // We lost a packet, so reduce ssthresh. + c.reduceSlowStartThreshold() + + // Reduce the congestion window to 1, i.e., enter slow-start. Per + // RFC 5681, page 7, we must use 1 regardless of the value of the + // initial congestion window. + c.s.SndCwnd = 1 +} + +// fastConvergence implements the logic for Fast Convergence algorithm as +// described in https://tools.ietf.org/html/rfc8312#section-4.6. +func (c *cubicState) fastConvergence() { + if c.WMax < c.WLastMax { + c.WLastMax = c.WMax + c.WMax = c.WMax * (1.0 + c.Beta) / 2.0 + } else { + c.WLastMax = c.WMax + } + // Recompute k as wMax may have changed. + c.K = math.Cbrt(c.WMax * (1 - c.Beta) / c.C) +} + +// PostRecovery implements congestionControl.PostRecovery. +func (c *cubicState) PostRecovery() { + c.T = c.s.ep.stack.Clock().NowMonotonic() +} + +// reduceSlowStartThreshold returns new SsThresh as described in +// https://tools.ietf.org/html/rfc8312#section-4.7. +// +// +checklocks:c.s.ep.mu +func (c *cubicState) reduceSlowStartThreshold() { + c.s.Ssthresh = int(math.Max(float64(c.s.SndCwnd)*c.Beta, 2.0)) +} diff --git a/pkg/tcpip/transport/tcp/dispatcher.go b/pkg/tcpip/transport/tcp/dispatcher.go new file mode 100644 index 0000000..f25aa27 --- /dev/null +++ b/pkg/tcpip/transport/tcp/dispatcher.go @@ -0,0 +1,534 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "encoding/binary" + "fmt" + "math/rand" + + "github.com/sagernet/gvisor/pkg/sleep" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/hash/jenkins" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// epQueue is a queue of endpoints. +// +// +stateify savable +type epQueue struct { + mu epQueueMutex `state:"nosave"` + list endpointList +} + +// enqueue adds e to the queue if the endpoint is not already on the queue. +func (q *epQueue) enqueue(e *Endpoint) { + q.mu.Lock() + defer q.mu.Unlock() + e.pendingProcessingMu.Lock() + defer e.pendingProcessingMu.Unlock() + + if e.pendingProcessing { + return + } + q.list.PushBack(e) + e.pendingProcessing = true +} + +// dequeue removes and returns the first element from the queue if available, +// returns nil otherwise. +func (q *epQueue) dequeue() *Endpoint { + q.mu.Lock() + if e := q.list.Front(); e != nil { + q.list.Remove(e) + e.pendingProcessingMu.Lock() + e.pendingProcessing = false + e.pendingProcessingMu.Unlock() + q.mu.Unlock() + return e + } + q.mu.Unlock() + return nil +} + +// empty returns true if the queue is empty, false otherwise. +func (q *epQueue) empty() bool { + q.mu.Lock() + v := q.list.Empty() + q.mu.Unlock() + return v +} + +// processor is responsible for processing packets queued to a tcp endpoint. +// +// +stateify savable +type processor struct { + epQ epQueue + sleeper sleep.Sleeper `state:"nosave"` + newEndpointWaker sleep.Waker `state:"nosave"` + closeWaker sleep.Waker `state:"nosave"` + pauseWaker sleep.Waker `state:"nosave"` + pauseChan chan struct{} `state:"nosave"` + resumeChan chan struct{} `state:"nosave"` +} + +func (p *processor) close() { + p.closeWaker.Assert() +} + +func (p *processor) queueEndpoint(ep *Endpoint) { + // Queue an endpoint for processing by the processor goroutine. + p.epQ.enqueue(ep) + p.newEndpointWaker.Assert() +} + +// deliverAccepted delivers a passively connected endpoint to the accept queue +// of its associated listening endpoint. +// +// +checklocks:ep.mu +func deliverAccepted(ep *Endpoint) bool { + lEP := ep.h.listenEP + lEP.acceptMu.Lock() + + // Remove endpoint from list of pendingEndpoints as the handshake is now + // complete. + delete(lEP.acceptQueue.pendingEndpoints, ep) + // Deliver this endpoint to the listening socket's accept queue. + if lEP.acceptQueue.capacity == 0 { + lEP.acceptMu.Unlock() + return false + } + + // NOTE: We always queue the endpoint and on purpose do not check if + // accept queue is full at this point. This is similar to linux because + // two racing incoming ACK's can both pass the acceptQueue.isFull check + // and proceed to ESTABLISHED state. In such a case its better to + // deliver both even if it temporarily exceeds the queue limit rather + // than drop a connection that is fully connected. + // + // For reference see: + // https://github.com/torvalds/linux/blob/169e77764adc041b1dacba84ea90516a895d43b2/net/ipv4/tcp_minisocks.c#L764 + // https://github.com/torvalds/linux/blob/169e77764adc041b1dacba84ea90516a895d43b2/net/ipv4/tcp_ipv4.c#L1500 + lEP.acceptQueue.endpoints.PushBack(ep) + lEP.acceptMu.Unlock() + ep.h.listenEP.waiterQueue.Notify(waiter.ReadableEvents) + + return true +} + +// handleConnecting is responsible for TCP processing for an endpoint in one of +// the connecting states. +func handleConnecting(ep *Endpoint) { + if !ep.TryLock() { + return + } + cleanup := func() { + ep.mu.Unlock() + ep.drainClosingSegmentQueue() + ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + } + if !ep.EndpointState().connecting() { + // If the endpoint has already transitioned out of a connecting + // stage then just return (only possible if it was closed or + // timed out by the time we got around to processing the wakeup. + ep.mu.Unlock() + return + } + if err := ep.h.processSegments(); err != nil { // +checklocksforce:ep.h.ep.mu + // handshake failed. clean up the tcp endpoint and handshake + // state. + if lEP := ep.h.listenEP; lEP != nil { + lEP.acceptMu.Lock() + delete(lEP.acceptQueue.pendingEndpoints, ep) + lEP.acceptMu.Unlock() + } + ep.handshakeFailed(err) + cleanup() + return + } + + if ep.EndpointState() == StateEstablished && ep.h.listenEP != nil { + ep.isConnectNotified = true + ep.stack.Stats().TCP.PassiveConnectionOpenings.Increment() + if !deliverAccepted(ep) { + ep.resetConnectionLocked(&tcpip.ErrConnectionAborted{}) + cleanup() + return + } + } + ep.mu.Unlock() +} + +// handleConnected is responsible for TCP processing for an endpoint in one of +// the connected states(StateEstablished, StateFinWait1 etc.) +func handleConnected(ep *Endpoint) { + if !ep.TryLock() { + return + } + + if !ep.EndpointState().connected() { + // If the endpoint has already transitioned out of a connected + // state then just return (only possible if it was closed or + // timed out by the time we got around to processing the wakeup. + ep.mu.Unlock() + return + } + + // NOTE: We read this outside of e.mu lock which means that by the time + // we get to handleSegments the endpoint may not be in ESTABLISHED. But + // this should be fine as all normal shutdown states are handled by + // handleSegmentsLocked. + switch err := ep.handleSegmentsLocked(); { + case err != nil: + // Send any active resets if required. + ep.resetConnectionLocked(err) + fallthrough + case ep.EndpointState() == StateClose: + ep.mu.Unlock() + ep.drainClosingSegmentQueue() + ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + return + case ep.EndpointState() == StateTimeWait: + startTimeWait(ep) + } + ep.mu.Unlock() +} + +// startTimeWait starts a new goroutine to handle TIME-WAIT. +// +// +checklocks:ep.mu +func startTimeWait(ep *Endpoint) { + // Disable close timer as we are now entering real TIME_WAIT. + if ep.finWait2Timer != nil { + ep.finWait2Timer.Stop() + } + // Wake up any waiters before we start TIME-WAIT. + ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + timeWaitDuration := ep.getTimeWaitDuration() + ep.timeWaitTimer = ep.stack.Clock().AfterFunc(timeWaitDuration, ep.timeWaitTimerExpired) +} + +// handleTimeWait is responsible for TCP processing for an endpoint in TIME-WAIT +// state. +func handleTimeWait(ep *Endpoint) { + if !ep.TryLock() { + return + } + + if ep.EndpointState() != StateTimeWait { + // If the endpoint has already transitioned out of a TIME-WAIT + // state then just return (only possible if it was closed or + // timed out by the time we got around to processing the wakeup. + ep.mu.Unlock() + return + } + + extendTimeWait, reuseTW := ep.handleTimeWaitSegments() + if reuseTW != nil { + ep.transitionToStateCloseLocked() + ep.mu.Unlock() + ep.drainClosingSegmentQueue() + ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + reuseTW() + return + } + if extendTimeWait { + ep.timeWaitTimer.Reset(ep.getTimeWaitDuration()) + } + ep.mu.Unlock() +} + +// handleListen is responsible for TCP processing for an endpoint in LISTEN +// state. +func handleListen(ep *Endpoint) { + if !ep.TryLock() { + return + } + defer ep.mu.Unlock() + + if ep.EndpointState() != StateListen { + // If the endpoint has already transitioned out of a LISTEN + // state then just return (only possible if it was closed or + // shutdown). + return + } + + for i := 0; i < maxSegmentsPerWake; i++ { + s := ep.segmentQueue.dequeue() + if s == nil { + break + } + + // TODO(gvisor.dev/issue/4690): Better handle errors instead of + // silently dropping. + _ = ep.handleListenSegment(ep.listenCtx, s) + s.DecRef() + } +} + +// start runs the main loop for a processor which is responsible for all TCP +// processing for TCP endpoints. +func (p *processor) start(wg *sync.WaitGroup) { + defer wg.Done() + defer p.sleeper.Done() + + for { + switch w := p.sleeper.Fetch(true); { + case w == &p.closeWaker: + return + case w == &p.pauseWaker: + if !p.epQ.empty() { + p.newEndpointWaker.Assert() + p.pauseWaker.Assert() + continue + } else { + p.pauseChan <- struct{}{} + <-p.resumeChan + } + case w == &p.newEndpointWaker: + for { + ep := p.epQ.dequeue() + if ep == nil { + break + } + if ep.segmentQueue.empty() { + continue + } + switch state := ep.EndpointState(); { + case state.connecting(): + handleConnecting(ep) + case state.connected() && state != StateTimeWait: + handleConnected(ep) + case state == StateTimeWait: + handleTimeWait(ep) + case state == StateListen: + handleListen(ep) + case state == StateError || state == StateClose: + // Try to redeliver any still queued + // packets to another endpoint or send a + // RST if it can't be delivered. + ep.mu.Lock() + if st := ep.EndpointState(); st == StateError || st == StateClose { + ep.drainClosingSegmentQueue() + } + ep.mu.Unlock() + default: + panic(fmt.Sprintf("unexpected tcp state in processor: %v", state)) + } + // If there are more segments to process and the + // endpoint lock is not held by user then + // requeue this endpoint for processing. + if !ep.segmentQueue.empty() && !ep.isOwnedByUser() { + p.epQ.enqueue(ep) + } + } + } + } +} + +// pause pauses the processor loop. +func (p *processor) pause() chan struct{} { + p.pauseWaker.Assert() + return p.pauseChan +} + +// resume resumes a previously paused loop. +// +// Precondition: Pause must have been called previously. +func (p *processor) resume() { + p.resumeChan <- struct{}{} +} + +// dispatcher manages a pool of TCP endpoint processors which are responsible +// for the processing of inbound segments. This fixed pool of processor +// goroutines do full tcp processing. The processor is selected based on the +// hash of the endpoint id to ensure that delivery for the same endpoint happens +// in-order. +// +// +stateify savable +type dispatcher struct { + processors []processor + wg sync.WaitGroup `state:"nosave"` + hasher jenkinsHasher + mu dispatcherMutex `state:"nosave"` + // +checklocks:mu + paused bool + // +checklocks:mu + closed bool +} + +// init initializes a dispatcher and starts the main loop for all the processors +// owned by this dispatcher. +func (d *dispatcher) init(rng *rand.Rand, nProcessors int) { + d.close() + d.wait() + + d.mu.Lock() + defer d.mu.Unlock() + + d.closed = false + d.processors = make([]processor, nProcessors) + d.hasher = jenkinsHasher{seed: rng.Uint32()} + d.startLocked() +} + +// +checklocks:d.mu +func (d *dispatcher) startLocked() { + if d.closed { + return + } + for i := range d.processors { + p := &d.processors[i] + p.sleeper.AddWaker(&p.newEndpointWaker) + p.sleeper.AddWaker(&p.closeWaker) + p.sleeper.AddWaker(&p.pauseWaker) + p.pauseChan = make(chan struct{}) + p.resumeChan = make(chan struct{}) + d.wg.Add(1) + // NB: sleeper-waker registration must happen synchronously to avoid races + // with `close`. It's possible to pull all this logic into `start`, but + // that results in a heap-allocated function literal. + go p.start(&d.wg) + } +} + +func (d *dispatcher) start() { + d.mu.Lock() + defer d.mu.Unlock() + + d.startLocked() +} + +// close closes a dispatcher and its processors. +func (d *dispatcher) close() { + d.mu.Lock() + d.closed = true + d.mu.Unlock() + for i := range d.processors { + d.processors[i].close() + } +} + +// wait waits for all processor goroutines to end. +func (d *dispatcher) wait() { + d.wg.Wait() +} + +// queuePacket queues an incoming packet to the matching tcp endpoint and +// also queues the endpoint to a processor queue for processing. +func (d *dispatcher) queuePacket(stackEP stack.TransportEndpoint, id stack.TransportEndpointID, clock tcpip.Clock, pkt *stack.PacketBuffer) { + d.mu.Lock() + closed := d.closed + d.mu.Unlock() + + if closed { + return + } + + ep := stackEP.(*Endpoint) + + s, err := newIncomingSegment(id, clock, pkt) + if err != nil { + ep.stack.Stats().TCP.InvalidSegmentsReceived.Increment() + ep.stats.ReceiveErrors.MalformedPacketsReceived.Increment() + return + } + defer s.DecRef() + + if !s.csumValid { + ep.stack.Stats().TCP.ChecksumErrors.Increment() + ep.stats.ReceiveErrors.ChecksumErrors.Increment() + return + } + + ep.stack.Stats().TCP.ValidSegmentsReceived.Increment() + ep.stats.SegmentsReceived.Increment() + if (s.flags & header.TCPFlagRst) != 0 { + ep.stack.Stats().TCP.ResetsReceived.Increment() + } + + if !ep.enqueueSegment(s) { + return + } + + // Only wakeup the processor if endpoint lock is not held by a user + // goroutine as endpoint.UnlockUser will wake up the processor if the + // segment queue is not empty. + if !ep.isOwnedByUser() { + d.selectProcessor(id).queueEndpoint(ep) + } +} + +// selectProcessor uses a hash of the transport endpoint ID to queue the +// endpoint to a specific processor. This is required to main TCP ordering as +// queueing the same endpoint to multiple processors can *potentially* result in +// out of order processing of incoming segments. It also ensures that a dispatcher +// evenly loads the processor goroutines. +func (d *dispatcher) selectProcessor(id stack.TransportEndpointID) *processor { + return &d.processors[d.hasher.hash(id)%uint32(len(d.processors))] +} + +// pause pauses a dispatcher and all its processor goroutines. +func (d *dispatcher) pause() { + d.mu.Lock() + d.paused = true + d.mu.Unlock() + for i := range d.processors { + <-d.processors[i].pause() + } +} + +// resume resumes a previously paused dispatcher and its processor goroutines. +// Calling resume on a dispatcher that was never paused is a no-op. +func (d *dispatcher) resume() { + d.mu.Lock() + + if !d.paused { + // If this was a restore run the stack is a new instance and + // it was never paused, so just return as there is nothing to + // resume. + d.mu.Unlock() + return + } + d.paused = false + d.mu.Unlock() + for i := range d.processors { + d.processors[i].resume() + } +} + +// jenkinsHasher contains state needed to for a jenkins hash. +// +// +stateify savable +type jenkinsHasher struct { + seed uint32 +} + +// hash hashes the provided TransportEndpointID using the jenkins hash +// algorithm. +func (j jenkinsHasher) hash(id stack.TransportEndpointID) uint32 { + var payload [4]byte + binary.LittleEndian.PutUint16(payload[0:], id.LocalPort) + binary.LittleEndian.PutUint16(payload[2:], id.RemotePort) + + h := jenkins.Sum32(j.seed) + h.Write(payload[:]) + h.Write(id.LocalAddress.AsSlice()) + h.Write(id.RemoteAddress.AsSlice()) + return h.Sum32() +} diff --git a/pkg/tcpip/transport/tcp/dispatcher_mutex.go b/pkg/tcpip/transport/tcp/dispatcher_mutex.go new file mode 100644 index 0000000..2c6bb0f --- /dev/null +++ b/pkg/tcpip/transport/tcp/dispatcher_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type dispatcherMutex struct { + mu sync.Mutex +} + +var dispatcherprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var dispatcherlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type dispatcherlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *dispatcherMutex) Lock() { + locking.AddGLock(dispatcherprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *dispatcherMutex) NestedLock(i dispatcherlockNameIndex) { + locking.AddGLock(dispatcherprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *dispatcherMutex) Unlock() { + locking.DelGLock(dispatcherprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *dispatcherMutex) NestedUnlock(i dispatcherlockNameIndex) { + locking.DelGLock(dispatcherprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func dispatcherinitLockNames() {} + +func init() { + dispatcherinitLockNames() + dispatcherprefixIndex = locking.NewMutexClass(reflect.TypeOf(dispatcherMutex{}), dispatcherlockNames) +} diff --git a/pkg/tcpip/transport/tcp/endpoint.go b/pkg/tcpip/transport/tcp/endpoint.go new file mode 100644 index 0000000..3497a60 --- /dev/null +++ b/pkg/tcpip/transport/tcp/endpoint.go @@ -0,0 +1,3367 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "container/heap" + "fmt" + "io" + "math" + "runtime" + "strings" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sleep" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// EndpointState represents the state of a TCP endpoint. +type EndpointState tcpip.EndpointState + +// Endpoint states. Note that are represented in a netstack-specific manner and +// may not be meaningful externally. Specifically, they need to be translated to +// Linux's representation for these states if presented to userspace. +const ( + _ EndpointState = iota + // TCP protocol states in sync with the definitions in + // https://github.com/torvalds/linux/blob/7acac4b3196/include/net/tcp_states.h#L13 + StateEstablished + StateSynSent + StateSynRecv + StateFinWait1 + StateFinWait2 + StateTimeWait + StateClose + StateCloseWait + StateLastAck + StateListen + StateClosing + + // Endpoint states internal to netstack. + StateInitial + StateBound + StateConnecting // Connect() called, but the initial SYN hasn't been sent. + StateError +) + +const ( + // rcvAdvWndScale is used to split the available socket buffer into + // application buffer and the window to be advertised to the peer. This is + // currently hard coded to split the available space equally. + rcvAdvWndScale = 1 + + // SegOverheadFactor is used to multiply the value provided by the + // user on a SetSockOpt for setting the socket send/receive buffer sizes. + SegOverheadFactor = 2 +) + +type connDirectionState uint32 + +// Connection direction states used for directionState checks in endpoint struct +// to detect half-closed connection and deliver POLLRDHUP +const ( + connDirectionStateOpen connDirectionState = 0 + connDirectionStateRcvClosed connDirectionState = 1 + connDirectionStateSndClosed connDirectionState = 2 + connDirectionStateAll connDirectionState = connDirectionStateOpen | connDirectionStateRcvClosed | connDirectionStateSndClosed +) + +// connected returns true when s is one of the states representing an +// endpoint connected to a peer. +func (s EndpointState) connected() bool { + switch s { + case StateEstablished, StateFinWait1, StateFinWait2, StateTimeWait, StateCloseWait, StateLastAck, StateClosing: + return true + default: + return false + } +} + +// connecting returns true when s is one of the states representing a +// connection in progress, but not yet fully established. +func (s EndpointState) connecting() bool { + switch s { + case StateConnecting, StateSynSent, StateSynRecv: + return true + default: + return false + } +} + +// internal returns true when the state is netstack internal. +func (s EndpointState) internal() bool { + switch s { + case StateInitial, StateBound, StateConnecting, StateError: + return true + default: + return false + } +} + +// handshake returns true when s is one of the states representing an endpoint +// in the middle of a TCP handshake. +func (s EndpointState) handshake() bool { + switch s { + case StateSynSent, StateSynRecv: + return true + default: + return false + } +} + +// closed returns true when s is one of the states an endpoint transitions to +// when closed or when it encounters an error. This is distinct from a newly +// initialized endpoint that was never connected. +func (s EndpointState) closed() bool { + switch s { + case StateClose, StateError: + return true + default: + return false + } +} + +// String implements fmt.Stringer.String. +func (s EndpointState) String() string { + switch s { + case StateInitial: + return "INITIAL" + case StateBound: + return "BOUND" + case StateConnecting: + return "CONNECTING" + case StateError: + return "ERROR" + case StateEstablished: + return "ESTABLISHED" + case StateSynSent: + return "SYN-SENT" + case StateSynRecv: + return "SYN-RCVD" + case StateFinWait1: + return "FIN-WAIT1" + case StateFinWait2: + return "FIN-WAIT2" + case StateTimeWait: + return "TIME-WAIT" + case StateClose: + return "CLOSED" + case StateCloseWait: + return "CLOSE-WAIT" + case StateLastAck: + return "LAST-ACK" + case StateListen: + return "LISTEN" + case StateClosing: + return "CLOSING" + default: + panic("unreachable") + } +} + +// SACKInfo holds TCP SACK related information for a given endpoint. +// +// +stateify savable +type SACKInfo struct { + // Blocks is the maximum number of SACK blocks we track + // per endpoint. + Blocks [MaxSACKBlocks]header.SACKBlock + + // NumBlocks is the number of valid SACK blocks stored in the + // blocks array above. + NumBlocks int +} + +// ReceiveErrors collect segment receive errors within transport layer. +// +// +stateify savable +type ReceiveErrors struct { + tcpip.ReceiveErrors + + // SegmentQueueDropped is the number of segments dropped due to + // a full segment queue. + SegmentQueueDropped tcpip.StatCounter + + // ChecksumErrors is the number of segments dropped due to bad checksums. + ChecksumErrors tcpip.StatCounter + + // ListenOverflowSynDrop is the number of times the listen queue overflowed + // and a SYN was dropped. + ListenOverflowSynDrop tcpip.StatCounter + + // ListenOverflowAckDrop is the number of times the final ACK + // in the handshake was dropped due to overflow. + ListenOverflowAckDrop tcpip.StatCounter + + // ZeroRcvWindowState is the number of times we advertised + // a zero receive window when rcvQueue is full. + ZeroRcvWindowState tcpip.StatCounter + + // WantZeroWindow is the number of times we wanted to advertise a + // zero receive window but couldn't because it would have caused + // the receive window's right edge to shrink. + WantZeroRcvWindow tcpip.StatCounter +} + +// SendErrors collect segment send errors within the transport layer. +// +// +stateify savable +type SendErrors struct { + tcpip.SendErrors + + // SegmentSendToNetworkFailed is the number of TCP segments failed to be sent + // to the network endpoint. + SegmentSendToNetworkFailed tcpip.StatCounter + + // SynSendToNetworkFailed is the number of TCP SYNs failed to be sent + // to the network endpoint. + SynSendToNetworkFailed tcpip.StatCounter + + // Retransmits is the number of TCP segments retransmitted. + Retransmits tcpip.StatCounter + + // FastRetransmit is the number of segments retransmitted in fast + // recovery. + FastRetransmit tcpip.StatCounter + + // Timeouts is the number of times the RTO expired. + Timeouts tcpip.StatCounter +} + +// Stats holds statistics about the endpoint. +// +// +stateify savable +type Stats struct { + // SegmentsReceived is the number of TCP segments received that + // the transport layer successfully parsed. + SegmentsReceived tcpip.StatCounter + + // SegmentsSent is the number of TCP segments sent. + SegmentsSent tcpip.StatCounter + + // FailedConnectionAttempts is the number of times we saw Connect and + // Accept errors. + FailedConnectionAttempts tcpip.StatCounter + + // ReceiveErrors collects segment receive errors within the + // transport layer. + ReceiveErrors ReceiveErrors + + // ReadErrors collects segment read errors from an endpoint read call. + ReadErrors tcpip.ReadErrors + + // SendErrors collects segment send errors within the transport layer. + SendErrors SendErrors + + // WriteErrors collects segment write errors from an endpoint write call. + WriteErrors tcpip.WriteErrors +} + +// IsEndpointStats is an empty method to implement the tcpip.EndpointStats +// marker interface. +func (*Stats) IsEndpointStats() {} + +// sndQueueInfo implements a send queue. +// +// +stateify savable +type sndQueueInfo struct { + sndQueueMu sndQueueMutex `state:"nosave"` + TCPSndBufState +} + +// CloneState clones sq into other. It is not thread safe +func (sq *sndQueueInfo) CloneState(other *TCPSndBufState) { + other.SndBufSize = sq.SndBufSize + other.SndBufUsed = sq.SndBufUsed + other.SndClosed = sq.SndClosed + other.PacketTooBigCount = sq.PacketTooBigCount + other.SndMTU = sq.SndMTU + other.AutoTuneSndBufDisabled = atomicbitops.FromUint32(sq.AutoTuneSndBufDisabled.RacyLoad()) +} + +// Endpoint represents a TCP endpoint. This struct serves as the interface +// between users of the endpoint and the protocol implementation; it is legal to +// have concurrent goroutines make calls into the endpoint, they are properly +// synchronized. The protocol implementation, however, runs in a single +// goroutine. +// +// Each endpoint has a few mutexes: +// +// e.mu -> Primary mutex for an endpoint must be held for all operations except +// in e.Readiness where acquiring it will result in a deadlock in epoll +// implementation. +// +// The following three mutexes can be acquired independent of e.mu but if +// acquired with e.mu then e.mu must be acquired first. +// +// e.acceptMu -> Protects e.acceptQueue. +// e.rcvQueueMu -> Protects e.rcvQueue's associated fields but not e.rcvQueue +// itself. +// e.sndQueueMu -> Protects the e.sndQueue and associated fields. +// e.lastErrorMu -> Protects the lastError field. +// +// LOCKING/UNLOCKING of the endpoint. The locking of an endpoint is different +// based on the context in which the lock is acquired. In the syscall context +// e.LockUser/e.UnlockUser should be used and when doing background processing +// e.mu.Lock/e.mu.Unlock should be used. The distinction is described below +// in brief. +// +// The reason for this locking behaviour is to avoid wakeups to handle packets. +// In cases where the endpoint is already locked the background processor can +// queue the packet up and go its merry way and the lock owner will eventually +// process the backlog when releasing the lock. Similarly when acquiring the +// lock from say a syscall goroutine we can implement a bit of spinning if we +// know that the lock is not held by another syscall goroutine. Background +// processors should never hold the lock for long and we can avoid an expensive +// sleep/wakeup by spinning for a shortwhile. +// +// For more details please see the detailed documentation on +// e.LockUser/e.UnlockUser methods. +// +// TODO(b/339664055): Checklocks should be used more extensively here. Coverage +// is currently sparse. +// +// +stateify savable +type Endpoint struct { + TCPEndpointStateInner + stack.TransportEndpointInfo + tcpip.DefaultSocketOptionsHandler + + // EndpointEntry is used to queue endpoints for processing to the + // a given tcp processor goroutine. + // + // Precondition: epQueue.mu must be held to read/write this field.. + endpointEntry `state:"nosave"` + + // pendingProcessingMu protects pendingProcessing. + pendingProcessingMu pendingProcessingMutex `state:"nosave"` + + // pendingProcessing is true if this endpoint is queued for processing + // to a TCP processor. + // +checklocks:pendingProcessingMu + pendingProcessing bool `state:"nosave"` + + // The following fields are initialized at creation time and do not + // change throughout the lifetime of the endpoint. + stack *stack.Stack + protocol *protocol + waiterQueue *waiter.Queue `state:"wait"` + + // hardError is meaningful only when state is stateError. It stores the + // error to be returned when read/write syscalls are called and the + // endpoint is in this state. hardError is protected by endpoint mu. + hardError tcpip.Error + + // lastError represents the last error that the endpoint reported; + // access to it is protected by the following mutex. + lastErrorMu lastErrorMutex `state:"nosave"` + lastError tcpip.Error + + rcvQueueMu rcvQueueMutex `state:"nosave"` + + // +checklocks:rcvQueueMu + TCPRcvBufState + + // rcvMemUsed tracks the total amount of memory in use by received segments + // held in rcvQueue, pendingRcvdSegments and the segment queue. This is used to + // compute the window and the actual available buffer space. This is distinct + // from rcvBufUsed above which is the actual number of payload bytes held in + // the buffer not including any segment overheads. + rcvMemUsed atomicbitops.Int32 + + // mu protects all endpoint fields unless documented otherwise. mu must + // be acquired before interacting with the endpoint fields. + // + // During handshake, mu is locked by the protocol listen goroutine and + // released by the handshake completion goroutine. + mu sync.CrossGoroutineMutex `state:"nosave"` + ownedByUser atomicbitops.Uint32 + + // rcvQueue is the queue for ready-for-delivery segments. + // + // +checklocks:mu + rcvQueue segmentList `state:"wait"` + + // state must be read/set using the EndpointState()/setEndpointState() + // methods. + state atomicbitops.Uint32 `state:".(EndpointState)"` + + // connectionDirectionState holds current state of send and receive, + // accessed atomically + connectionDirectionState atomicbitops.Uint32 + + // origEndpointState is only used during a restore phase to save the + // endpoint state at restore time as the socket is moved to it's correct + // state. + origEndpointState uint32 `state:"nosave"` + + isPortReserved bool + isRegistered bool + boundNICID tcpip.NICID + route *stack.Route `state:"nosave"` + ipv4TTL uint8 + ipv6HopLimit int16 + isConnectNotified bool + + // h stores a reference to the current handshake state if the endpoint is in + // the SYN-SENT or SYN-RECV states, in which case endpoint == endpoint.h.ep. + // nil otherwise. + // +checklocks:mu + h *handshake + + // portFlags stores the current values of port related flags. + portFlags ports.Flags + + // Values used to reserve a port or register a transport endpoint + // (which ever happens first). + boundBindToDevice tcpip.NICID + boundPortFlags ports.Flags + boundDest tcpip.FullAddress + + // effectiveNetProtos contains the network protocols actually in use. In + // most cases it will only contain "netProto", but in cases like IPv6 + // endpoints with v6only set to false, this could include multiple + // protocols (e.g., IPv6 and IPv4) or a single different protocol (e.g., + // IPv4 when IPv6 endpoint is bound or connected to an IPv4 mapped + // address). + effectiveNetProtos []tcpip.NetworkProtocolNumber + + // recentTSTime is the unix time when we last updated + // TCPEndpointStateInner.RecentTS. + recentTSTime tcpip.MonotonicTime + + // shutdownFlags represent the current shutdown state of the endpoint. + shutdownFlags tcpip.ShutdownFlags + + // tcpRecovery is the loss recovery algorithm used by TCP. + tcpRecovery tcpip.TCPRecovery + + // sack holds TCP SACK related information for this endpoint. + sack SACKInfo + + // delay enables Nagle's algorithm. + // + // delay is a boolean (0 is false) and must be accessed atomically. + delay uint32 + + // scoreboard holds TCP SACK Scoreboard information for this endpoint. + scoreboard *SACKScoreboard + + // segmentQueue is used to hand received segments to the protocol + // goroutine. Segments are queued as long as the queue is not full, + // and dropped when it is. + segmentQueue segmentQueue `state:"wait"` + + // userMSS if non-zero is the MSS value explicitly set by the user + // for this endpoint using the TCP_MAXSEG setsockopt. + userMSS uint16 + + // maxSynRetries is the maximum number of SYN retransmits that TCP should + // send before aborting the attempt to connect. It cannot exceed 255. + // + // NOTE: This is currently a no-op and does not change the SYN + // retransmissions. + maxSynRetries uint8 + + // windowClamp is used to bound the size of the advertised window to + // this value. + windowClamp uint32 + + // sndQueueInfo contains the implementation of the endpoint's send queue. + sndQueueInfo sndQueueInfo + + // cc stores the name of the Congestion Control algorithm to use for + // this endpoint. + cc tcpip.CongestionControlOption + + // keepalive manages TCP keepalive state. When the connection is idle + // (no data sent or received) for keepaliveIdle, we start sending + // keepalives every keepalive.interval. If we send keepalive.count + // without hearing a response, the connection is closed. + keepalive keepalive + + // userTimeout if non-zero specifies a user specified timeout for + // a connection w/ pending data to send. A connection that has pending + // unacked data will be forcibily aborted if the timeout is reached + // without any data being acked. + userTimeout time.Duration + + // deferAccept if non-zero specifies a user specified time during + // which the final ACK of a handshake will be dropped provided the + // ACK is a bare ACK and carries no data. If the timeout is crossed then + // the bare ACK is accepted and the connection is delivered to the + // listener. + deferAccept time.Duration + + // acceptMu protects accepQueue + acceptMu sync.Mutex `state:"nosave"` + + // acceptQueue is used by a listening endpoint to send newly accepted + // connections to the endpoint so that they can be read by Accept() + // calls. + // + // +checklocks:acceptMu + acceptQueue acceptQueue + + rcv *receiver `state:"wait"` + + snd *sender `state:"wait"` + + // The goroutine drain completion notification channel. + drainDone chan struct{} `state:"nosave"` + + // The goroutine undrain notification channel. This is currently used as + // a way to block the worker goroutines. Today nothing closes/writes + // this channel and this causes any goroutines waiting on this to just + // block. This is used during save/restore to prevent worker goroutines + // from mutating state as it's being saved. + undrain chan struct{} `state:"nosave"` + + // probe if not nil is invoked on every received segment. It is passed + // a copy of the current state of the endpoint. + probe TCPProbeFunc `state:"nosave"` + + // The following are only used to assist the restore run to re-connect. + connectingAddress tcpip.Address + + // amss is the advertised MSS to the peer by this endpoint. + amss uint16 + + // sendTOS represents IPv4 TOS or IPv6 TrafficClass, + // applied while sending packets. Defaults to 0 as on Linux. + sendTOS uint8 + + gso stack.GSO + + stats Stats + + // tcpLingerTimeout is the maximum amount of a time a socket + // a socket stays in TIME_WAIT state before being marked + // closed. + tcpLingerTimeout time.Duration + + // closed indicates that the user has called closed on the + // endpoint and at this point the endpoint is only around + // to complete the TCP shutdown. + closed bool + + // txHash is the transport layer hash to be set on outbound packets + // emitted by this endpoint. + txHash uint32 + + // owner is used to get uid and gid of the packet. + owner tcpip.PacketOwner + + // ops is used to get socket level options. + ops tcpip.SocketOptions + + // lastOutOfWindowAckTime is the time at which the an ACK was sent in response + // to an out of window segment being received by this endpoint. + lastOutOfWindowAckTime tcpip.MonotonicTime + + // finWait2Timer is used to reap orphaned sockets in FIN-WAIT-2 where the peer + // is yet to send a FIN but on our end the socket is fully closed i.e. endpoint.Close() + // has been called on the socket. This timer is not started for sockets that + // are waiting for a peer FIN but are not closed. + finWait2Timer tcpip.Timer `state:"nosave"` + + // timeWaitTimer is used to reap a socket once a socket has been in TIME-WAIT state + // for tcp.DefaultTCPTimeWaitTimeout seconds. + timeWaitTimer tcpip.Timer `state:"nosave"` + + // listenCtx is used by listening endpoints to store state used while listening for + // connections. Nil otherwise. + listenCtx *listenContext `state:"nosave"` + + // limRdr is reused to avoid allocations. + // + // +checklocks:mu + limRdr *io.LimitedReader `state:"nosave"` + + // pmtud is the PMTUD strategy to use. + // + // +checklocks:mu + pmtud tcpip.PMTUDStrategy + + // alsoBindToV4 indicates if `any` address was used to bind a port. + // + // +checklocks:mu + alsoBindToV4 bool +} + +// calculateAdvertisedMSS calculates the MSS to advertise. +// +// If userMSS is non-zero and is not greater than the maximum possible MSS for +// r, it will be used; otherwise, the maximum possible MSS will be used. +func calculateAdvertisedMSS(userMSS uint16, r *stack.Route) uint16 { + // The maximum possible MSS is dependent on the route. + // TODO(b/143359391): Respect TCP Min and Max size. + maxMSS := uint16(r.MTU() - header.TCPMinimumSize) + + if userMSS != 0 && userMSS < maxMSS { + return userMSS + } + + return maxMSS +} + +// isOwnedByUser() returns true if the endpoint lock is currently +// held by a user(syscall) goroutine. +func (e *Endpoint) isOwnedByUser() bool { + return e.ownedByUser.Load() == 1 +} + +// LockUser tries to lock e.mu and if it fails it will check if the lock is held +// by another syscall goroutine. If yes, then it will goto sleep waiting for the +// lock to be released, if not then it will spin till it acquires the lock or +// another syscall goroutine acquires it in which case it will goto sleep as +// described above. +// +// The assumption behind spinning here being that background packet processing +// should not be holding the lock for long and spinning reduces latency as we +// avoid an expensive sleep/wakeup of the syscall goroutine). +// +checklocksacquire:e.mu +// +checklocksacquire:e.snd.ep.mu +func (e *Endpoint) LockUser() { + const iterations = 5 + for i := 0; i < iterations; i++ { + // Try first if the sock is locked then check if it's owned + // by another user goroutine if not then we spin, otherwise + // we just go to sleep on the Lock() and wait. + if !e.TryLock() { + // If socket is owned by the user then just go to sleep + // as the lock could be held for a reasonably long time. + if e.ownedByUser.Load() == 1 { + e.mu.Lock() + e.ownedByUser.Store(1) + return // +checklocksforce: this locks e.snd.ep.mu + } + // Spin but don't yield the processor since the lower half + // should yield the lock soon. + continue + } + e.ownedByUser.Store(1) + return // +checklocksforce: this locks e.snd.ep.mu + } + + for i := 0; i < iterations; i++ { + // Try first if the sock is locked then check if it's owned + // by another user goroutine if not then we spin, otherwise + // we just go to sleep on the Lock() and wait. + if !e.TryLock() { + // If socket is owned by the user then just go to sleep + // as the lock could be held for a reasonably long time. + if e.ownedByUser.Load() == 1 { + e.mu.Lock() + e.ownedByUser.Store(1) + return // +checklocksforce: this locks e.snd.ep.mu + } + // Spin but yield the processor since the lower half + // should yield the lock soon. + runtime.Gosched() + continue + } + e.ownedByUser.Store(1) + return // +checklocksforce: this locks e.snd.ep.mu + } + + // Finally just give up and wait for the Lock. + e.mu.Lock() + e.ownedByUser.Store(1) +} + +// UnlockUser will check if there are any segments already queued for processing +// and wake up a processor goroutine to process them before unlocking e.mu. +// This is required because we when packets arrive and endpoint lock is already +// held then such packets are queued up to be processed. +// +// Precondition: e.LockUser() must have been called before calling e.UnlockUser() +// +checklocksrelease:e.mu +func (e *Endpoint) UnlockUser() { + // Lock segment queue before checking so that we avoid a race where + // segments can be queued between the time we check if queue is empty + // and actually unlock the endpoint mutex. + e.segmentQueue.mu.Lock() + if e.segmentQueue.emptyLocked() { + if e.ownedByUser.Swap(0) != 1 { + panic("e.UnlockUser() called without calling e.LockUser()") + } + e.mu.Unlock() + e.segmentQueue.mu.Unlock() + return + } + e.segmentQueue.mu.Unlock() + + // Since we are waking the processor goroutine here just unlock + // and let it process the queued segments. + if e.ownedByUser.Swap(0) != 1 { + panic("e.UnlockUser() called without calling e.LockUser()") + } + processor := e.protocol.dispatcher.selectProcessor(e.ID) + e.mu.Unlock() + + // Wake up the processor for this endpoint to process any queued + // segments after releasing the lock to avoid the case where if the + // processor goroutine starts running before we release the lock here + // then it will fail to process as TryLock() will fail. + processor.queueEndpoint(e) + return +} + +// StopWork halts packet processing. Only to be used in tests. +// +checklocksacquire:e.mu +func (e *Endpoint) StopWork() { + e.mu.Lock() +} + +// ResumeWork resumes packet processing. Only to be used in tests. +// +checklocksrelease:e.mu +func (e *Endpoint) ResumeWork() { + e.mu.Unlock() +} + +// AssertLockHeld forces the checklocks analyzer to consider e.mu held. This is +// used in places where we know that e.mu is held, but checklocks does not, +// which can happen when creating new locked objects. You must pass the known +// locked endpoint to this function and it must be the same as the caller +// endpoint. +// TODO(b/226403629): Remove this function once checklocks understands local +// variable locks. +// +checklocks:locked.mu +// +checklocksacquire:e.mu +func (e *Endpoint) AssertLockHeld(locked *Endpoint) { + if e != locked { + panic("AssertLockHeld failed: locked endpoint != asserting endpoint") + } +} + +// TryLock is a helper that calls TryLock on the endpoint's mutex and +// adds the necessary checklocks annotations. +// TODO(b/226403629): Remove this once checklocks understands TryLock. +// +checklocksacquire:e.mu +func (e *Endpoint) TryLock() bool { + if e.mu.TryLock() { + return true // +checklocksforce + } + return false // +checklocksignore +} + +// setEndpointState updates the state of the endpoint to state atomically. This +// method is unexported as the only place we should update the state is in this +// package but we allow the state to be read freely without holding e.mu. +// +// +checklocks:e.mu +func (e *Endpoint) setEndpointState(state EndpointState) { + oldstate := EndpointState(e.state.Swap(uint32(state))) + switch state { + case StateEstablished: + e.stack.Stats().TCP.CurrentEstablished.Increment() + e.stack.Stats().TCP.CurrentConnected.Increment() + case StateError: + fallthrough + case StateClose: + if oldstate == StateCloseWait || oldstate == StateEstablished { + e.stack.Stats().TCP.EstablishedResets.Increment() + } + if oldstate.connected() { + e.stack.Stats().TCP.CurrentConnected.Decrement() + } + fallthrough + default: + if oldstate == StateEstablished { + e.stack.Stats().TCP.CurrentEstablished.Decrement() + } + } +} + +// EndpointState returns the current state of the endpoint. +func (e *Endpoint) EndpointState() EndpointState { + return EndpointState(e.state.Load()) +} + +// setRecentTimestamp sets the recentTS field to the provided value. +func (e *Endpoint) setRecentTimestamp(recentTS uint32) { + e.RecentTS = recentTS + e.recentTSTime = e.stack.Clock().NowMonotonic() +} + +// recentTimestamp returns the value of the recentTS field. +func (e *Endpoint) recentTimestamp() uint32 { + return e.RecentTS +} + +// TODO(gvisor.dev/issue/6974): Remove once tcp endpoints are composed with a +// network.Endpoint, which also defines this function. +func calculateTTL(route *stack.Route, ipv4TTL uint8, ipv6HopLimit int16) uint8 { + switch netProto := route.NetProto(); netProto { + case header.IPv4ProtocolNumber: + if ipv4TTL == tcpip.UseDefaultIPv4TTL { + return route.DefaultTTL() + } + return ipv4TTL + case header.IPv6ProtocolNumber: + if ipv6HopLimit == tcpip.UseDefaultIPv6HopLimit { + return route.DefaultTTL() + } + return uint8(ipv6HopLimit) + default: + panic(fmt.Sprintf("invalid protocol number = %d", netProto)) + } +} + +// keepalive is a synchronization wrapper used to appease stateify. See the +// comment in endpoint, where it is used. +// +// +stateify savable +type keepalive struct { + keepaliveMutex `state:"nosave"` + idle time.Duration + interval time.Duration + count int + unacked int + // should never be a zero timer if the endpoint is not closed. + timer timer `state:"nosave"` + waker sleep.Waker `state:"nosave"` +} + +func newEndpoint(s *stack.Stack, protocol *protocol, netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) *Endpoint { + e := &Endpoint{ + stack: s, + protocol: protocol, + TransportEndpointInfo: stack.TransportEndpointInfo{ + NetProto: netProto, + TransProto: header.TCPProtocolNumber, + }, + sndQueueInfo: sndQueueInfo{ + TCPSndBufState: TCPSndBufState{ + SndMTU: math.MaxInt32, + }, + }, + waiterQueue: waiterQueue, + state: atomicbitops.FromUint32(uint32(StateInitial)), + keepalive: keepalive{ + idle: DefaultKeepaliveIdle, + interval: DefaultKeepaliveInterval, + count: DefaultKeepaliveCount, + }, + ipv4TTL: tcpip.UseDefaultIPv4TTL, + ipv6HopLimit: tcpip.UseDefaultIPv6HopLimit, + // txHash only determines which outgoing queue to use, so + // InsecureRNG is fine. + txHash: s.InsecureRNG().Uint32(), + windowClamp: DefaultReceiveBufferSize, + maxSynRetries: DefaultSynRetries, + limRdr: &io.LimitedReader{}, + } + e.ops.InitHandler(e, e.stack, GetTCPSendBufferLimits, GetTCPReceiveBufferLimits) + e.ops.SetMulticastLoop(true) + e.ops.SetQuickAck(true) + e.ops.SetSendBufferSize(DefaultSendBufferSize, false /* notify */) + e.ops.SetReceiveBufferSize(DefaultReceiveBufferSize, false /* notify */) + + var ss tcpip.TCPSendBufferSizeRangeOption + if err := s.TransportProtocolOption(ProtocolNumber, &ss); err == nil { + e.ops.SetSendBufferSize(int64(ss.Default), false /* notify */) + } + + var rs tcpip.TCPReceiveBufferSizeRangeOption + if err := s.TransportProtocolOption(ProtocolNumber, &rs); err == nil { + e.ops.SetReceiveBufferSize(int64(rs.Default), false /* notify */) + } + + var cs tcpip.CongestionControlOption + if err := s.TransportProtocolOption(ProtocolNumber, &cs); err == nil { + e.cc = cs + } + + var mrb tcpip.TCPModerateReceiveBufferOption + if err := s.TransportProtocolOption(ProtocolNumber, &mrb); err == nil { + e.RcvAutoParams.Disabled = !bool(mrb) + } + + var de tcpip.TCPDelayEnabled + if err := s.TransportProtocolOption(ProtocolNumber, &de); err == nil && de { + e.ops.SetDelayOption(true) + } + + var tcpLT tcpip.TCPLingerTimeoutOption + if err := s.TransportProtocolOption(ProtocolNumber, &tcpLT); err == nil { + e.tcpLingerTimeout = time.Duration(tcpLT) + } + + var synRetries tcpip.TCPSynRetriesOption + if err := s.TransportProtocolOption(ProtocolNumber, &synRetries); err == nil { + e.maxSynRetries = uint8(synRetries) + } + + e.probe = protocol.probe + e.segmentQueue.ep = e + + // TODO(https://gvisor.dev/issues/7493): Defer creating the timer until TCP connection becomes + // established. + e.keepalive.timer.init(e.stack.Clock(), timerHandler(e, e.keepaliveTimerExpired)) + + return e +} + +// Readiness returns the current readiness of the endpoint. For example, if +// waiter.EventIn is set, the endpoint is immediately readable. +func (e *Endpoint) Readiness(mask waiter.EventMask) waiter.EventMask { + result := waiter.EventMask(0) + + switch e.EndpointState() { + case StateInitial, StateBound: + // This prevents blocking of new sockets which are not + // connected when SO_LINGER is set. + result |= waiter.EventHUp + + case StateConnecting, StateSynSent, StateSynRecv: + // Ready for nothing. + + case StateClose, StateError, StateTimeWait: + // Ready for anything. + result = mask + + case StateListen: + // Check if there's anything in the accepted queue. + if (mask & waiter.ReadableEvents) != 0 { + e.acceptMu.Lock() + if e.acceptQueue.endpoints.Len() != 0 { + result |= waiter.ReadableEvents + } + e.acceptMu.Unlock() + } + } + if e.EndpointState().connected() { + // Determine if the endpoint is writable if requested. + if (mask & waiter.WritableEvents) != 0 { + e.sndQueueInfo.sndQueueMu.Lock() + sndBufSize := e.getSendBufferSize() + if e.sndQueueInfo.SndClosed || e.sndQueueInfo.SndBufUsed < sndBufSize { + result |= waiter.WritableEvents + } + if e.sndQueueInfo.SndClosed { + e.updateConnDirectionState(connDirectionStateSndClosed) + } + e.sndQueueInfo.sndQueueMu.Unlock() + } + + // Determine if the endpoint is readable if requested. + if (mask & waiter.ReadableEvents) != 0 { + e.rcvQueueMu.Lock() + if e.RcvBufUsed > 0 || e.RcvClosed { + result |= waiter.ReadableEvents + } + if e.RcvClosed { + e.updateConnDirectionState(connDirectionStateRcvClosed) + } + e.rcvQueueMu.Unlock() + } + } + + // Determine whether endpoint is half-closed with rcv shutdown + if e.connDirectionState() == connDirectionStateRcvClosed { + result |= waiter.EventRdHUp + } + + return result +} + +// Purging pending rcv segments is only necessary on RST. +func (e *Endpoint) purgePendingRcvQueue() { + if e.rcv != nil { + for e.rcv.pendingRcvdSegments.Len() > 0 { + s := heap.Pop(&e.rcv.pendingRcvdSegments).(*segment) + s.DecRef() + } + } +} + +// +checklocks:e.mu +func (e *Endpoint) purgeReadQueue() { + if e.rcv != nil { + e.rcvQueueMu.Lock() + defer e.rcvQueueMu.Unlock() + for { + s := e.rcvQueue.Front() + if s == nil { + break + } + e.rcvQueue.Remove(s) + s.DecRef() + } + e.RcvBufUsed = 0 + } +} + +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) purgeWriteQueue() { + if e.snd != nil { + e.sndQueueInfo.sndQueueMu.Lock() + defer e.sndQueueInfo.sndQueueMu.Unlock() + e.snd.updateWriteNext(nil) + for s := e.snd.writeList.Front(); s != nil; s = e.snd.writeList.Front() { + e.snd.writeList.Remove(s) + s.DecRef() + } + e.sndQueueInfo.SndBufUsed = 0 + e.sndQueueInfo.SndClosed = true + e.snd.SndNxt = e.snd.SndUna + } +} + +// Abort implements stack.TransportEndpoint.Abort. +func (e *Endpoint) Abort() { + defer e.drainClosingSegmentQueue() + e.LockUser() + defer e.UnlockUser() + defer e.purgeReadQueue() + // Reset all connected endpoints. + switch state := e.EndpointState(); { + case state.connected(): + e.resetConnectionLocked(&tcpip.ErrAborted{}) + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + return + } + e.closeLocked() +} + +// Close puts the endpoint in a closed state and frees all resources associated +// with it. It must be called only once and with no other concurrent calls to +// the endpoint. +func (e *Endpoint) Close() { + e.LockUser() + if e.closed { + e.UnlockUser() + return + } + + // We always want to purge the read queue, but do so after the checks in + // shutdownLocked. + e.closeLocked() + e.purgeReadQueue() + if e.EndpointState() == StateClose || e.EndpointState() == StateError { + // It should be safe to purge the read queue now as the endpoint + // is now closed or in an error state and further reads are not + // permitted. + e.UnlockUser() + e.drainClosingSegmentQueue() + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + return + } + e.UnlockUser() +} + +// +checklocks:e.mu +func (e *Endpoint) closeLocked() { + linger := e.SocketOptions().GetLinger() + if linger.Enabled && linger.Timeout == 0 { + s := e.EndpointState() + isResetState := s == StateEstablished || s == StateCloseWait || s == StateFinWait1 || s == StateFinWait2 || s == StateSynRecv + if isResetState { + // Close the endpoint without doing full shutdown and + // send a RST. + e.resetConnectionLocked(&tcpip.ErrConnectionAborted{}) + return + } + } + + // Issue a shutdown so that the peer knows we won't send any more data + // if we're connected, or stop accepting if we're listening. + e.shutdownLocked(tcpip.ShutdownWrite | tcpip.ShutdownRead) + e.closeNoShutdownLocked() +} + +// closeNoShutdown closes the endpoint without doing a full shutdown. +// +checklocks:e.mu +func (e *Endpoint) closeNoShutdownLocked() { + // For listening sockets, we always release ports inline so that they + // are immediately available for reuse after Close() is called. If also + // registered, we unregister as well otherwise the next user would fail + // in Listen() when trying to register. + if e.EndpointState() == StateListen && e.isPortReserved { + if e.isRegistered { + e.stack.StartTransportEndpointCleanup(e.effectiveNetProtos, ProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice) + e.isRegistered = false + } + + portRes := ports.Reservation{ + Networks: e.effectiveNetProtos, + Transport: ProtocolNumber, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: e.TransportEndpointInfo.ID.LocalPort, + Flags: e.boundPortFlags, + BindToDevice: e.boundBindToDevice, + Dest: e.boundDest, + } + e.stack.ReleasePort(portRes) + e.isPortReserved = false + e.boundBindToDevice = 0 + e.boundPortFlags = ports.Flags{} + e.boundDest = tcpip.FullAddress{} + } + + // Mark endpoint as closed. + e.closed = true + tcpip.AddDanglingEndpoint(e) + + eventMask := waiter.ReadableEvents | waiter.WritableEvents + + switch e.EndpointState() { + case StateInitial, StateBound, StateListen: + e.setEndpointState(StateClose) + fallthrough + case StateClose, StateError: + eventMask |= waiter.EventHUp + e.cleanupLocked() + case StateConnecting, StateSynSent, StateSynRecv: + // Abort the handshake and set the error. + // Notify that the endpoint is closed. + eventMask |= waiter.EventHUp + e.handshakeFailed(&tcpip.ErrAborted{}) + // Notify that the endpoint is closed. + eventMask |= waiter.EventHUp + case StateFinWait2: + // The socket has been closed and we are in FIN-WAIT-2 so start + // the FIN-WAIT-2 timer. + if e.finWait2Timer == nil { + e.finWait2Timer = e.stack.Clock().AfterFunc(e.tcpLingerTimeout, e.finWait2TimerExpired) + } + } + + e.waiterQueue.Notify(eventMask) +} + +// closePendingAcceptableConnections closes all connections that have completed +// handshake but not yet been delivered to the application. +func (e *Endpoint) closePendingAcceptableConnectionsLocked() { + e.acceptMu.Lock() + + pendingEndpoints := e.acceptQueue.pendingEndpoints + e.acceptQueue.pendingEndpoints = nil + + completedEndpoints := make([]*Endpoint, 0, e.acceptQueue.endpoints.Len()) + for n := e.acceptQueue.endpoints.Front(); n != nil; n = n.Next() { + completedEndpoints = append(completedEndpoints, n.Value.(*Endpoint)) + } + e.acceptQueue.endpoints.Init() + e.acceptQueue.capacity = 0 + e.acceptMu.Unlock() + + // Close any endpoints in SYN-RCVD state. + for n := range pendingEndpoints { + n.Abort() + } + + // Reset all connections that are waiting to be accepted. + for _, n := range completedEndpoints { + n.Abort() + } +} + +// cleanupLocked frees all resources associated with the endpoint. +// +checklocks:e.mu +func (e *Endpoint) cleanupLocked() { + if e.snd != nil { + e.snd.resendTimer.cleanup() + e.snd.probeTimer.cleanup() + e.snd.reorderTimer.cleanup() + e.snd.corkTimer.cleanup() + } + + if e.finWait2Timer != nil { + e.finWait2Timer.Stop() + } + + if e.timeWaitTimer != nil { + e.timeWaitTimer.Stop() + } + + // Remove current EP from its lEP acceptQueue.pendingEndpoint if exists. + if e.h != nil && e.h.listenEP != nil { + lEP := e.h.listenEP + lEP.acceptMu.Lock() + delete(lEP.acceptQueue.pendingEndpoints, e) + lEP.acceptMu.Unlock() + } + + // Close all endpoints that might have been accepted by TCP but not by + // the client. + e.closePendingAcceptableConnectionsLocked() + e.keepalive.timer.cleanup() + + if e.isRegistered { + e.stack.StartTransportEndpointCleanup(e.effectiveNetProtos, ProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice) + e.isRegistered = false + } + + if e.isPortReserved { + portRes := ports.Reservation{ + Networks: e.effectiveNetProtos, + Transport: ProtocolNumber, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: e.TransportEndpointInfo.ID.LocalPort, + Flags: e.boundPortFlags, + BindToDevice: e.boundBindToDevice, + Dest: e.boundDest, + } + e.stack.ReleasePort(portRes) + e.isPortReserved = false + } + e.boundBindToDevice = 0 + e.boundPortFlags = ports.Flags{} + e.boundDest = tcpip.FullAddress{} + + if e.route != nil { + e.route.Release() + e.route = nil + } + + e.purgeWriteQueue() + // Only purge the read queue here if the socket is fully closed by the + // user. + if e.closed { + e.purgeReadQueue() + } + e.stack.CompleteTransportEndpointCleanup(e) + tcpip.DeleteDanglingEndpoint(e) +} + +// wndFromSpace returns the window that we can advertise based on the available +// receive buffer space. +func wndFromSpace(space int) int { + return space >> rcvAdvWndScale +} + +// initialReceiveWindow returns the initial receive window to advertise in the +// SYN/SYN-ACK. +func (e *Endpoint) initialReceiveWindow() int { + rcvWnd := wndFromSpace(e.receiveBufferAvailable()) + if rcvWnd > math.MaxUint16 { + rcvWnd = math.MaxUint16 + } + + // Use the user supplied MSS, if available. + routeWnd := InitialCwnd * int(calculateAdvertisedMSS(e.userMSS, e.route)) * 2 + if rcvWnd > routeWnd { + rcvWnd = routeWnd + } + rcvWndScale := e.rcvWndScaleForHandshake() + + // Round-down the rcvWnd to a multiple of wndScale. This ensures that the + // window offered in SYN won't be reduced due to the loss of precision if + // window scaling is enabled after the handshake. + rcvWnd = (rcvWnd >> uint8(rcvWndScale)) << uint8(rcvWndScale) + + // Ensure we can always accept at least 1 byte if the scale specified + // was too high for the provided rcvWnd. + if rcvWnd == 0 { + rcvWnd = 1 + } + + return rcvWnd +} + +// ModerateRecvBuf adjusts the receive buffer and the advertised window +// based on the number of bytes copied to userspace. +func (e *Endpoint) ModerateRecvBuf(copied int) { + e.LockUser() + defer e.UnlockUser() + + sendNonZeroWindowUpdate := false + + e.rcvQueueMu.Lock() + if e.RcvAutoParams.Disabled { + e.rcvQueueMu.Unlock() + return + } + now := e.stack.Clock().NowMonotonic() + if rtt := e.RcvAutoParams.RTT; rtt == 0 || now.Sub(e.RcvAutoParams.MeasureTime) < rtt { + e.RcvAutoParams.CopiedBytes += copied + e.rcvQueueMu.Unlock() + return + } + prevRTTCopied := e.RcvAutoParams.CopiedBytes + copied + prevCopied := e.RcvAutoParams.PrevCopiedBytes + rcvWnd := 0 + if prevRTTCopied > prevCopied { + // The minimal receive window based on what was copied by the app + // in the immediate preceding RTT and some extra buffer for 16 + // segments to account for variations. + // We multiply by 2 to account for packet losses. + rcvWnd = prevRTTCopied*2 + 16*int(e.amss) + + // Scale for slow start based on bytes copied in this RTT vs previous. + grow := (rcvWnd * (prevRTTCopied - prevCopied)) / prevCopied + + // Multiply growth factor by 2 again to account for sender being + // in slow-start where the sender grows it's congestion window + // by 100% per RTT. + rcvWnd += grow * 2 + + // Make sure auto tuned buffer size can always receive upto 2x + // the initial window of 10 segments. + if minRcvWnd := int(e.amss) * InitialCwnd * 2; rcvWnd < minRcvWnd { + rcvWnd = minRcvWnd + } + + // Cap the auto tuned buffer size by the maximum permissible + // receive buffer size. + if max := e.maxReceiveBufferSize(); rcvWnd > max { + rcvWnd = max + } + + // We do not adjust downwards as that can cause the receiver to + // reject valid data that might already be in flight as the + // acceptable window will shrink. + rcvBufSize := int(e.ops.GetReceiveBufferSize()) + if rcvWnd > rcvBufSize { + availBefore := wndFromSpace(e.receiveBufferAvailableLocked(rcvBufSize)) + e.ops.SetReceiveBufferSize(int64(rcvWnd), false /* notify */) + availAfter := wndFromSpace(e.receiveBufferAvailableLocked(rcvWnd)) + if crossed, above := e.windowCrossedACKThresholdLocked(availAfter-availBefore, rcvBufSize); crossed && above { + sendNonZeroWindowUpdate = true + } + } + + // We only update PrevCopiedBytes when we grow the buffer because in cases + // where PrevCopiedBytes > prevRTTCopied the existing buffer is already big + // enough to handle the current rate and we don't need to do any + // adjustments. + e.RcvAutoParams.PrevCopiedBytes = prevRTTCopied + } + e.RcvAutoParams.MeasureTime = now + e.RcvAutoParams.CopiedBytes = 0 + e.rcvQueueMu.Unlock() + + // Send the update after unlocking rcvQueueMu as sending a segment acquires + // the lock to calculate the window to be sent. + if e.EndpointState().connected() && sendNonZeroWindowUpdate { + e.rcv.nonZeroWindow() // +checklocksforce:e.rcv.ep.mu + } +} + +// SetOwner implements tcpip.Endpoint.SetOwner. +func (e *Endpoint) SetOwner(owner tcpip.PacketOwner) { + e.owner = owner +} + +// +checklocks:e.mu +func (e *Endpoint) hardErrorLocked() tcpip.Error { + err := e.hardError + e.hardError = nil + return err +} + +// +checklocks:e.mu +func (e *Endpoint) lastErrorLocked() tcpip.Error { + e.lastErrorMu.Lock() + defer e.lastErrorMu.Unlock() + err := e.lastError + e.lastError = nil + return err +} + +// LastError implements tcpip.Endpoint.LastError. +func (e *Endpoint) LastError() tcpip.Error { + e.LockUser() + defer e.UnlockUser() + if err := e.hardErrorLocked(); err != nil { + return err + } + return e.lastErrorLocked() +} + +// LastErrorLocked reads and clears lastError. +// Only to be used in tests. +// +checklocks:e.mu +func (e *Endpoint) LastErrorLocked() tcpip.Error { + return e.lastErrorLocked() +} + +// UpdateLastError implements tcpip.SocketOptionsHandler.UpdateLastError. +func (e *Endpoint) UpdateLastError(err tcpip.Error) { + e.LockUser() + e.lastErrorMu.Lock() + e.lastError = err + e.lastErrorMu.Unlock() + e.UnlockUser() +} + +// Read implements tcpip.Endpoint.Read. +func (e *Endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult, tcpip.Error) { + e.LockUser() + defer e.UnlockUser() + + if err := e.checkReadLocked(); err != nil { + if _, ok := err.(*tcpip.ErrClosedForReceive); ok { + e.stats.ReadErrors.ReadClosed.Increment() + } + return tcpip.ReadResult{}, err + } + + var err error + done := 0 + // N.B. Here we get the first segment to be processed. It is safe to not + // hold rcvQueueMu when processing, since we hold e.mu to ensure we only + // remove segments from the list through Read() and that new segments + // cannot be appended. + s := e.rcvQueue.Front() + for s != nil { + var n int + n, err = s.ReadTo(dst, opts.Peek) + // Book keeping first then error handling. + done += n + + if opts.Peek { + s = s.Next() + } else { + sendNonZeroWindowUpdate := false + memDelta := 0 + for { + seg := e.rcvQueue.Front() + if seg == nil || seg.payloadSize() != 0 { + break + } + e.rcvQueue.Remove(seg) + // Memory is only considered released when the whole segment has been + // read. + memDelta += seg.segMemSize() + seg.DecRef() + } + e.rcvQueueMu.Lock() + e.RcvBufUsed -= n + s = e.rcvQueue.Front() + + if memDelta > 0 { + // If the window was small before this read and if the read freed up + // enough buffer space, to either fit an aMSS or half a receive buffer + // (whichever smaller), then send a window update. + if crossed, above := e.windowCrossedACKThresholdLocked(memDelta, int(e.ops.GetReceiveBufferSize())); crossed && above { + sendNonZeroWindowUpdate = true + } + } + e.rcvQueueMu.Unlock() + + if e.EndpointState().connected() && sendNonZeroWindowUpdate { + e.rcv.nonZeroWindow() // +checklocksforce:e.rcv.ep.mu + } + } + + if err != nil { + break + } + } + + // If something is read, we must report it. Report error when nothing is read. + if done == 0 && err != nil { + return tcpip.ReadResult{}, &tcpip.ErrBadBuffer{} + } + return tcpip.ReadResult{ + Count: done, + Total: done, + }, nil +} + +// checkRead checks that endpoint is in a readable state. +// +// +checklocks:e.mu +func (e *Endpoint) checkReadLocked() tcpip.Error { + e.rcvQueueMu.Lock() + defer e.rcvQueueMu.Unlock() + // When in SYN-SENT state, let the caller block on the receive. + // An application can initiate a non-blocking connect and then block + // on a receive. It can expect to read any data after the handshake + // is complete. RFC793, section 3.9, p58. + if e.EndpointState() == StateSynSent { + return &tcpip.ErrWouldBlock{} + } + + // The endpoint can be read if it's connected, or if it's already closed + // but has some pending unread data. Also note that a RST being received + // would cause the state to become StateError so we should allow the + // reads to proceed before returning a ECONNRESET. + bufUsed := e.RcvBufUsed + if s := e.EndpointState(); !s.connected() && s != StateClose && bufUsed == 0 { + if s == StateError { + if err := e.hardErrorLocked(); err != nil { + return err + } + return &tcpip.ErrClosedForReceive{} + } + e.stats.ReadErrors.NotConnected.Increment() + return &tcpip.ErrNotConnected{} + } + + if e.RcvBufUsed == 0 { + if e.RcvClosed || !e.EndpointState().connected() { + return &tcpip.ErrClosedForReceive{} + } + return &tcpip.ErrWouldBlock{} + } + + return nil +} + +// isEndpointWritableLocked checks if a given endpoint is writable +// and also returns the number of bytes that can be written at this +// moment. If the endpoint is not writable then it returns an error +// indicating the reason why it's not writable. +// +checklocks:e.mu +// +checklocks:e.sndQueueInfo.sndQueueMu +func (e *Endpoint) isEndpointWritableLocked() (int, tcpip.Error) { + // The endpoint cannot be written to if it's not connected. + switch s := e.EndpointState(); { + case s == StateError: + if err := e.hardErrorLocked(); err != nil { + return 0, err + } + return 0, &tcpip.ErrClosedForSend{} + case !s.connecting() && !s.connected(): + return 0, &tcpip.ErrClosedForSend{} + case s.connecting(): + // As per RFC793, page 56, a send request arriving when in connecting + // state, can be queued to be completed after the state becomes + // connected. Return an error code for the caller of endpoint Write to + // try again, until the connection handshake is complete. + return 0, &tcpip.ErrWouldBlock{} + } + + // Check if the connection has already been closed for sends. + if e.sndQueueInfo.SndClosed { + return 0, &tcpip.ErrClosedForSend{} + } + + sndBufSize := e.getSendBufferSize() + avail := sndBufSize - e.sndQueueInfo.SndBufUsed + if avail <= 0 { + return 0, &tcpip.ErrWouldBlock{} + } + return avail, nil +} + +// readFromPayloader reads a slice from the Payloader. +// +checklocks:e.mu +// +checklocks:e.sndQueueInfo.sndQueueMu +func (e *Endpoint) readFromPayloader(p tcpip.Payloader, opts tcpip.WriteOptions, avail int) (buffer.Buffer, tcpip.Error) { + // We can release locks while copying data. + // + // This is not possible if atomic is set, because we can't allow the + // available buffer space to be consumed by some other caller while we + // are copying data in. + limRdr := e.limRdr + if !opts.Atomic { + defer func() { + e.limRdr = limRdr + }() + e.limRdr = nil + + e.sndQueueInfo.sndQueueMu.Unlock() + defer e.sndQueueInfo.sndQueueMu.Lock() + + e.UnlockUser() + defer e.LockUser() + } + + // Fetch data. + var payload buffer.Buffer + if l := p.Len(); l < avail { + avail = l + } + if avail == 0 { + return payload, nil + } + if _, err := payload.WriteFromReaderAndLimitedReader(p, int64(avail), limRdr); err != nil { + payload.Release() + return buffer.Buffer{}, &tcpip.ErrBadBuffer{} + } + return payload, nil +} + +// queueSegment reads data from the payloader and returns a segment to be sent. +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) queueSegment(p tcpip.Payloader, opts tcpip.WriteOptions) (*segment, int, tcpip.Error) { + e.sndQueueInfo.sndQueueMu.Lock() + defer e.sndQueueInfo.sndQueueMu.Unlock() + + avail, err := e.isEndpointWritableLocked() + if err != nil { + e.stats.WriteErrors.WriteClosed.Increment() + return nil, 0, err + } + + buf, err := e.readFromPayloader(p, opts, avail) + if err != nil { + return nil, 0, err + } + + // Do not queue zero length segments. + if buf.Size() == 0 { + return nil, 0, nil + } + + if !opts.Atomic { + // Since we released locks in between it's possible that the + // endpoint transitioned to a CLOSED/ERROR states so make + // sure endpoint is still writable before trying to write. + avail, err := e.isEndpointWritableLocked() + if err != nil { + e.stats.WriteErrors.WriteClosed.Increment() + buf.Release() + return nil, 0, err + } + + // A simultaneous call to write on the socket can reduce avail. Discard + // excess data copied if this is the case. + if int64(avail) < buf.Size() { + buf.Truncate(int64(avail)) + } + } + + // Add data to the send queue. + size := int(buf.Size()) + s := newOutgoingSegment(e.TransportEndpointInfo.ID, e.stack.Clock(), buf) + e.sndQueueInfo.SndBufUsed += size + e.snd.writeList.PushBack(s) + + return s, size, nil +} + +// Write writes data to the endpoint's peer. +func (e *Endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + // Linux completely ignores any address passed to sendto(2) for TCP sockets + // (without the MSG_FASTOPEN flag). Corking is unimplemented, so opts.More + // and opts.EndOfRecord are also ignored. + + e.LockUser() + defer e.UnlockUser() + + // Return if either we didn't queue anything or if an error occurred while + // attempting to queue data. + nextSeg, n, err := e.queueSegment(p, opts) + if n == 0 || err != nil { + return 0, err + } + + e.sendData(nextSeg) + return int64(n), nil +} + +// selectWindowLocked returns the new window without checking for shrinking or scaling +// applied. +// +checklocks:e.mu +// +checklocks:e.rcvQueueMu +func (e *Endpoint) selectWindowLocked(rcvBufSize int) (wnd seqnum.Size) { + wndFromAvailable := wndFromSpace(e.receiveBufferAvailableLocked(rcvBufSize)) + maxWindow := wndFromSpace(rcvBufSize) + wndFromUsedBytes := maxWindow - e.RcvBufUsed + + // We take the lesser of the wndFromAvailable and wndFromUsedBytes because in + // cases where we receive a lot of small segments the segment overhead is a + // lot higher and we can run out socket buffer space before we can fill the + // previous window we advertised. In cases where we receive MSS sized or close + // MSS sized segments we will probably run out of window space before we + // exhaust receive buffer. + newWnd := wndFromAvailable + if newWnd > wndFromUsedBytes { + newWnd = wndFromUsedBytes + } + if newWnd < 0 { + newWnd = 0 + } + return seqnum.Size(newWnd) +} + +// selectWindow invokes selectWindowLocked after acquiring e.rcvQueueMu. +// +checklocks:e.mu +func (e *Endpoint) selectWindow() (wnd seqnum.Size) { + e.rcvQueueMu.Lock() + wnd = e.selectWindowLocked(int(e.ops.GetReceiveBufferSize())) + e.rcvQueueMu.Unlock() + return wnd +} + +// windowCrossedACKThresholdLocked checks if the receive window to be announced +// would be under aMSS or under the window derived from half receive buffer, +// whichever smaller. This is useful as a receive side silly window syndrome +// prevention mechanism. If window grows to reasonable value, we should send ACK +// to the sender to inform the rx space is now large. We also want ensure a +// series of small read()'s won't trigger a flood of spurious tiny ACK's. +// +// For large receive buffers, the threshold is aMSS - once reader reads more +// than aMSS we'll send ACK. For tiny receive buffers, the threshold is half of +// receive buffer size. This is chosen arbitrarily. +// crossed will be true if the window size crossed the ACK threshold. +// above will be true if the new window is >= ACK threshold and false +// otherwise. +// +// +checklocks:e.mu +// +checklocks:e.rcvQueueMu +func (e *Endpoint) windowCrossedACKThresholdLocked(deltaBefore int, rcvBufSize int) (crossed bool, above bool) { + newAvail := int(e.selectWindowLocked(rcvBufSize)) + oldAvail := newAvail - deltaBefore + if oldAvail < 0 { + oldAvail = 0 + } + threshold := int(e.amss) + // rcvBufFraction is the inverse of the fraction of receive buffer size that + // is used to decide if the available buffer space is now above it. + const rcvBufFraction = 2 + if wndThreshold := wndFromSpace(rcvBufSize / rcvBufFraction); threshold > wndThreshold { + threshold = wndThreshold + } + + switch { + case oldAvail < threshold && newAvail >= threshold: + return true, true + case oldAvail >= threshold && newAvail < threshold: + return true, false + } + return false, false +} + +// OnReuseAddressSet implements tcpip.SocketOptionsHandler.OnReuseAddressSet. +func (e *Endpoint) OnReuseAddressSet(v bool) { + e.LockUser() + e.portFlags.TupleOnly = v + e.UnlockUser() +} + +// OnReusePortSet implements tcpip.SocketOptionsHandler.OnReusePortSet. +func (e *Endpoint) OnReusePortSet(v bool) { + e.LockUser() + e.portFlags.LoadBalanced = v + e.UnlockUser() +} + +// OnKeepAliveSet implements tcpip.SocketOptionsHandler.OnKeepAliveSet. +func (e *Endpoint) OnKeepAliveSet(bool) { + e.LockUser() + e.resetKeepaliveTimer(true /* receivedData */) + e.UnlockUser() +} + +// OnDelayOptionSet implements tcpip.SocketOptionsHandler.OnDelayOptionSet. +func (e *Endpoint) OnDelayOptionSet(v bool) { + if !v { + e.LockUser() + defer e.UnlockUser() + // Handle delayed data. + if e.EndpointState().connected() { + e.sendData(nil /* next */) + } + } +} + +// OnCorkOptionSet implements tcpip.SocketOptionsHandler.OnCorkOptionSet. +func (e *Endpoint) OnCorkOptionSet(v bool) { + if !v { + e.LockUser() + defer e.UnlockUser() + if e.snd != nil { + e.snd.corkTimer.disable() + } + // Handle the corked data. + if e.EndpointState().connected() { + e.sendData(nil /* next */) + } + } +} + +func (e *Endpoint) getSendBufferSize() int { + return int(e.ops.GetSendBufferSize()) +} + +// OnSetReceiveBufferSize implements tcpip.SocketOptionsHandler.OnSetReceiveBufferSize. +func (e *Endpoint) OnSetReceiveBufferSize(rcvBufSz, oldSz int64) (newSz int64, postSet func()) { + e.LockUser() + + sendNonZeroWindowUpdate := false + e.rcvQueueMu.Lock() + + // Make sure the receive buffer size allows us to send a + // non-zero window size. + scale := uint8(0) + if e.rcv != nil { + scale = e.rcv.RcvWndScale + } + if rcvBufSz>>scale == 0 { + rcvBufSz = 1 << scale + } + + availBefore := wndFromSpace(e.receiveBufferAvailableLocked(int(oldSz))) + availAfter := wndFromSpace(e.receiveBufferAvailableLocked(int(rcvBufSz))) + e.RcvAutoParams.Disabled = true + + // Immediately send an ACK to uncork the sender silly window + // syndrome prevetion, when our available space grows above aMSS + // or half receive buffer, whichever smaller. + if crossed, above := e.windowCrossedACKThresholdLocked(availAfter-availBefore, int(rcvBufSz)); crossed && above { + sendNonZeroWindowUpdate = true + } + + e.rcvQueueMu.Unlock() + + postSet = func() { + e.LockUser() + defer e.UnlockUser() + if e.EndpointState().connected() && sendNonZeroWindowUpdate { + e.rcv.nonZeroWindow() // +checklocksforce:e.rcv.ep.mu + } + } + e.UnlockUser() + return rcvBufSz, postSet +} + +// OnSetSendBufferSize implements tcpip.SocketOptionsHandler.OnSetSendBufferSize. +func (e *Endpoint) OnSetSendBufferSize(sz int64) int64 { + e.sndQueueInfo.TCPSndBufState.AutoTuneSndBufDisabled.Store(1) + return sz +} + +// WakeupWriters implements tcpip.SocketOptionsHandler.WakeupWriters. +func (e *Endpoint) WakeupWriters() { + e.LockUser() + defer e.UnlockUser() + + sendBufferSize := e.getSendBufferSize() + e.sndQueueInfo.sndQueueMu.Lock() + notify := (sendBufferSize - e.sndQueueInfo.SndBufUsed) >= e.sndQueueInfo.SndBufUsed>>1 + e.sndQueueInfo.sndQueueMu.Unlock() + + if notify { + e.waiterQueue.Notify(waiter.WritableEvents) + } +} + +// SetSockOptInt sets a socket option. +func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { + // Lower 2 bits represents ECN bits. RFC 3168, section 23.1 + const inetECNMask = 3 + + switch opt { + case tcpip.KeepaliveCountOption: + e.LockUser() + e.keepalive.Lock() + e.keepalive.count = v + e.keepalive.Unlock() + e.resetKeepaliveTimer(true /* receivedData */) + e.UnlockUser() + + case tcpip.IPv4TOSOption: + e.LockUser() + // TODO(gvisor.dev/issue/995): ECN is not currently supported, + // ignore the bits for now. + e.sendTOS = uint8(v) & ^uint8(inetECNMask) + e.UnlockUser() + + case tcpip.IPv6TrafficClassOption: + e.LockUser() + // TODO(gvisor.dev/issue/995): ECN is not currently supported, + // ignore the bits for now. + e.sendTOS = uint8(v) & ^uint8(inetECNMask) + e.UnlockUser() + + case tcpip.MaxSegOption: + userMSS := v + if userMSS < header.TCPMinimumMSS || userMSS > header.TCPMaximumMSS { + return &tcpip.ErrInvalidOptionValue{} + } + e.LockUser() + e.userMSS = uint16(userMSS) + e.UnlockUser() + + case tcpip.MTUDiscoverOption: + switch v := tcpip.PMTUDStrategy(v); v { + case tcpip.PMTUDiscoveryWant, tcpip.PMTUDiscoveryDont, tcpip.PMTUDiscoveryDo: + e.LockUser() + e.pmtud = v + e.UnlockUser() + case tcpip.PMTUDiscoveryProbe: + // We don't support a way to ignore MTU updates; it's + // either on or it's off. + return &tcpip.ErrNotSupported{} + default: + return &tcpip.ErrNotSupported{} + } + + case tcpip.IPv4TTLOption: + e.LockUser() + e.ipv4TTL = uint8(v) + e.UnlockUser() + + case tcpip.IPv6HopLimitOption: + e.LockUser() + e.ipv6HopLimit = int16(v) + e.UnlockUser() + + case tcpip.TCPSynCountOption: + if v < 1 || v > 255 { + return &tcpip.ErrInvalidOptionValue{} + } + e.LockUser() + e.maxSynRetries = uint8(v) + e.UnlockUser() + + case tcpip.TCPWindowClampOption: + if v == 0 { + e.LockUser() + switch e.EndpointState() { + case StateClose, StateInitial: + e.windowClamp = 0 + e.UnlockUser() + return nil + default: + e.UnlockUser() + return &tcpip.ErrInvalidOptionValue{} + } + } + var rs tcpip.TCPReceiveBufferSizeRangeOption + if err := e.stack.TransportProtocolOption(ProtocolNumber, &rs); err == nil { + if v < rs.Min/2 { + v = rs.Min / 2 + } + } + e.LockUser() + e.windowClamp = uint32(v) + e.UnlockUser() + } + return nil +} + +// HasNIC returns true if the NICID is defined in the stack or id is 0. +func (e *Endpoint) HasNIC(id int32) bool { + return id == 0 || e.stack.HasNIC(tcpip.NICID(id)) +} + +// SetSockOpt sets a socket option. +func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { + switch v := opt.(type) { + case *tcpip.KeepaliveIdleOption: + e.LockUser() + e.keepalive.Lock() + e.keepalive.idle = time.Duration(*v) + e.keepalive.Unlock() + e.resetKeepaliveTimer(true /* receivedData */) + e.UnlockUser() + + case *tcpip.KeepaliveIntervalOption: + e.LockUser() + e.keepalive.Lock() + e.keepalive.interval = time.Duration(*v) + e.keepalive.Unlock() + e.resetKeepaliveTimer(true /* receivedData */) + e.UnlockUser() + + case *tcpip.TCPUserTimeoutOption: + e.LockUser() + e.userTimeout = time.Duration(*v) + e.UnlockUser() + + case *tcpip.CongestionControlOption: + // Query the available cc algorithms in the stack and + // validate that the specified algorithm is actually + // supported in the stack. + var avail tcpip.TCPAvailableCongestionControlOption + if err := e.stack.TransportProtocolOption(ProtocolNumber, &avail); err != nil { + return err + } + availCC := strings.Split(string(avail), " ") + for _, cc := range availCC { + if *v == tcpip.CongestionControlOption(cc) { + e.LockUser() + state := e.EndpointState() + e.cc = *v + switch state { + case StateEstablished: + if e.EndpointState() == state { + e.snd.cc = e.snd.initCongestionControl(e.cc) + } + } + e.UnlockUser() + return nil + } + } + + // Linux returns ENOENT when an invalid congestion + // control algorithm is specified. + return &tcpip.ErrNoSuchFile{} + + case *tcpip.TCPLingerTimeoutOption: + e.LockUser() + + switch { + case *v < 0: + // Same as effectively disabling TCPLinger timeout. + *v = -1 + case *v == 0: + // Same as the stack default. + var stackLingerTimeout tcpip.TCPLingerTimeoutOption + if err := e.stack.TransportProtocolOption(ProtocolNumber, &stackLingerTimeout); err != nil { + panic(fmt.Sprintf("e.stack.TransportProtocolOption(%d, %+v) = %v", ProtocolNumber, &stackLingerTimeout, err)) + } + *v = stackLingerTimeout + case *v > tcpip.TCPLingerTimeoutOption(MaxTCPLingerTimeout): + // Cap it to Stack's default TCP_LINGER2 timeout. + *v = tcpip.TCPLingerTimeoutOption(MaxTCPLingerTimeout) + default: + } + + e.tcpLingerTimeout = time.Duration(*v) + e.UnlockUser() + + case *tcpip.TCPDeferAcceptOption: + e.LockUser() + if time.Duration(*v) > MaxRTO { + *v = tcpip.TCPDeferAcceptOption(MaxRTO) + } + e.deferAccept = time.Duration(*v) + e.UnlockUser() + + case *tcpip.SocketDetachFilterOption: + return nil + + default: + return nil + } + return nil +} + +// readyReceiveSize returns the number of bytes ready to be received. +func (e *Endpoint) readyReceiveSize() (int, tcpip.Error) { + e.LockUser() + defer e.UnlockUser() + + // The endpoint cannot be in listen state. + if e.EndpointState() == StateListen { + return 0, &tcpip.ErrInvalidEndpointState{} + } + + e.rcvQueueMu.Lock() + defer e.rcvQueueMu.Unlock() + + return e.RcvBufUsed, nil +} + +// GetSockOptInt implements tcpip.Endpoint.GetSockOptInt. +func (e *Endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { + switch opt { + case tcpip.KeepaliveCountOption: + e.keepalive.Lock() + v := e.keepalive.count + e.keepalive.Unlock() + return v, nil + + case tcpip.IPv4TOSOption: + e.LockUser() + v := int(e.sendTOS) + e.UnlockUser() + return v, nil + + case tcpip.IPv6TrafficClassOption: + e.LockUser() + v := int(e.sendTOS) + e.UnlockUser() + return v, nil + + case tcpip.MaxSegOption: + // Linux only returns user_mss value if user_mss is set and the socket is + // unconnected. Otherwise Linux returns the actual current MSS. Netstack + // mimics the user_mss behavior, but otherwise just returns the defaultMSS + // for now. + v := header.TCPDefaultMSS + e.LockUser() + if state := e.EndpointState(); e.userMSS > 0 && (state.internal() || state == StateClose || state == StateListen) { + v = int(e.userMSS) + } + e.UnlockUser() + return v, nil + + case tcpip.MTUDiscoverOption: + e.LockUser() + v := e.pmtud + e.UnlockUser() + return int(v), nil + + case tcpip.ReceiveQueueSizeOption: + return e.readyReceiveSize() + + case tcpip.IPv4TTLOption: + e.LockUser() + v := int(e.ipv4TTL) + e.UnlockUser() + return v, nil + + case tcpip.IPv6HopLimitOption: + e.LockUser() + v := int(e.ipv6HopLimit) + e.UnlockUser() + return v, nil + + case tcpip.TCPSynCountOption: + e.LockUser() + v := int(e.maxSynRetries) + e.UnlockUser() + return v, nil + + case tcpip.TCPWindowClampOption: + e.LockUser() + v := int(e.windowClamp) + e.UnlockUser() + return v, nil + + case tcpip.MulticastTTLOption: + return 1, nil + + default: + return -1, &tcpip.ErrUnknownProtocolOption{} + } +} + +func (e *Endpoint) getTCPInfo() tcpip.TCPInfoOption { + info := tcpip.TCPInfoOption{} + e.LockUser() + if state := e.EndpointState(); state.internal() { + info.State = tcpip.EndpointState(StateClose) + } else { + info.State = tcpip.EndpointState(state) + } + snd := e.snd + if snd != nil { + // We do not calculate RTT before sending the data packets. If + // the connection did not send and receive data, then RTT will + // be zero. + snd.rtt.Lock() + info.RTT = snd.rtt.TCPRTTState.SRTT + info.RTTVar = snd.rtt.TCPRTTState.RTTVar + snd.rtt.Unlock() + + info.RTO = snd.RTO + info.CcState = snd.state + info.SndSsthresh = uint32(snd.Ssthresh) + info.SndCwnd = uint32(snd.SndCwnd) + info.ReorderSeen = snd.rc.Reord + } + e.UnlockUser() + return info +} + +// GetSockOpt implements tcpip.Endpoint.GetSockOpt. +func (e *Endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { + switch o := opt.(type) { + case *tcpip.TCPInfoOption: + *o = e.getTCPInfo() + + case *tcpip.KeepaliveIdleOption: + e.keepalive.Lock() + *o = tcpip.KeepaliveIdleOption(e.keepalive.idle) + e.keepalive.Unlock() + + case *tcpip.KeepaliveIntervalOption: + e.keepalive.Lock() + *o = tcpip.KeepaliveIntervalOption(e.keepalive.interval) + e.keepalive.Unlock() + + case *tcpip.TCPUserTimeoutOption: + e.LockUser() + *o = tcpip.TCPUserTimeoutOption(e.userTimeout) + e.UnlockUser() + + case *tcpip.CongestionControlOption: + e.LockUser() + *o = e.cc + e.UnlockUser() + + case *tcpip.TCPLingerTimeoutOption: + e.LockUser() + *o = tcpip.TCPLingerTimeoutOption(e.tcpLingerTimeout) + e.UnlockUser() + + case *tcpip.TCPDeferAcceptOption: + e.LockUser() + *o = tcpip.TCPDeferAcceptOption(e.deferAccept) + e.UnlockUser() + + case *tcpip.OriginalDestinationOption: + e.LockUser() + ipt := e.stack.IPTables() + addr, port, err := ipt.OriginalDst(e.TransportEndpointInfo.ID, e.NetProto, ProtocolNumber) + e.UnlockUser() + if err != nil { + return err + } + *o = tcpip.OriginalDestinationOption{ + Addr: addr, + Port: port, + } + + default: + return &tcpip.ErrUnknownProtocolOption{} + } + return nil +} + +// checkV4MappedLocked determines the effective network protocol and converts +// addr to its canonical form. +// +checklocks:e.mu +func (e *Endpoint) checkV4MappedLocked(addr tcpip.FullAddress, bind bool) (tcpip.FullAddress, tcpip.NetworkProtocolNumber, tcpip.Error) { + unwrapped, netProto, err := e.TransportEndpointInfo.AddrNetProtoLocked(addr, e.ops.GetV6Only(), bind) + if err != nil { + return tcpip.FullAddress{}, 0, err + } + return unwrapped, netProto, nil +} + +// Disconnect implements tcpip.Endpoint.Disconnect. +func (*Endpoint) Disconnect() tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Connect connects the endpoint to its peer. +func (e *Endpoint) Connect(addr tcpip.FullAddress) tcpip.Error { + e.LockUser() + defer e.UnlockUser() + err := e.connect(addr, true) + if err != nil { + if !err.IgnoreStats() { + // Connect failed. Let's wake up any waiters. + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + e.stack.Stats().TCP.FailedConnectionAttempts.Increment() + e.stats.FailedConnectionAttempts.Increment() + } + } + return err +} + +// registerEndpoint registers the endpoint with the provided address. +// +// +checklocks:e.mu +func (e *Endpoint) registerEndpoint(addr tcpip.FullAddress, netProto tcpip.NetworkProtocolNumber, nicID tcpip.NICID) tcpip.Error { + netProtos := []tcpip.NetworkProtocolNumber{netProto} + if e.TransportEndpointInfo.ID.LocalPort != 0 { + // The endpoint is bound to a port, attempt to register it. + err := e.stack.RegisterTransportEndpoint(netProtos, ProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice) + if err != nil { + return err + } + } else { + // The endpoint doesn't have a local port yet, so try to get + // one. Make sure that it isn't one that will result in the same + // address/port for both local and remote (otherwise this + // endpoint would be trying to connect to itself). + sameAddr := e.TransportEndpointInfo.ID.LocalAddress == e.TransportEndpointInfo.ID.RemoteAddress + + var twReuse tcpip.TCPTimeWaitReuseOption + if err := e.stack.TransportProtocolOption(ProtocolNumber, &twReuse); err != nil { + panic(fmt.Sprintf("e.stack.TransportProtocolOption(%d, %#v) = %s", ProtocolNumber, &twReuse, err)) + } + + reuse := twReuse == tcpip.TCPTimeWaitReuseGlobal + if twReuse == tcpip.TCPTimeWaitReuseLoopbackOnly { + switch netProto { + case header.IPv4ProtocolNumber: + reuse = header.IsV4LoopbackAddress(e.TransportEndpointInfo.ID.LocalAddress) && header.IsV4LoopbackAddress(e.TransportEndpointInfo.ID.RemoteAddress) + case header.IPv6ProtocolNumber: + reuse = e.TransportEndpointInfo.ID.LocalAddress == header.IPv6Loopback && e.TransportEndpointInfo.ID.RemoteAddress == header.IPv6Loopback + } + } + + bindToDevice := tcpip.NICID(e.ops.GetBindToDevice()) + if _, err := e.stack.PickEphemeralPort(e.stack.SecureRNG(), func(p uint16) (bool, tcpip.Error) { + if sameAddr && p == e.TransportEndpointInfo.ID.RemotePort { + return false, nil + } + portRes := ports.Reservation{ + Networks: netProtos, + Transport: ProtocolNumber, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: p, + Flags: e.portFlags, + BindToDevice: bindToDevice, + Dest: addr, + } + if _, err := e.stack.ReservePort(e.stack.SecureRNG(), portRes, nil /* testPort */); err != nil { + if _, ok := err.(*tcpip.ErrPortInUse); !ok || !reuse { + return false, nil + } + transEPID := e.TransportEndpointInfo.ID + transEPID.LocalPort = p + // Check if an endpoint is registered with demuxer in TIME-WAIT and if + // we can reuse it. If we can't find a transport endpoint then we just + // skip using this port as it's possible that either an endpoint has + // bound the port but not registered with demuxer yet (no listen/connect + // done yet) or the reservation was freed between the check above and + // the FindTransportEndpoint below. But rather than retry the same port + // we just skip it and move on. + transEP := e.stack.FindTransportEndpoint(netProto, ProtocolNumber, transEPID, nicID) + if transEP == nil { + // ReservePort failed but there is no registered endpoint with + // demuxer. Which indicates there is at least some endpoint that has + // bound the port. + return false, nil + } + + tcpEP := transEP.(*Endpoint) + tcpEP.LockUser() + // If the endpoint is not in TIME-WAIT or if it is in TIME-WAIT but + // less than 1 second has elapsed since its recentTS was updated then + // we cannot reuse the port. + if tcpEP.EndpointState() != StateTimeWait || e.stack.Clock().NowMonotonic().Sub(tcpEP.recentTSTime) < 1*time.Second { + tcpEP.UnlockUser() + return false, nil + } + // Since the endpoint is in TIME-WAIT it should be safe to acquire its + // Lock while holding the lock for this endpoint as endpoints in + // TIME-WAIT do not acquire locks on other endpoints. + tcpEP.transitionToStateCloseLocked() + tcpEP.drainClosingSegmentQueue() + tcpEP.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + tcpEP.UnlockUser() + // Now try and Reserve again if it fails then we skip. + portRes := ports.Reservation{ + Networks: netProtos, + Transport: ProtocolNumber, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: p, + Flags: e.portFlags, + BindToDevice: bindToDevice, + Dest: addr, + } + if _, err := e.stack.ReservePort(e.stack.SecureRNG(), portRes, nil /* testPort */); err != nil { + return false, nil + } + } + + id := e.TransportEndpointInfo.ID + id.LocalPort = p + if err := e.stack.RegisterTransportEndpoint(netProtos, ProtocolNumber, id, e, e.portFlags, bindToDevice); err != nil { + portRes := ports.Reservation{ + Networks: netProtos, + Transport: ProtocolNumber, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: p, + Flags: e.portFlags, + BindToDevice: bindToDevice, + Dest: addr, + } + e.stack.ReleasePort(portRes) + if _, ok := err.(*tcpip.ErrPortInUse); ok { + return false, nil + } + return false, err + } + + // Port picking successful. Save the details of + // the selected port. + e.TransportEndpointInfo.ID = id + e.isPortReserved = true + e.boundBindToDevice = bindToDevice + e.boundPortFlags = e.portFlags + e.boundDest = addr + return true, nil + }); err != nil { + e.stack.Stats().TCP.FailedPortReservations.Increment() + return err + } + } + return nil +} + +// connect connects the endpoint to its peer. +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) connect(addr tcpip.FullAddress, handshake bool) tcpip.Error { + connectingAddr := addr.Addr + + addr, netProto, err := e.checkV4MappedLocked(addr, false /* bind */) + if err != nil { + return err + } + + if e.EndpointState().connected() { + // The endpoint is already connected. If caller hasn't been + // notified yet, return success. + if !e.isConnectNotified { + e.isConnectNotified = true + return nil + } + // Otherwise return that it's already connected. + return &tcpip.ErrAlreadyConnected{} + } + + nicID := addr.NIC + switch e.EndpointState() { + case StateBound: + // If we're already bound to a NIC but the caller is requesting + // that we use a different one now, we cannot proceed. + if e.boundNICID == 0 { + break + } + + if nicID != 0 && nicID != e.boundNICID { + return &tcpip.ErrHostUnreachable{} + } + + nicID = e.boundNICID + + case StateInitial: + // Nothing to do. We'll eventually fill-in the gaps in the ID (if any) + // when we find a route. + + case StateConnecting, StateSynSent, StateSynRecv: + // A connection request has already been issued but hasn't completed + // yet. + return &tcpip.ErrAlreadyConnecting{} + + case StateError: + if err := e.hardErrorLocked(); err != nil { + return err + } + return &tcpip.ErrConnectionAborted{} + + default: + return &tcpip.ErrInvalidEndpointState{} + } + + // Find a route to the desired destination. + r, err := e.stack.FindRoute(nicID, e.TransportEndpointInfo.ID.LocalAddress, addr.Addr, netProto, false /* multicastLoop */) + if err != nil { + return err + } + defer r.Release() + + e.TransportEndpointInfo.ID.LocalAddress = r.LocalAddress() + e.TransportEndpointInfo.ID.RemoteAddress = r.RemoteAddress() + e.TransportEndpointInfo.ID.RemotePort = addr.Port + + oldState := e.EndpointState() + e.setEndpointState(StateConnecting) + if err := e.registerEndpoint(addr, netProto, r.NICID()); err != nil { + e.setEndpointState(oldState) + if _, ok := err.(*tcpip.ErrPortInUse); ok { + return &tcpip.ErrBadLocalAddress{} + } + return err + } + + e.isRegistered = true + r.Acquire() + e.route = r + e.boundNICID = nicID + e.effectiveNetProtos = []tcpip.NetworkProtocolNumber{netProto} + e.connectingAddress = connectingAddr + + if e.alsoBindToV4 { + // If the endpoint was bound to `any` address the port will be + // reserved for both IPv4 and IPv6 addresses. Release the port + // reservation for the IPv4 address here so that the future bind + // for IPv4 socket will not fail. + portRes := ports.Reservation{ + Networks: []tcpip.NetworkProtocolNumber{header.IPv4ProtocolNumber}, + Transport: ProtocolNumber, + Port: e.TransportEndpointInfo.ID.LocalPort, + } + e.stack.ReleasePort(portRes) + } + + e.initGSO() + + // Connect in the restore phase does not perform handshake. Restore its + // connection setting here. + if !handshake { + e.segmentQueue.mu.Lock() + for _, l := range []segmentList{e.segmentQueue.list, e.snd.writeList.writeList} { + for s := l.Front(); s != nil; s = s.Next() { + s.id = e.TransportEndpointInfo.ID + } + } + e.segmentQueue.mu.Unlock() + e.snd.updateMaxPayloadSize(int(e.route.MTU()), 0) + e.setEndpointState(StateEstablished) + // Set the new auto tuned send buffer size after entering + // established state. + e.ops.SetSendBufferSize(e.computeTCPSendBufferSize(), false /* notify */) + return &tcpip.ErrConnectStarted{} + } + + // Start a new handshake. + h := e.newHandshake() + e.setEndpointState(StateSynSent) + h.start() + e.stack.Stats().TCP.ActiveConnectionOpenings.Increment() + + return &tcpip.ErrConnectStarted{} +} + +// ConnectEndpoint is not supported. +func (*Endpoint) ConnectEndpoint(tcpip.Endpoint) tcpip.Error { + return &tcpip.ErrInvalidEndpointState{} +} + +// Shutdown closes the read and/or write end of the endpoint connection to its +// peer. +func (e *Endpoint) Shutdown(flags tcpip.ShutdownFlags) tcpip.Error { + e.LockUser() + defer e.UnlockUser() + + if e.EndpointState().connecting() { + // When calling shutdown(2) on a connecting socket, the endpoint must + // enter the error state. But this logic cannot belong to the shutdownLocked + // method because that method is called during a close(2) (and closing a + // connecting socket is not an error). + e.handshakeFailed(&tcpip.ErrConnectionReset{}) + e.waiterQueue.Notify(waiter.WritableEvents | waiter.EventHUp | waiter.EventErr) + return nil + } + + return e.shutdownLocked(flags) +} + +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) shutdownLocked(flags tcpip.ShutdownFlags) tcpip.Error { + e.shutdownFlags |= flags + switch { + case e.EndpointState().connected(): + // Close for read. + if e.shutdownFlags&tcpip.ShutdownRead != 0 { + // Mark read side as closed. + e.rcvQueueMu.Lock() + e.RcvClosed = true + rcvBufUsed := e.RcvBufUsed + e.rcvQueueMu.Unlock() + // If we're fully closed and we have unread data we need to abort + // the connection with a RST. + if e.shutdownFlags&tcpip.ShutdownWrite != 0 && rcvBufUsed > 0 { + e.resetConnectionLocked(&tcpip.ErrConnectionAborted{}) + return nil + } + // Wake up any readers that maybe waiting for the stream to become + // readable. + events := waiter.ReadableEvents + if e.shutdownFlags&tcpip.ShutdownWrite == 0 { + // If ShutdownWrite is not set, write end won't close and + // we end up with a half-closed connection + events |= waiter.EventRdHUp + } + e.waiterQueue.Notify(events) + } + + // Close for write. + if e.shutdownFlags&tcpip.ShutdownWrite != 0 { + e.sndQueueInfo.sndQueueMu.Lock() + if e.sndQueueInfo.SndClosed { + // Already closed. + e.sndQueueInfo.sndQueueMu.Unlock() + if e.EndpointState() == StateTimeWait { + return &tcpip.ErrNotConnected{} + } + return nil + } + + // Queue fin segment. + s := newOutgoingSegment(e.TransportEndpointInfo.ID, e.stack.Clock(), buffer.Buffer{}) + e.snd.writeList.PushBack(s) + // Mark endpoint as closed. + e.sndQueueInfo.SndClosed = true + e.sndQueueInfo.sndQueueMu.Unlock() + + // Drain the send queue. + e.sendData(s) + + // Mark send side as closed. + e.snd.Closed = true + + // Wake up any writers that maybe waiting for the stream to become + // writable. + e.waiterQueue.Notify(waiter.WritableEvents) + } + + return nil + case e.EndpointState() == StateListen: + if e.shutdownFlags&tcpip.ShutdownRead != 0 { + // Reset all connections from the accept queue and keep the + // worker running so that it can continue handling incoming + // segments by replying with RST. + // + // By not removing this endpoint from the demuxer mapping, we + // ensure that any other bind to the same port fails, as on Linux. + e.rcvQueueMu.Lock() + e.RcvClosed = true + e.rcvQueueMu.Unlock() + e.closePendingAcceptableConnectionsLocked() + // Notify waiters that the endpoint is shutdown. + e.waiterQueue.Notify(waiter.ReadableEvents | waiter.WritableEvents | waiter.EventHUp | waiter.EventErr) + } + return nil + default: + return &tcpip.ErrNotConnected{} + } +} + +// Listen puts the endpoint in "listen" mode, which allows it to accept +// new connections. +func (e *Endpoint) Listen(backlog int) tcpip.Error { + if err := e.listen(backlog); err != nil { + if !err.IgnoreStats() { + e.stack.Stats().TCP.FailedConnectionAttempts.Increment() + e.stats.FailedConnectionAttempts.Increment() + } + return err + } + return nil +} + +func (e *Endpoint) listen(backlog int) tcpip.Error { + e.LockUser() + defer e.UnlockUser() + + if e.EndpointState() == StateListen && !e.closed { + e.acceptMu.Lock() + defer e.acceptMu.Unlock() + + // Adjust the size of the backlog iff we can fit + // existing pending connections into the new one. + if e.acceptQueue.endpoints.Len() > backlog { + return &tcpip.ErrInvalidEndpointState{} + } + e.acceptQueue.capacity = backlog + + if e.acceptQueue.pendingEndpoints == nil { + e.acceptQueue.pendingEndpoints = make(map[*Endpoint]struct{}) + } + + e.shutdownFlags = 0 + e.updateConnDirectionState(connDirectionStateOpen) + e.rcvQueueMu.Lock() + e.RcvClosed = false + e.rcvQueueMu.Unlock() + + return nil + } + + if e.EndpointState() == StateInitial { + // The listen is called on an unbound socket, the socket is + // automatically bound to a random free port with the local + // address set to INADDR_ANY. + if err := e.bindLocked(tcpip.FullAddress{}); err != nil { + return err + } + } + + // Endpoint must be bound before it can transition to listen mode. + if e.EndpointState() != StateBound { + e.stats.ReadErrors.InvalidEndpointState.Increment() + return &tcpip.ErrInvalidEndpointState{} + } + + // Setting this state after RegisterTransportEndpoint will result in a + // race where the endpoint is in Bound but reachable via the demuxer. Instead + // we set it to listen so that incoming packets will just be queued to the + // inbound segment queue by the TCP processor. + e.setEndpointState(StateListen) + // Register the endpoint. + if err := e.stack.RegisterTransportEndpoint(e.effectiveNetProtos, ProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice); err != nil { + e.transitionToStateCloseLocked() + return err + } + + e.isRegistered = true + + // The queue may be non-zero when we're restoring the endpoint, and it + // may be pre-populated with some previously accepted (but not Accepted) + // endpoints. + e.acceptMu.Lock() + if e.acceptQueue.pendingEndpoints == nil { + e.acceptQueue.pendingEndpoints = make(map[*Endpoint]struct{}) + } + if e.acceptQueue.capacity == 0 { + e.acceptQueue.capacity = backlog + } + e.acceptMu.Unlock() + + // Initialize the listening context. + rcvWnd := seqnum.Size(e.receiveBufferAvailable()) + e.listenCtx = newListenContext(e.stack, e.protocol, e, rcvWnd, e.ops.GetV6Only(), e.NetProto) + + return nil +} + +// Accept returns a new endpoint if a peer has established a connection +// to an endpoint previously set to listen mode. +// +// addr if not-nil will contain the peer address of the returned endpoint. +func (e *Endpoint) Accept(peerAddr *tcpip.FullAddress) (tcpip.Endpoint, *waiter.Queue, tcpip.Error) { + e.LockUser() + defer e.UnlockUser() + + e.rcvQueueMu.Lock() + rcvClosed := e.RcvClosed + e.rcvQueueMu.Unlock() + // Endpoint must be in listen state before it can accept connections. + if rcvClosed || e.EndpointState() != StateListen { + return nil, nil, &tcpip.ErrInvalidEndpointState{} + } + + // Get the new accepted endpoint. + var n *Endpoint + e.acceptMu.Lock() + if element := e.acceptQueue.endpoints.Front(); element != nil { + n = e.acceptQueue.endpoints.Remove(element).(*Endpoint) + } + e.acceptMu.Unlock() + if n == nil { + return nil, nil, &tcpip.ErrWouldBlock{} + } + if peerAddr != nil { + *peerAddr = n.getRemoteAddress() + } + return n, n.waiterQueue, nil +} + +// Bind binds the endpoint to a specific local port and optionally address. +func (e *Endpoint) Bind(addr tcpip.FullAddress) (err tcpip.Error) { + e.LockUser() + defer e.UnlockUser() + + return e.bindLocked(addr) +} + +// +checklocks:e.mu +func (e *Endpoint) bindLocked(addr tcpip.FullAddress) (err tcpip.Error) { + // Don't allow binding once endpoint is not in the initial state + // anymore. This is because once the endpoint goes into a connected or + // listen state, it is already bound. + if e.EndpointState() != StateInitial { + return &tcpip.ErrAlreadyBound{} + } + + e.BindAddr = addr.Addr + addr, netProto, err := e.checkV4MappedLocked(addr, true /* bind */) + if err != nil { + return err + } + + netProtos := []tcpip.NetworkProtocolNumber{netProto} + + // Expand netProtos to include v4 and v6 under dual-stack if the caller is + // binding to a wildcard (empty) address, and this is an IPv6 endpoint with + // v6only set to false. + if netProto == header.IPv6ProtocolNumber { + stackHasV4 := e.stack.CheckNetworkProtocol(header.IPv4ProtocolNumber) + alsoBindToV4 := !e.ops.GetV6Only() && addr.Addr == tcpip.Address{} && stackHasV4 + if alsoBindToV4 { + netProtos = append(netProtos, header.IPv4ProtocolNumber) + e.alsoBindToV4 = true + } + } + + var nic tcpip.NICID + // If an address is specified, we must ensure that it's one of our + // local addresses. + if addr.Addr.Len() != 0 { + nic = e.stack.CheckLocalAddress(addr.NIC, netProto, addr.Addr) + if nic == 0 { + return &tcpip.ErrBadLocalAddress{} + } + e.TransportEndpointInfo.ID.LocalAddress = addr.Addr + } + + bindToDevice := tcpip.NICID(e.ops.GetBindToDevice()) + portRes := ports.Reservation{ + Networks: netProtos, + Transport: ProtocolNumber, + Addr: addr.Addr, + Port: addr.Port, + Flags: e.portFlags, + BindToDevice: bindToDevice, + Dest: tcpip.FullAddress{}, + } + port, err := e.stack.ReservePort(e.stack.SecureRNG(), portRes, func(p uint16) (bool, tcpip.Error) { + id := e.TransportEndpointInfo.ID + id.LocalPort = p + // CheckRegisterTransportEndpoint should only return an error if there is a + // listening endpoint bound with the same id and portFlags and bindToDevice + // options. + // + // NOTE: Only listening and connected endpoint register with + // demuxer. Further connected endpoints always have a remote + // address/port. Hence this will only return an error if there is a matching + // listening endpoint. + if err := e.stack.CheckRegisterTransportEndpoint(netProtos, ProtocolNumber, id, e.portFlags, bindToDevice); err != nil { + return false, nil + } + return true, nil + }) + if err != nil { + e.stack.Stats().TCP.FailedPortReservations.Increment() + return err + } + + e.boundBindToDevice = bindToDevice + e.boundPortFlags = e.portFlags + // TODO(gvisor.dev/issue/3691): Add test to verify boundNICID is correct. + e.boundNICID = nic + e.isPortReserved = true + e.effectiveNetProtos = netProtos + e.TransportEndpointInfo.ID.LocalPort = port + + // Mark endpoint as bound. + e.setEndpointState(StateBound) + + return nil +} + +// GetLocalAddress returns the address to which the endpoint is bound. +func (e *Endpoint) GetLocalAddress() (tcpip.FullAddress, tcpip.Error) { + e.LockUser() + defer e.UnlockUser() + + return tcpip.FullAddress{ + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: e.TransportEndpointInfo.ID.LocalPort, + NIC: e.boundNICID, + }, nil +} + +// GetRemoteAddress returns the address to which the endpoint is connected. +func (e *Endpoint) GetRemoteAddress() (tcpip.FullAddress, tcpip.Error) { + e.LockUser() + defer e.UnlockUser() + + if !e.EndpointState().connected() { + return tcpip.FullAddress{}, &tcpip.ErrNotConnected{} + } + + return e.getRemoteAddress(), nil +} + +func (e *Endpoint) getRemoteAddress() tcpip.FullAddress { + return tcpip.FullAddress{ + Addr: e.TransportEndpointInfo.ID.RemoteAddress, + Port: e.TransportEndpointInfo.ID.RemotePort, + NIC: e.boundNICID, + } +} + +// HandlePacket implements stack.TransportEndpoint.HandlePacket. +func (*Endpoint) HandlePacket(stack.TransportEndpointID, *stack.PacketBuffer) { + // TCP HandlePacket is not required anymore as inbound packets first + // land at the Dispatcher which then can either deliver using the + // worker go routine or directly do the invoke the tcp processing inline + // based on the state of the endpoint. +} + +func (e *Endpoint) enqueueSegment(s *segment) bool { + // Send packet to worker goroutine. + if !e.segmentQueue.enqueue(s) { + // The queue is full, so we drop the segment. + e.stack.Stats().DroppedPackets.Increment() + e.stats.ReceiveErrors.SegmentQueueDropped.Increment() + return false + } + return true +} + +func (e *Endpoint) onICMPError(err tcpip.Error, transErr stack.TransportError, pkt *stack.PacketBuffer) { + // Update last error first. + e.lastErrorMu.Lock() + e.lastError = err + e.lastErrorMu.Unlock() + + var recvErr bool + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + recvErr = e.SocketOptions().GetIPv4RecvError() + case header.IPv6ProtocolNumber: + recvErr = e.SocketOptions().GetIPv6RecvError() + default: + panic(fmt.Sprintf("unhandled network protocol number = %d", pkt.NetworkProtocolNumber)) + } + + if recvErr { + e.SocketOptions().QueueErr(&tcpip.SockError{ + Err: err, + Cause: transErr, + // Linux passes the payload with the TCP header. We don't know if the TCP + // header even exists, it may not for fragmented packets. + Payload: pkt.Data().AsRange().ToView(), + Dst: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: e.TransportEndpointInfo.ID.RemoteAddress, + Port: e.TransportEndpointInfo.ID.RemotePort, + }, + Offender: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: e.TransportEndpointInfo.ID.LocalAddress, + Port: e.TransportEndpointInfo.ID.LocalPort, + }, + NetProto: pkt.NetworkProtocolNumber, + }) + } + + if e.EndpointState().connecting() { + e.mu.Lock() + if lEP := e.h.listenEP; lEP != nil { + // Remove from listening endpoints pending list. + lEP.acceptMu.Lock() + delete(lEP.acceptQueue.pendingEndpoints, e) + lEP.acceptMu.Unlock() + lEP.stats.FailedConnectionAttempts.Increment() + } + e.stack.Stats().TCP.FailedConnectionAttempts.Increment() + e.cleanupLocked() + e.hardError = err + e.setEndpointState(StateError) + e.mu.Unlock() + e.drainClosingSegmentQueue() + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) + } +} + +// HandleError implements stack.TransportEndpoint. +func (e *Endpoint) HandleError(transErr stack.TransportError, pkt *stack.PacketBuffer) { + handlePacketTooBig := func(mtu uint32) { + e.sndQueueInfo.sndQueueMu.Lock() + update := false + if v := int(mtu); v < e.sndQueueInfo.SndMTU { + e.sndQueueInfo.SndMTU = v + update = true + } + newMTU := e.sndQueueInfo.SndMTU + e.sndQueueInfo.sndQueueMu.Unlock() + if update { + e.mu.Lock() + defer e.mu.Unlock() + if e.snd != nil { + e.snd.updateMaxPayloadSize(newMTU, 1 /* count */) // +checklocksforce:e.snd.ep.mu + } + } + } + + // TODO(gvisor.dev/issues/5270): Handle all transport errors. + switch transErr.Kind() { + case stack.PacketTooBigTransportError: + handlePacketTooBig(transErr.Info()) + case stack.DestinationHostUnreachableTransportError: + e.onICMPError(&tcpip.ErrHostUnreachable{}, transErr, pkt) + case stack.DestinationNetworkUnreachableTransportError: + e.onICMPError(&tcpip.ErrNetworkUnreachable{}, transErr, pkt) + case stack.DestinationPortUnreachableTransportError: + e.onICMPError(&tcpip.ErrConnectionRefused{}, transErr, pkt) + case stack.DestinationProtoUnreachableTransportError: + e.onICMPError(&tcpip.ErrUnknownProtocolOption{}, transErr, pkt) + case stack.SourceRouteFailedTransportError: + e.onICMPError(&tcpip.ErrNotSupported{}, transErr, pkt) + case stack.SourceHostIsolatedTransportError: + e.onICMPError(&tcpip.ErrNoNet{}, transErr, pkt) + case stack.DestinationHostDownTransportError: + e.onICMPError(&tcpip.ErrHostDown{}, transErr, pkt) + } +} + +// updateSndBufferUsage is called by when room opens up in the send buffer. The +// number of newly available bytes is v. +// +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) updateSndBufferUsage(v int) { + sendBufferSize := e.getSendBufferSize() + e.sndQueueInfo.sndQueueMu.Lock() + notify := e.sndQueueInfo.SndBufUsed >= sendBufferSize>>1 + e.sndQueueInfo.SndBufUsed -= v + + // Get the new send buffer size with auto tuning, but do not set it + // unless we decide to notify the writers. + newSndBufSz := e.computeTCPSendBufferSize() + + // We only notify when there is half the sendBufferSize available after + // a full buffer event occurs. This ensures that we don't wake up + // writers to queue just 1-2 segments and go back to sleep. + notify = notify && e.sndQueueInfo.SndBufUsed < int(newSndBufSz)>>1 + e.sndQueueInfo.sndQueueMu.Unlock() + + if notify { + // Set the new send buffer size calculated from auto tuning. + e.ops.SetSendBufferSize(newSndBufSz, false /* notify */) + e.waiterQueue.Notify(waiter.WritableEvents) + } +} + +// readyToRead is called when a new segment is ready to be read, or when the +// connection is closed for receiving (in which case s will be nil). +// +// +checklocks:e.mu +func (e *Endpoint) readyToRead(s *segment) { + e.rcvQueueMu.Lock() + if s != nil { + e.RcvBufUsed += s.payloadSize() + s.IncRef() + e.rcvQueue.PushBack(s) + } else { + e.RcvClosed = true + } + e.rcvQueueMu.Unlock() + e.waiterQueue.Notify(waiter.ReadableEvents) +} + +// receiveBufferAvailableLocked calculates how many bytes are still available +// in the receive buffer. +// +checklocks:e.rcvQueueMu +func (e *Endpoint) receiveBufferAvailableLocked(rcvBufSize int) int { + // We may use more bytes than the buffer size when the receive buffer + // shrinks. + memUsed := e.receiveMemUsed() + if memUsed >= rcvBufSize { + return 0 + } + + return rcvBufSize - memUsed +} + +// receiveBufferAvailable calculates how many bytes are still available in the +// receive buffer based on the actual memory used by all segments held in +// receive buffer/pending and segment queue. +func (e *Endpoint) receiveBufferAvailable() int { + e.rcvQueueMu.Lock() + available := e.receiveBufferAvailableLocked(int(e.ops.GetReceiveBufferSize())) + e.rcvQueueMu.Unlock() + return available +} + +// receiveBufferUsed returns the amount of in-use receive buffer. +func (e *Endpoint) receiveBufferUsed() int { + e.rcvQueueMu.Lock() + used := e.RcvBufUsed + e.rcvQueueMu.Unlock() + return used +} + +// receiveMemUsed returns the total memory in use by segments held by this +// endpoint. +func (e *Endpoint) receiveMemUsed() int { + return int(e.rcvMemUsed.Load()) +} + +// updateReceiveMemUsed adds the provided delta to e.rcvMemUsed. +func (e *Endpoint) updateReceiveMemUsed(delta int) { + e.rcvMemUsed.Add(int32(delta)) +} + +// maxReceiveBufferSize returns the stack wide maximum receive buffer size for +// an endpoint. +func (e *Endpoint) maxReceiveBufferSize() int { + var rs tcpip.TCPReceiveBufferSizeRangeOption + if err := e.stack.TransportProtocolOption(ProtocolNumber, &rs); err != nil { + // As a fallback return the hardcoded max buffer size. + return MaxBufferSize + } + return rs.Max +} + +// directionState returns the close state of send and receive part of the endpoint +func (e *Endpoint) connDirectionState() connDirectionState { + return connDirectionState(e.connectionDirectionState.Load()) +} + +// updateDirectionState updates the close state of send and receive part of the endpoint +func (e *Endpoint) updateConnDirectionState(state connDirectionState) connDirectionState { + return connDirectionState(e.connectionDirectionState.Swap(uint32(e.connDirectionState() | state))) +} + +// rcvWndScaleForHandshake computes the receive window scale to offer to the +// peer when window scaling is enabled (true by default). If auto-tuning is +// disabled then the window scaling factor is based on the size of the +// receiveBuffer otherwise we use the max permissible receive buffer size to +// compute the scale. +func (e *Endpoint) rcvWndScaleForHandshake() int { + bufSizeForScale := e.ops.GetReceiveBufferSize() + + e.rcvQueueMu.Lock() + autoTuningDisabled := e.RcvAutoParams.Disabled + e.rcvQueueMu.Unlock() + if autoTuningDisabled { + return FindWndScale(seqnum.Size(bufSizeForScale)) + } + + return FindWndScale(seqnum.Size(e.maxReceiveBufferSize())) +} + +// updateRecentTimestamp updates the recent timestamp using the algorithm +// described in https://tools.ietf.org/html/rfc7323#section-4.3 +func (e *Endpoint) updateRecentTimestamp(tsVal uint32, maxSentAck seqnum.Value, segSeq seqnum.Value) { + if e.SendTSOk && seqnum.Value(e.recentTimestamp()).LessThan(seqnum.Value(tsVal)) && segSeq.LessThanEq(maxSentAck) { + e.setRecentTimestamp(tsVal) + } +} + +// maybeEnableTimestamp marks the timestamp option enabled for this endpoint if +// the SYN options indicate that timestamp option was negotiated. It also +// initializes the recentTS with the value provided in synOpts.TSval. +func (e *Endpoint) maybeEnableTimestamp(synOpts header.TCPSynOptions) { + if synOpts.TS { + e.SendTSOk = true + e.setRecentTimestamp(synOpts.TSVal) + } +} + +func (e *Endpoint) tsVal(now tcpip.MonotonicTime) uint32 { + return e.TSOffset.TSVal(now) +} + +func (e *Endpoint) tsValNow() uint32 { + return e.tsVal(e.stack.Clock().NowMonotonic()) +} + +func (e *Endpoint) elapsed(now tcpip.MonotonicTime, tsEcr uint32) time.Duration { + return e.TSOffset.Elapsed(now, tsEcr) +} + +// maybeEnableSACKPermitted marks the SACKPermitted option enabled for this endpoint +// if the SYN options indicate that the SACK option was negotiated and the TCP +// stack is configured to enable TCP SACK option. +func (e *Endpoint) maybeEnableSACKPermitted(synOpts header.TCPSynOptions) { + var v tcpip.TCPSACKEnabled + if err := e.stack.TransportProtocolOption(ProtocolNumber, &v); err != nil { + // Stack doesn't support SACK. So just return. + return + } + if bool(v) && synOpts.SACKPermitted { + e.SACKPermitted = true + e.stack.TransportProtocolOption(ProtocolNumber, &e.tcpRecovery) + } +} + +// maxOptionSize return the maximum size of TCP options. +func (e *Endpoint) maxOptionSize() (size int) { + var maxSackBlocks [header.TCPMaxSACKBlocks]header.SACKBlock + options := e.makeOptions(maxSackBlocks[:]) + size = len(options) + putOptions(options) + + return size +} + +// completeStateLocked makes a full copy of the endpoint and returns it. This is +// used before invoking the probe. +// +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) completeStateLocked(s *TCPEndpointState) { + s.TCPEndpointStateInner = e.TCPEndpointStateInner + s.ID = TCPEndpointID(e.TransportEndpointInfo.ID) + s.SegTime = e.stack.Clock().NowMonotonic() + s.Receiver = e.rcv.TCPReceiverState + s.Sender = e.snd.TCPSenderState + + sndBufSize := e.getSendBufferSize() + // Copy the send buffer atomically. + e.sndQueueInfo.sndQueueMu.Lock() + e.sndQueueInfo.CloneState(&s.SndBufState) + s.SndBufState.SndBufSize = sndBufSize + e.sndQueueInfo.sndQueueMu.Unlock() + + // Copy the receive buffer atomically. + e.rcvQueueMu.Lock() + s.RcvBufState = e.TCPRcvBufState + e.rcvQueueMu.Unlock() + + // Copy the endpoint TCP Option state. + s.SACK.Blocks = make([]header.SACKBlock, e.sack.NumBlocks) + copy(s.SACK.Blocks, e.sack.Blocks[:e.sack.NumBlocks]) + s.SACK.ReceivedBlocks, s.SACK.MaxSACKED = e.scoreboard.Copy() + + e.snd.rtt.Lock() + s.Sender.RTTState = e.snd.rtt.TCPRTTState + e.snd.rtt.Unlock() + + if cubic, ok := e.snd.cc.(*cubicState); ok { + s.Sender.Cubic = cubic.TCPCubicState + s.Sender.Cubic.TimeSinceLastCongestion = e.stack.Clock().NowMonotonic().Sub(s.Sender.Cubic.T) + } + + s.Sender.RACKState = e.snd.rc.TCPRACKState + s.Sender.RetransmitTS = e.snd.retransmitTS + s.Sender.SpuriousRecovery = e.snd.spuriousRecovery +} + +func (e *Endpoint) initHostGSO() { + switch e.route.NetProto() { + case header.IPv4ProtocolNumber: + e.gso.Type = stack.GSOTCPv4 + e.gso.L3HdrLen = header.IPv4MinimumSize + case header.IPv6ProtocolNumber: + e.gso.Type = stack.GSOTCPv6 + e.gso.L3HdrLen = header.IPv6MinimumSize + default: + panic(fmt.Sprintf("Unknown netProto: %v", e.NetProto)) + } + e.gso.NeedsCsum = true + e.gso.CsumOffset = header.TCPChecksumOffset + e.gso.MaxSize = e.route.GSOMaxSize() +} + +func (e *Endpoint) initGSO() { + if e.route.HasHostGSOCapability() { + e.initHostGSO() + } else if e.route.HasGVisorGSOCapability() { + e.gso = stack.GSO{ + MaxSize: e.route.GSOMaxSize(), + Type: stack.GSOGvisor, + NeedsCsum: false, + } + } +} + +// State implements tcpip.Endpoint.State. It exports the endpoint's protocol +// state for diagnostics. +func (e *Endpoint) State() uint32 { + return uint32(e.EndpointState()) +} + +// Info returns a copy of the endpoint info. +func (e *Endpoint) Info() tcpip.EndpointInfo { + e.LockUser() + // Make a copy of the endpoint info. + ret := e.TransportEndpointInfo + e.UnlockUser() + return &ret +} + +// Stats returns a pointer to the endpoint stats. +func (e *Endpoint) Stats() tcpip.EndpointStats { + return &e.stats +} + +// Wait implements stack.TransportEndpoint.Wait. +func (e *Endpoint) Wait() { + waitEntry, notifyCh := waiter.NewChannelEntry(waiter.EventHUp) + e.waiterQueue.EventRegister(&waitEntry) + defer e.waiterQueue.EventUnregister(&waitEntry) + switch e.EndpointState() { + case StateClose, StateError: + return + } + <-notifyCh +} + +// SocketOptions implements tcpip.Endpoint.SocketOptions. +func (e *Endpoint) SocketOptions() *tcpip.SocketOptions { + return &e.ops +} + +// GetTCPSendBufferLimits is used to get send buffer size limits for TCP. +func GetTCPSendBufferLimits(sh tcpip.StackHandler) tcpip.SendBufferSizeOption { + // This type assertion is safe because only the TCP stack calls this + // function. + ss := sh.(*stack.Stack).TCPSendBufferLimits() + return tcpip.SendBufferSizeOption{ + Min: ss.Min, + Default: ss.Default, + Max: ss.Max, + } +} + +// allowOutOfWindowAck returns true if an out-of-window ACK can be sent now. +func (e *Endpoint) allowOutOfWindowAck() bool { + now := e.stack.Clock().NowMonotonic() + + if e.lastOutOfWindowAckTime != (tcpip.MonotonicTime{}) { + var limit stack.TCPInvalidRateLimitOption + if err := e.stack.Option(&limit); err != nil { + panic(fmt.Sprintf("e.stack.Option(%+v) failed with error: %s", limit, err)) + } + if now.Sub(e.lastOutOfWindowAckTime) < time.Duration(limit) { + return false + } + } + + e.lastOutOfWindowAckTime = now + return true +} + +// GetTCPReceiveBufferLimits is used to get send buffer size limits for TCP. +func GetTCPReceiveBufferLimits(s tcpip.StackHandler) tcpip.ReceiveBufferSizeOption { + var ss tcpip.TCPReceiveBufferSizeRangeOption + if err := s.TransportProtocolOption(header.TCPProtocolNumber, &ss); err != nil { + panic(fmt.Sprintf("s.TransportProtocolOption(%d, %#v) = %s", header.TCPProtocolNumber, ss, err)) + } + + return tcpip.ReceiveBufferSizeOption{ + Min: ss.Min, + Default: ss.Default, + Max: ss.Max, + } +} + +// computeTCPSendBufferSize implements auto tuning of send buffer size and +// returns the new send buffer size. +// +// +checklocks:e.mu +// +checklocksalias:e.snd.ep.mu=e.mu +func (e *Endpoint) computeTCPSendBufferSize() int64 { + curSndBufSz := int64(e.getSendBufferSize()) + + // Auto tuning is disabled when the user explicitly sets the send + // buffer size with SO_SNDBUF option. + if disabled := e.sndQueueInfo.TCPSndBufState.AutoTuneSndBufDisabled.Load(); disabled == 1 { + return curSndBufSz + } + + const packetOverheadFactor = 2 + curMSS := e.snd.MaxPayloadSize + numSeg := InitialCwnd + if numSeg < e.snd.SndCwnd { + numSeg = e.snd.SndCwnd + } + + // SndCwnd indicates the number of segments that can be sent. This means + // that the sender can send upto #SndCwnd segments and the send buffer + // size should be set to SndCwnd*MSS to accommodate sending of all the + // segments. + newSndBufSz := int64(numSeg * curMSS * packetOverheadFactor) + if newSndBufSz < curSndBufSz { + return curSndBufSz + } + if ss := GetTCPSendBufferLimits(e.stack); int64(ss.Max) < newSndBufSz { + newSndBufSz = int64(ss.Max) + } + + return newSndBufSz +} + +// GetAcceptConn implements tcpip.SocketOptionsHandler. +func (e *Endpoint) GetAcceptConn() bool { + return EndpointState(e.State()) == StateListen +} + +// getExperimentOptionValue returns the experiment option value set on the +// endpoint if experiment IP options are enabled on outgoing NIC of the route. +func (e *Endpoint) getExperimentOptionValue(route *stack.Route) uint16 { + if nic, err := e.stack.GetNICByID(route.OutgoingNIC()); err == nil && nic.GetExperimentIPOptionEnabled() { + return e.ops.GetExperimentOptionValue() + } + return 0 +} diff --git a/pkg/tcpip/transport/tcp/endpoint_state.go b/pkg/tcpip/transport/tcp/endpoint_state.go new file mode 100644 index 0000000..96ef64d --- /dev/null +++ b/pkg/tcpip/transport/tcp/endpoint_state.go @@ -0,0 +1,347 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "context" + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// logDisconnectOnce ensures we don't spam logs when many connections are terminated. +var logDisconnectOnce sync.Once + +func logDisconnect() { + logDisconnectOnce.Do(func() { + log.Infof("One or more TCP connections terminated during save") + }) +} + +// beforeSave is invoked by stateify. +func (e *Endpoint) beforeSave() { + // Stop incoming packets. + e.segmentQueue.freeze() + + e.mu.Lock() + defer e.mu.Unlock() + + epState := e.EndpointState() + switch { + case epState == StateInitial || epState == StateBound: + case epState.connected() || epState.handshake(): + if !e.route.HasSaveRestoreCapability() { + if !e.route.HasDisconnectOkCapability() { + panic(&tcpip.ErrSaveRejection{ + Err: fmt.Errorf("endpoint cannot be saved in connected state: local %s:%d, remote %s:%d", e.TransportEndpointInfo.ID.LocalAddress, e.TransportEndpointInfo.ID.LocalPort, e.TransportEndpointInfo.ID.RemoteAddress, e.TransportEndpointInfo.ID.RemotePort), + }) + } + logDisconnect() + e.resetConnectionLocked(&tcpip.ErrConnectionAborted{}) + e.mu.Unlock() + e.Close() + e.mu.Lock() + } + fallthrough + case epState == StateListen: + // Nothing to do. + case epState.closed(): + // Nothing to do. + default: + panic(fmt.Sprintf("endpoint in unknown state %v", e.EndpointState())) + } + + e.stack.RegisterResumableEndpoint(e) +} + +// saveEndpoints is invoked by stateify. +func (a *acceptQueue) saveEndpoints() []*Endpoint { + acceptedEndpoints := make([]*Endpoint, a.endpoints.Len()) + for i, e := 0, a.endpoints.Front(); e != nil; i, e = i+1, e.Next() { + acceptedEndpoints[i] = e.Value.(*Endpoint) + } + return acceptedEndpoints +} + +// loadEndpoints is invoked by stateify. +func (a *acceptQueue) loadEndpoints(_ context.Context, acceptedEndpoints []*Endpoint) { + for _, ep := range acceptedEndpoints { + a.endpoints.PushBack(ep) + } +} + +// saveState is invoked by stateify. +func (e *Endpoint) saveState() EndpointState { + return e.EndpointState() +} + +// Endpoint loading must be done in the following ordering by their state, to +// avoid dangling connecting w/o listening peer, and to avoid conflicts in port +// reservation. +var ( + connectedLoading sync.WaitGroup + listenLoading sync.WaitGroup + connectingLoading sync.WaitGroup +) + +// Bound endpoint loading happens last. + +// loadState is invoked by stateify. +func (e *Endpoint) loadState(_ context.Context, epState EndpointState) { + // This is to ensure that the loading wait groups include all applicable + // endpoints before any asynchronous calls to the Wait() methods. + // For restore purposes we treat all endpoints with state after + // StateEstablished and before StateClosed like connected endpoint. + if epState.connected() { + connectedLoading.Add(1) + } + switch { + case epState == StateListen: + listenLoading.Add(1) + case epState.connecting(): + connectingLoading.Add(1) + } + // Directly update the state here rather than using e.setEndpointState + // as the endpoint is still being loaded and the stack reference is not + // yet initialized. + e.state.Store(uint32(epState)) +} + +// afterLoad is invoked by stateify. +func (e *Endpoint) afterLoad(ctx context.Context) { + // RacyLoad() can be used because we are initializing e. + e.origEndpointState = e.state.RacyLoad() + // Restore the endpoint to InitialState as it will be moved to + // its origEndpointState during Restore. + e.state = atomicbitops.FromUint32(uint32(StateInitial)) + if e.stack.IsSaveRestoreEnabled() { + e.stack.RegisterRestoredEndpoint(e) + } else { + stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) + } +} + +// Restore implements tcpip.RestoredEndpoint.Restore. +func (e *Endpoint) Restore(s *stack.Stack) { + if !e.EndpointState().closed() { + e.keepalive.timer.init(s.Clock(), timerHandler(e, e.keepaliveTimerExpired)) + } + if snd := e.snd; snd != nil { + snd.resendTimer.init(s.Clock(), timerHandler(e, e.snd.retransmitTimerExpired)) + snd.reorderTimer.init(s.Clock(), timerHandler(e, e.snd.rc.reorderTimerExpired)) + snd.probeTimer.init(s.Clock(), timerHandler(e, e.snd.probeTimerExpired)) + snd.corkTimer.init(s.Clock(), timerHandler(e, e.snd.corkTimerExpired)) + } + saveRestoreEnabled := e.stack.IsSaveRestoreEnabled() + if !saveRestoreEnabled { + e.stack = s + e.protocol = protocolFromStack(s) + } + e.ops.InitHandler(e, e.stack, GetTCPSendBufferLimits, GetTCPReceiveBufferLimits) + e.segmentQueue.thaw() + + e.mu.Lock() + id := e.ID + e.mu.Unlock() + + bind := func() { + e.mu.Lock() + defer e.mu.Unlock() + if !saveRestoreEnabled { + addr, _, err := e.checkV4MappedLocked(tcpip.FullAddress{Addr: e.BindAddr, Port: e.TransportEndpointInfo.ID.LocalPort}, true /* bind */) + if err != nil { + panic("unable to parse BindAddr: " + err.String()) + } + portRes := ports.Reservation{ + Networks: e.effectiveNetProtos, + Transport: ProtocolNumber, + Addr: addr.Addr, + Port: addr.Port, + Flags: e.boundPortFlags, + BindToDevice: e.boundBindToDevice, + Dest: e.boundDest, + } + if ok := e.stack.ReserveTuple(portRes); !ok { + panic(fmt.Sprintf("unable to re-reserve tuple (%v, %q, %d, %+v, %d, %v)", e.effectiveNetProtos, addr.Addr, addr.Port, e.boundPortFlags, e.boundBindToDevice, e.boundDest)) + } + } + e.isPortReserved = true + + // Mark endpoint as bound. + e.setEndpointState(StateBound) + } + + epState := EndpointState(e.origEndpointState) + switch { + case epState.connected(): + bind() + if e.connectingAddress.BitLen() == 0 { + e.connectingAddress = e.TransportEndpointInfo.ID.RemoteAddress + // This endpoint is accepted by netstack but not yet by + // the app. If the endpoint is IPv6 but the remote + // address is IPv4, we need to connect as IPv6 so that + // dual-stack mode can be properly activated. + if e.NetProto == header.IPv6ProtocolNumber && e.TransportEndpointInfo.ID.RemoteAddress.BitLen() != header.IPv6AddressSizeBits { + e.connectingAddress = tcpip.AddrFrom16Slice(append( + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff}, + e.TransportEndpointInfo.ID.RemoteAddress.AsSlice()..., + )) + } + } + // Reset the scoreboard to reinitialize the sack information as + // we do not restore SACK information. + e.scoreboard.Reset() + if saveRestoreEnabled { + // Unregister the endpoint before registering again during Connect. + e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, header.TCPProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice) + } + e.mu.Lock() + err := e.connect(tcpip.FullAddress{NIC: e.boundNICID, Addr: e.connectingAddress, Port: e.TransportEndpointInfo.ID.RemotePort}, false /* handshake */) + if _, ok := err.(*tcpip.ErrConnectStarted); !ok { + log.Warningf("TCP endpoint connect failed for connected endpoint with ID: %+v err: %v", id, err) + e.mu.Unlock() + e.Close() + connectedLoading.Done() + return + } + e.state.Store(e.origEndpointState) + // For FIN-WAIT-2 and TIME-WAIT we need to start the appropriate timers so + // that the socket is closed correctly. + switch epState { + case StateFinWait2: + e.finWait2Timer = e.stack.Clock().AfterFunc(e.tcpLingerTimeout, e.finWait2TimerExpired) + case StateTimeWait: + e.timeWaitTimer = e.stack.Clock().AfterFunc(e.getTimeWaitDuration(), e.timeWaitTimerExpired) + } + + if e.ops.GetCorkOption() { + // Rearm the timer if TCP_CORK is enabled which will + // drain all the segments in the queue after restore. + e.snd.corkTimer.enable(MinRTO) + } + e.mu.Unlock() + connectedLoading.Done() + case epState == StateListen: + tcpip.AsyncLoading.Add(1) + if !saveRestoreEnabled { + go func() { + connectedLoading.Wait() + bind() + e.acceptMu.Lock() + backlog := e.acceptQueue.capacity + e.acceptMu.Unlock() + if err := e.Listen(backlog); err != nil { + panic("endpoint listening failed: " + err.String()) + } + e.LockUser() + if e.shutdownFlags != 0 { + e.shutdownLocked(e.shutdownFlags) + } + e.UnlockUser() + listenLoading.Done() + tcpip.AsyncLoading.Done() + }() + } else { + go func() { + connectedLoading.Wait() + e.LockUser() + // All endpoints will be moved to initial state after + // restore. Set endpoint to its originial listen state. + e.setEndpointState(StateListen) + // Initialize the listening context. + rcvWnd := seqnum.Size(e.receiveBufferAvailable()) + e.listenCtx = newListenContext(e.stack, e.protocol, e, rcvWnd, e.ops.GetV6Only(), e.NetProto) + e.UnlockUser() + listenLoading.Done() + tcpip.AsyncLoading.Done() + }() + } + case epState == StateConnecting: + // Initial SYN hasn't been sent yet so initiate a connect. + tcpip.AsyncLoading.Add(1) + go func() { + connectedLoading.Wait() + listenLoading.Wait() + bind() + err := e.Connect(tcpip.FullAddress{NIC: e.boundNICID, Addr: e.connectingAddress, Port: e.TransportEndpointInfo.ID.RemotePort}) + if _, ok := err.(*tcpip.ErrConnectStarted); !ok { + log.Warningf("TCP endpoint connect failed for connecting endpoint with ID: %+v err: %v", id, err) + e.Close() + } + connectingLoading.Done() + tcpip.AsyncLoading.Done() + }() + case epState == StateSynSent || epState == StateSynRecv: + tcpip.AsyncLoading.Add(1) + go func() { + connectedLoading.Wait() + listenLoading.Wait() + // Initial SYN has been sent/received so we should bind the + // ports start the retransmit timer for the SYNs and let it + // naturally complete the connection. + bind() + e.mu.Lock() + e.setEndpointState(epState) + r, err := e.stack.FindRoute(e.boundNICID, e.TransportEndpointInfo.ID.LocalAddress, e.TransportEndpointInfo.ID.RemoteAddress, e.effectiveNetProtos[0], false /* multicastLoop */) + if err != nil { + e.mu.Unlock() + log.Warningf("FindRoute failed when restoring endpoint w/ ID: %+v err: %v", id, err) + e.Close() + connectingLoading.Done() + tcpip.AsyncLoading.Done() + return + } + e.route = r + timer, err := newBackoffTimer(e.stack.Clock(), InitialRTO, MaxRTO, timerHandler(e, e.h.retransmitHandlerLocked)) + if err != nil { + panic(fmt.Sprintf("newBackOffTimer(_, %s, %s, _) failed: %s", InitialRTO, MaxRTO, err)) + } + e.h.retransmitTimer = timer + connectingLoading.Done() + tcpip.AsyncLoading.Done() + e.mu.Unlock() + }() + case epState == StateBound: + tcpip.AsyncLoading.Add(1) + go func() { + connectedLoading.Wait() + listenLoading.Wait() + connectingLoading.Wait() + bind() + tcpip.AsyncLoading.Done() + }() + case epState == StateClose: + e.isPortReserved = false + e.state.Store(uint32(StateClose)) + e.stack.CompleteTransportEndpointCleanup(e) + tcpip.DeleteDanglingEndpoint(e) + case epState == StateError: + e.state.Store(uint32(StateError)) + e.stack.CompleteTransportEndpointCleanup(e) + tcpip.DeleteDanglingEndpoint(e) + } +} + +// Resume implements tcpip.ResumableEndpoint.Resume. +func (e *Endpoint) Resume() { + e.segmentQueue.thaw() +} diff --git a/pkg/tcpip/transport/tcp/ep_queue_mutex.go b/pkg/tcpip/transport/tcp/ep_queue_mutex.go new file mode 100644 index 0000000..41b4513 --- /dev/null +++ b/pkg/tcpip/transport/tcp/ep_queue_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type epQueueMutex struct { + mu sync.Mutex +} + +var epQueueprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var epQueuelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type epQueuelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *epQueueMutex) Lock() { + locking.AddGLock(epQueueprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *epQueueMutex) NestedLock(i epQueuelockNameIndex) { + locking.AddGLock(epQueueprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *epQueueMutex) Unlock() { + locking.DelGLock(epQueueprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *epQueueMutex) NestedUnlock(i epQueuelockNameIndex) { + locking.DelGLock(epQueueprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func epQueueinitLockNames() {} + +func init() { + epQueueinitLockNames() + epQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(epQueueMutex{}), epQueuelockNames) +} diff --git a/pkg/tcpip/transport/tcp/forwarder.go b/pkg/tcpip/transport/tcp/forwarder.go new file mode 100644 index 0000000..7710478 --- /dev/null +++ b/pkg/tcpip/transport/tcp/forwarder.go @@ -0,0 +1,231 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// Forwarder is a connection request forwarder, which allows clients to decide +// what to do with a connection request, for example: ignore it, send a RST, or +// attempt to complete the 3-way handshake. +// +// The canonical way of using it is to pass the Forwarder.HandlePacket function +// to stack.SetTransportProtocolHandler. +type Forwarder struct { + stack *stack.Stack + + maxInFlight int + handler func(*ForwarderRequest) + + mu forwarderMutex + inFlight map[stack.TransportEndpointID]struct{} + listen *listenContext +} + +// NewForwarder allocates and initializes a new forwarder with the given +// maximum number of in-flight connection attempts. Once the maximum is reached +// new incoming connection requests will be ignored. +// +// If rcvWnd is set to zero, the default buffer size is used instead. +func NewForwarder(s *stack.Stack, rcvWnd, maxInFlight int, handler func(*ForwarderRequest)) *Forwarder { + if rcvWnd == 0 { + rcvWnd = DefaultReceiveBufferSize + } + return &Forwarder{ + stack: s, + maxInFlight: maxInFlight, + handler: handler, + inFlight: make(map[stack.TransportEndpointID]struct{}), + listen: newListenContext(s, protocolFromStack(s), nil /* listenEP */, seqnum.Size(rcvWnd), true, 0), + } +} + +// HandlePacket handles a packet if it is of interest to the forwarder (i.e., if +// it's a SYN packet), returning true if it's the case. Otherwise the packet +// is not handled and false is returned. +// +// This function is expected to be passed as an argument to the +// stack.SetTransportProtocolHandler function. +func (f *Forwarder) HandlePacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) bool { + s, err := newIncomingSegment(id, f.stack.Clock(), pkt) + if err != nil { + return false + } + defer s.DecRef() + + // We only care about well-formed SYN packets (not SYN-ACK) packets. + if !s.csumValid || !s.flags.Contains(header.TCPFlagSyn) || s.flags.Contains(header.TCPFlagAck) { + return false + } + + opts := parseSynSegmentOptions(s) + + f.mu.Lock() + defer f.mu.Unlock() + + // We have an inflight request for this id, ignore this one for now. + if _, ok := f.inFlight[id]; ok { + return true + } + + // Ignore the segment if we're beyond the limit. + if len(f.inFlight) >= f.maxInFlight { + f.stack.Stats().TCP.ForwardMaxInFlightDrop.Increment() + return true + } + + // Launch a new goroutine to handle the request. + f.inFlight[id] = struct{}{} + s.IncRef() + go f.handler(&ForwarderRequest{ // S/R-SAFE: not used by Sentry. + forwarder: f, + segment: s, + synOptions: opts, + }) + + return true +} + +// ForwarderRequest represents a connection request received by the forwarder +// and passed to the client. Clients must eventually call Complete() on it, and +// may optionally create an endpoint to represent it via CreateEndpoint. +type ForwarderRequest struct { + mu forwarderRequestMutex + forwarder *Forwarder + segment *segment + synOptions header.TCPSynOptions +} + +// ID returns the 4-tuple (src address, src port, dst address, dst port) that +// represents the connection request. +func (r *ForwarderRequest) ID() stack.TransportEndpointID { + return r.segment.id +} + +// Complete completes the request, and optionally sends a RST segment back to the +// sender. +func (r *ForwarderRequest) Complete(sendReset bool) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.segment == nil { + panic("Completing already completed forwarder request") + } + + // Remove request from the forwarder. + r.forwarder.mu.Lock() + delete(r.forwarder.inFlight, r.segment.id) + r.forwarder.mu.Unlock() + + if sendReset { + replyWithReset(r.forwarder.stack, r.segment, stack.DefaultTOS, tcpip.UseDefaultIPv4TTL, tcpip.UseDefaultIPv6HopLimit) + } + + // Release all resources. + r.segment.DecRef() + r.segment = nil + r.forwarder = nil +} + +// CreateEndpoint creates a TCP endpoint for the connection request, performing +// the 3-way handshake in the process. +func (r *ForwarderRequest) CreateEndpoint(queue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.segment == nil { + return nil, &tcpip.ErrInvalidEndpointState{} + } + + f := r.forwarder + ep, err := f.listen.performHandshake(r.segment, header.TCPSynOptions{ + MSS: r.synOptions.MSS, + WS: r.synOptions.WS, + TS: r.synOptions.TS, + TSVal: r.synOptions.TSVal, + TSEcr: r.synOptions.TSEcr, + SACKPermitted: r.synOptions.SACKPermitted, + }, queue, nil) + if err != nil { + return nil, err + } + + return ep, nil +} + +// ForwardedPacketExperimentOption returns the experiment option value from the +// forwarded packet and a bool indicating whether an experiment option value was +// found. +func (r *ForwarderRequest) ForwardedPacketExperimentOption() (uint16, bool) { + r.mu.Lock() + defer r.mu.Unlock() + + switch r.segment.pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + h := header.IPv4(r.segment.pkt.NetworkHeader().Slice()) + opts := h.Options() + iter := opts.MakeIterator() + for { + opt, done, err := iter.Next() + if err != nil { + return 0, false + } + if done { + return 0, false + } + if opt.Type() == header.IPv4OptionExperimentType { + return opt.(*header.IPv4OptionExperiment).Value(), true + } + } + case header.IPv6ProtocolNumber: + h := header.IPv6(r.segment.pkt.NetworkHeader().Slice()) + v := r.segment.pkt.NetworkHeader().View() + if v != nil { + v.TrimFront(header.IPv6MinimumSize) + } + buf := buffer.MakeWithView(v) + buf.Append(r.segment.pkt.TransportHeader().View()) + dataBuf := r.segment.pkt.Data().ToBuffer() + buf.Merge(&dataBuf) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(h.NextHeader()), buf) + + for { + hdr, done, err := it.Next() + if done || err != nil { + break + } + if h, ok := hdr.(header.IPv6ExperimentExtHdr); ok { + hdr.Release() + return h.Value, true + } + hdr.Release() + } + default: + panic(fmt.Sprintf("Unexpected network protocol number %d", r.segment.pkt.NetworkProtocolNumber)) + } + return 0, false +} + +func (r *ForwarderRequest) Packet() *stack.PacketBuffer { + return r.segment.pkt +} diff --git a/pkg/tcpip/transport/tcp/forwarder_mutex.go b/pkg/tcpip/transport/tcp/forwarder_mutex.go new file mode 100644 index 0000000..22f136a --- /dev/null +++ b/pkg/tcpip/transport/tcp/forwarder_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type forwarderMutex struct { + mu sync.Mutex +} + +var forwarderprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var forwarderlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type forwarderlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *forwarderMutex) Lock() { + locking.AddGLock(forwarderprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *forwarderMutex) NestedLock(i forwarderlockNameIndex) { + locking.AddGLock(forwarderprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *forwarderMutex) Unlock() { + locking.DelGLock(forwarderprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *forwarderMutex) NestedUnlock(i forwarderlockNameIndex) { + locking.DelGLock(forwarderprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func forwarderinitLockNames() {} + +func init() { + forwarderinitLockNames() + forwarderprefixIndex = locking.NewMutexClass(reflect.TypeOf(forwarderMutex{}), forwarderlockNames) +} diff --git a/pkg/tcpip/transport/tcp/forwarder_request_mutex.go b/pkg/tcpip/transport/tcp/forwarder_request_mutex.go new file mode 100644 index 0000000..c863f58 --- /dev/null +++ b/pkg/tcpip/transport/tcp/forwarder_request_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type forwarderRequestMutex struct { + mu sync.Mutex +} + +var forwarderRequestprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var forwarderRequestlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type forwarderRequestlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *forwarderRequestMutex) Lock() { + locking.AddGLock(forwarderRequestprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *forwarderRequestMutex) NestedLock(i forwarderRequestlockNameIndex) { + locking.AddGLock(forwarderRequestprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *forwarderRequestMutex) Unlock() { + locking.DelGLock(forwarderRequestprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *forwarderRequestMutex) NestedUnlock(i forwarderRequestlockNameIndex) { + locking.DelGLock(forwarderRequestprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func forwarderRequestinitLockNames() {} + +func init() { + forwarderRequestinitLockNames() + forwarderRequestprefixIndex = locking.NewMutexClass(reflect.TypeOf(forwarderRequestMutex{}), forwarderRequestlockNames) +} diff --git a/pkg/tcpip/transport/tcp/hasher_mutex.go b/pkg/tcpip/transport/tcp/hasher_mutex.go new file mode 100644 index 0000000..9151751 --- /dev/null +++ b/pkg/tcpip/transport/tcp/hasher_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type hasherMutex struct { + mu sync.Mutex +} + +var hasherprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var hasherlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type hasherlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *hasherMutex) Lock() { + locking.AddGLock(hasherprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *hasherMutex) NestedLock(i hasherlockNameIndex) { + locking.AddGLock(hasherprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *hasherMutex) Unlock() { + locking.DelGLock(hasherprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *hasherMutex) NestedUnlock(i hasherlockNameIndex) { + locking.DelGLock(hasherprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func hasherinitLockNames() {} + +func init() { + hasherinitLockNames() + hasherprefixIndex = locking.NewMutexClass(reflect.TypeOf(hasherMutex{}), hasherlockNames) +} diff --git a/pkg/tcpip/transport/tcp/keepalive_mutex.go b/pkg/tcpip/transport/tcp/keepalive_mutex.go new file mode 100644 index 0000000..05e4b61 --- /dev/null +++ b/pkg/tcpip/transport/tcp/keepalive_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type keepaliveMutex struct { + mu sync.Mutex +} + +var keepaliveprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var keepalivelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type keepalivelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *keepaliveMutex) Lock() { + locking.AddGLock(keepaliveprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *keepaliveMutex) NestedLock(i keepalivelockNameIndex) { + locking.AddGLock(keepaliveprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *keepaliveMutex) Unlock() { + locking.DelGLock(keepaliveprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *keepaliveMutex) NestedUnlock(i keepalivelockNameIndex) { + locking.DelGLock(keepaliveprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func keepaliveinitLockNames() {} + +func init() { + keepaliveinitLockNames() + keepaliveprefixIndex = locking.NewMutexClass(reflect.TypeOf(keepaliveMutex{}), keepalivelockNames) +} diff --git a/pkg/tcpip/transport/tcp/last_error_mutex.go b/pkg/tcpip/transport/tcp/last_error_mutex.go new file mode 100644 index 0000000..a43798e --- /dev/null +++ b/pkg/tcpip/transport/tcp/last_error_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type lastErrorMutex struct { + mu sync.Mutex +} + +var lastErrorprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var lastErrorlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type lastErrorlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *lastErrorMutex) Lock() { + locking.AddGLock(lastErrorprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *lastErrorMutex) NestedLock(i lastErrorlockNameIndex) { + locking.AddGLock(lastErrorprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *lastErrorMutex) Unlock() { + locking.DelGLock(lastErrorprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *lastErrorMutex) NestedUnlock(i lastErrorlockNameIndex) { + locking.DelGLock(lastErrorprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func lastErrorinitLockNames() {} + +func init() { + lastErrorinitLockNames() + lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeOf(lastErrorMutex{}), lastErrorlockNames) +} diff --git a/pkg/tcpip/transport/tcp/pending_processing_mutex.go b/pkg/tcpip/transport/tcp/pending_processing_mutex.go new file mode 100644 index 0000000..621e9b1 --- /dev/null +++ b/pkg/tcpip/transport/tcp/pending_processing_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type pendingProcessingMutex struct { + mu sync.Mutex +} + +var pendingProcessingprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var pendingProcessinglockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type pendingProcessinglockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *pendingProcessingMutex) Lock() { + locking.AddGLock(pendingProcessingprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *pendingProcessingMutex) NestedLock(i pendingProcessinglockNameIndex) { + locking.AddGLock(pendingProcessingprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *pendingProcessingMutex) Unlock() { + locking.DelGLock(pendingProcessingprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *pendingProcessingMutex) NestedUnlock(i pendingProcessinglockNameIndex) { + locking.DelGLock(pendingProcessingprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func pendingProcessinginitLockNames() {} + +func init() { + pendingProcessinginitLockNames() + pendingProcessingprefixIndex = locking.NewMutexClass(reflect.TypeOf(pendingProcessingMutex{}), pendingProcessinglockNames) +} diff --git a/pkg/tcpip/transport/tcp/protocol.go b/pkg/tcpip/transport/tcp/protocol.go new file mode 100644 index 0000000..0f4d3ca --- /dev/null +++ b/pkg/tcpip/transport/tcp/protocol.go @@ -0,0 +1,606 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tcp contains the implementation of the TCP transport protocol. +package tcp + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "runtime" + "strings" + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" + "github.com/sagernet/gvisor/pkg/tcpip/internal/tcp" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport/raw" + "github.com/sagernet/gvisor/pkg/waiter" +) + +const ( + // ProtocolNumber is the tcp protocol number. + ProtocolNumber = header.TCPProtocolNumber + + // MinBufferSize is the smallest size of a receive or send buffer. + MinBufferSize = 4 << 10 // 4096 bytes. + + // DefaultSendBufferSize is the default size of the send buffer for + // an endpoint. + DefaultSendBufferSize = 1 << 20 // 1MB + + // DefaultReceiveBufferSize is the default size of the receive buffer + // for an endpoint. + DefaultReceiveBufferSize = 1 << 20 // 1MB + + // MaxBufferSize is the largest size a receive/send buffer can grow to. + MaxBufferSize = 4 << 20 // 4MB + + // DefaultTCPLingerTimeout is the amount of time that sockets linger in + // FIN_WAIT_2 state before being marked closed. + DefaultTCPLingerTimeout = 60 * time.Second + + // MaxTCPLingerTimeout is the maximum amount of time that sockets + // linger in FIN_WAIT_2 state before being marked closed. + MaxTCPLingerTimeout = 120 * time.Second + + // DefaultTCPTimeWaitTimeout is the amount of time that sockets linger + // in TIME_WAIT state before being marked closed. + DefaultTCPTimeWaitTimeout = 60 * time.Second + + // DefaultSynRetries is the default value for the number of SYN retransmits + // before a connect is aborted. + DefaultSynRetries = 6 + + // DefaultKeepaliveIdle is the idle time for a connection before keep-alive + // probes are sent. + DefaultKeepaliveIdle = 2 * time.Hour + + // DefaultKeepaliveInterval is the time between two successive keep-alive + // probes. + DefaultKeepaliveInterval = 75 * time.Second + + // DefaultKeepaliveCount is the number of keep-alive probes that are sent + // before declaring the connection dead. + DefaultKeepaliveCount = 9 +) + +const ( + ccReno = "reno" + ccCubic = "cubic" +) + +// +stateify savable +type protocol struct { + stack *stack.Stack + + mu protocolRWMutex `state:"nosave"` + sackEnabled bool + recovery tcpip.TCPRecovery + delayEnabled bool + alwaysUseSynCookies bool + sendBufferSize tcpip.TCPSendBufferSizeRangeOption + recvBufferSize tcpip.TCPReceiveBufferSizeRangeOption + congestionControl string + availableCongestionControl []string + moderateReceiveBuffer bool + lingerTimeout time.Duration + timeWaitTimeout time.Duration + timeWaitReuse tcpip.TCPTimeWaitReuseOption + minRTO time.Duration + maxRTO time.Duration + maxRetries uint32 + synRetries uint8 + dispatcher dispatcher + + // probe, if not nil, will be invoked any time an endpoint receives a + // TCP segment. + // + // This is immutable after creation. + probe TCPProbeFunc `state:"nosave"` + + // The following secrets are initialized once and stay unchanged after. + seqnumSecret [16]byte + tsOffsetSecret [16]byte +} + +// Number returns the tcp protocol number. +func (*protocol) Number() tcpip.TransportProtocolNumber { + return ProtocolNumber +} + +// NewEndpoint creates a new tcp endpoint. +func (p *protocol) NewEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return newEndpoint(p.stack, p, netProto, waiterQueue), nil +} + +// NewRawEndpoint creates a new raw TCP endpoint. Raw TCP sockets are currently +// unsupported. It implements stack.TransportProtocol.NewRawEndpoint. +func (p *protocol) NewRawEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return raw.NewEndpoint(p.stack, netProto, header.TCPProtocolNumber, waiterQueue) +} + +// MinimumPacketSize returns the minimum valid tcp packet size. +func (*protocol) MinimumPacketSize() int { + return header.TCPMinimumSize +} + +// ParsePorts returns the source and destination ports stored in the given tcp +// packet. +func (*protocol) ParsePorts(v []byte) (src, dst uint16, err tcpip.Error) { + h := header.TCP(v) + return h.SourcePort(), h.DestinationPort(), nil +} + +// QueuePacket queues packets targeted at an endpoint after hashing the packet +// to a specific processing queue. Each queue is serviced by its own processor +// goroutine which is responsible for dequeuing and doing full TCP dispatch of +// the packet. +func (p *protocol) QueuePacket(ep stack.TransportEndpoint, id stack.TransportEndpointID, pkt *stack.PacketBuffer) { + p.dispatcher.queuePacket(ep, id, p.stack.Clock(), pkt) +} + +// HandleUnknownDestinationPacket handles packets targeted at this protocol but +// that don't match any existing endpoint. +// +// RFC 793, page 36, states that "If the connection does not exist (CLOSED) then +// a reset is sent in response to any incoming segment except another reset. In +// particular, SYNs addressed to a non-existent connection are rejected by this +// means." +func (p *protocol) HandleUnknownDestinationPacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) stack.UnknownDestinationPacketDisposition { + s, err := newIncomingSegment(id, p.stack.Clock(), pkt) + if err != nil { + return stack.UnknownDestinationPacketMalformed + } + defer s.DecRef() + if !s.csumValid { + return stack.UnknownDestinationPacketMalformed + } + + if !s.flags.Contains(header.TCPFlagRst) { + replyWithReset(p.stack, s, stack.DefaultTOS, tcpip.UseDefaultIPv4TTL, tcpip.UseDefaultIPv6HopLimit) + } + + return stack.UnknownDestinationPacketHandled +} + +func (p *protocol) tsOffset(src, dst tcpip.Address) tcp.TSOffset { + // Initialize a random tsOffset that will be added to the recentTS + // everytime the timestamp is sent when the Timestamp option is enabled. + // + // See https://tools.ietf.org/html/rfc7323#section-5.4 for details on + // why this is required. + h := sha256.New() + + // Per hash.Hash.Writer: + // + // It never returns an error. + _, _ = h.Write(p.tsOffsetSecret[:]) + _, _ = h.Write(src.AsSlice()) + _, _ = h.Write(dst.AsSlice()) + return tcp.NewTSOffset(binary.LittleEndian.Uint32(h.Sum(nil)[:4])) +} + +// replyWithReset replies to the given segment with a reset segment. +// +// If the relevant TTL has its reset value (0 for ipv4TTL, -1 for ipv6HopLimit), +// then the route's default TTL will be used. +func replyWithReset(st *stack.Stack, s *segment, tos, ipv4TTL uint8, ipv6HopLimit int16) tcpip.Error { + net := s.pkt.Network() + route, err := st.FindRoute(s.pkt.NICID, net.DestinationAddress(), net.SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */) + if err != nil { + return err + } + defer route.Release() + + ttl := calculateTTL(route, ipv4TTL, ipv6HopLimit) + + // Get the seqnum from the packet if the ack flag is set. + seq := seqnum.Value(0) + ack := seqnum.Value(0) + flags := header.TCPFlagRst + // As per RFC 793 page 35 (Reset Generation) + // 1. If the connection does not exist (CLOSED) then a reset is sent + // in response to any incoming segment except another reset. In + // particular, SYNs addressed to a non-existent connection are rejected + // by this means. + + // If the incoming segment has an ACK field, the reset takes its + // sequence number from the ACK field of the segment, otherwise the + // reset has sequence number zero and the ACK field is set to the sum + // of the sequence number and segment length of the incoming segment. + // The connection remains in the CLOSED state. + if s.flags.Contains(header.TCPFlagAck) { + seq = s.ackNumber + } else { + flags |= header.TCPFlagAck + ack = s.sequenceNumber.Add(s.logicalLen()) + } + + var expOptVal uint16 + if s.ep != nil { + expOptVal = s.ep.getExperimentOptionValue(route) + } + hdrSize := header.TCPMinimumSize + int(route.MaxHeaderLength()) + if route.NetProto() == header.IPv6ProtocolNumber && expOptVal != 0 { + hdrSize += header.IPv6ExperimentHdrLength + } + p := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize}) + defer p.DecRef() + + return sendTCP(route, tcpFields{ + id: s.id, + ttl: ttl, + tos: tos, + flags: flags, + seq: seq, + ack: ack, + rcvWnd: 0, + expOptVal: expOptVal, + }, p, stack.GSO{}, nil /* PacketOwner */) +} + +// SetOption implements stack.TransportProtocol.SetOption. +func (p *protocol) SetOption(option tcpip.SettableTransportProtocolOption) tcpip.Error { + switch v := option.(type) { + case *tcpip.TCPSACKEnabled: + p.mu.Lock() + p.sackEnabled = bool(*v) + p.mu.Unlock() + return nil + + case *tcpip.TCPRecovery: + p.mu.Lock() + p.recovery = *v + p.mu.Unlock() + return nil + + case *tcpip.TCPDelayEnabled: + p.mu.Lock() + p.delayEnabled = bool(*v) + p.mu.Unlock() + return nil + + case *tcpip.TCPSendBufferSizeRangeOption: + if v.Min <= 0 || v.Default < v.Min || v.Default > v.Max { + return &tcpip.ErrInvalidOptionValue{} + } + p.mu.Lock() + p.sendBufferSize = *v + p.mu.Unlock() + return nil + + case *tcpip.TCPReceiveBufferSizeRangeOption: + if v.Min <= 0 || v.Default < v.Min || v.Default > v.Max { + return &tcpip.ErrInvalidOptionValue{} + } + p.mu.Lock() + p.recvBufferSize = *v + p.mu.Unlock() + return nil + + case *tcpip.CongestionControlOption: + for _, c := range p.availableCongestionControl { + if string(*v) == c { + p.mu.Lock() + p.congestionControl = string(*v) + p.mu.Unlock() + return nil + } + } + // linux returns ENOENT when an invalid congestion control + // is specified. + return &tcpip.ErrNoSuchFile{} + + case *tcpip.TCPModerateReceiveBufferOption: + p.mu.Lock() + p.moderateReceiveBuffer = bool(*v) + p.mu.Unlock() + return nil + + case *tcpip.TCPLingerTimeoutOption: + p.mu.Lock() + if *v < 0 { + p.lingerTimeout = 0 + } else { + p.lingerTimeout = time.Duration(*v) + } + p.mu.Unlock() + return nil + + case *tcpip.TCPTimeWaitTimeoutOption: + p.mu.Lock() + if *v < 0 { + p.timeWaitTimeout = 0 + } else { + p.timeWaitTimeout = time.Duration(*v) + } + p.mu.Unlock() + return nil + + case *tcpip.TCPTimeWaitReuseOption: + if *v < tcpip.TCPTimeWaitReuseDisabled || *v > tcpip.TCPTimeWaitReuseLoopbackOnly { + return &tcpip.ErrInvalidOptionValue{} + } + p.mu.Lock() + p.timeWaitReuse = *v + p.mu.Unlock() + return nil + + case *tcpip.TCPMinRTOOption: + p.mu.Lock() + defer p.mu.Unlock() + if *v < 0 { + p.minRTO = MinRTO + } else if minRTO := time.Duration(*v); minRTO <= p.maxRTO { + p.minRTO = minRTO + } else { + return &tcpip.ErrInvalidOptionValue{} + } + return nil + + case *tcpip.TCPMaxRTOOption: + p.mu.Lock() + defer p.mu.Unlock() + if *v < 0 { + p.maxRTO = MaxRTO + } else if maxRTO := time.Duration(*v); maxRTO >= p.minRTO { + p.maxRTO = maxRTO + } else { + return &tcpip.ErrInvalidOptionValue{} + } + return nil + + case *tcpip.TCPMaxRetriesOption: + p.mu.Lock() + p.maxRetries = uint32(*v) + p.mu.Unlock() + return nil + + case *tcpip.TCPAlwaysUseSynCookies: + p.mu.Lock() + p.alwaysUseSynCookies = bool(*v) + p.mu.Unlock() + return nil + + case *tcpip.TCPSynRetriesOption: + if *v < 1 { + return &tcpip.ErrInvalidOptionValue{} + } + p.mu.Lock() + p.synRetries = uint8(*v) + p.mu.Unlock() + return nil + + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// Option implements stack.TransportProtocol.Option. +func (p *protocol) Option(option tcpip.GettableTransportProtocolOption) tcpip.Error { + switch v := option.(type) { + case *tcpip.TCPSACKEnabled: + p.mu.RLock() + *v = tcpip.TCPSACKEnabled(p.sackEnabled) + p.mu.RUnlock() + return nil + + case *tcpip.TCPRecovery: + p.mu.RLock() + *v = p.recovery + p.mu.RUnlock() + return nil + + case *tcpip.TCPDelayEnabled: + p.mu.RLock() + *v = tcpip.TCPDelayEnabled(p.delayEnabled) + p.mu.RUnlock() + return nil + + case *tcpip.TCPSendBufferSizeRangeOption: + p.mu.RLock() + *v = p.sendBufferSize + p.mu.RUnlock() + return nil + + case *tcpip.TCPReceiveBufferSizeRangeOption: + p.mu.RLock() + *v = p.recvBufferSize + p.mu.RUnlock() + return nil + + case *tcpip.CongestionControlOption: + p.mu.RLock() + *v = tcpip.CongestionControlOption(p.congestionControl) + p.mu.RUnlock() + return nil + + case *tcpip.TCPAvailableCongestionControlOption: + p.mu.RLock() + *v = tcpip.TCPAvailableCongestionControlOption(strings.Join(p.availableCongestionControl, " ")) + p.mu.RUnlock() + return nil + + case *tcpip.TCPModerateReceiveBufferOption: + p.mu.RLock() + *v = tcpip.TCPModerateReceiveBufferOption(p.moderateReceiveBuffer) + p.mu.RUnlock() + return nil + + case *tcpip.TCPLingerTimeoutOption: + p.mu.RLock() + *v = tcpip.TCPLingerTimeoutOption(p.lingerTimeout) + p.mu.RUnlock() + return nil + + case *tcpip.TCPTimeWaitTimeoutOption: + p.mu.RLock() + *v = tcpip.TCPTimeWaitTimeoutOption(p.timeWaitTimeout) + p.mu.RUnlock() + return nil + + case *tcpip.TCPTimeWaitReuseOption: + p.mu.RLock() + *v = p.timeWaitReuse + p.mu.RUnlock() + return nil + + case *tcpip.TCPMinRTOOption: + p.mu.RLock() + *v = tcpip.TCPMinRTOOption(p.minRTO) + p.mu.RUnlock() + return nil + + case *tcpip.TCPMaxRTOOption: + p.mu.RLock() + *v = tcpip.TCPMaxRTOOption(p.maxRTO) + p.mu.RUnlock() + return nil + + case *tcpip.TCPMaxRetriesOption: + p.mu.RLock() + *v = tcpip.TCPMaxRetriesOption(p.maxRetries) + p.mu.RUnlock() + return nil + + case *tcpip.TCPAlwaysUseSynCookies: + p.mu.RLock() + *v = tcpip.TCPAlwaysUseSynCookies(p.alwaysUseSynCookies) + p.mu.RUnlock() + return nil + + case *tcpip.TCPSynRetriesOption: + p.mu.RLock() + *v = tcpip.TCPSynRetriesOption(p.synRetries) + p.mu.RUnlock() + return nil + + default: + return &tcpip.ErrUnknownProtocolOption{} + } +} + +// SendBufferSize implements stack.SendBufSizeProto. +func (p *protocol) SendBufferSize() tcpip.TCPSendBufferSizeRangeOption { + p.mu.RLock() + defer p.mu.RUnlock() + return p.sendBufferSize +} + +// Close implements stack.TransportProtocol.Close. +func (p *protocol) Close() { + p.dispatcher.close() +} + +// Wait implements stack.TransportProtocol.Wait. +func (p *protocol) Wait() { + p.dispatcher.wait() +} + +// Pause implements stack.TransportProtocol.Pause. +func (p *protocol) Pause() { + p.dispatcher.pause() +} + +// Resume implements stack.TransportProtocol.Resume. +func (p *protocol) Resume() { + p.dispatcher.resume() +} + +// Restore implements stack.TransportProtocol.Restore. +func (p *protocol) Restore() { + p.dispatcher.start() +} + +// Parse implements stack.TransportProtocol.Parse. +func (*protocol) Parse(pkt *stack.PacketBuffer) bool { + return parse.TCP(pkt) +} + +// NewProtocol returns a TCP transport protocol with Reno congestion control. +func NewProtocol(s *stack.Stack) stack.TransportProtocol { + return newProtocol(s, ccReno, nil) +} + +// NewProtocolProbe returns a TCP transport protocol with Reno congestion +// control and the given probe. +// +// The probe will be invoked on every segment received by TCP endpoints. The +// probe function is passed a copy of the TCP endpoint state before and after +// processing of the segment. +func NewProtocolProbe(probe TCPProbeFunc) func(*stack.Stack) stack.TransportProtocol { + return func(s *stack.Stack) stack.TransportProtocol { + return newProtocol(s, ccReno, probe) + } +} + +// NewProtocolCUBIC returns a TCP transport protocol with CUBIC congestion +// control. +// +// TODO(b/345835636): Remove this and make CUBIC the default across the board. +func NewProtocolCUBIC(s *stack.Stack) stack.TransportProtocol { + return newProtocol(s, ccCubic, nil) +} + +func newProtocol(s *stack.Stack, cc string, probe TCPProbeFunc) stack.TransportProtocol { + rng := s.SecureRNG() + var seqnumSecret [16]byte + var tsOffsetSecret [16]byte + if n, err := rng.Reader.Read(seqnumSecret[:]); err != nil || n != len(seqnumSecret) { + panic(fmt.Sprintf("Read() failed: %v", err)) + } + if n, err := rng.Reader.Read(tsOffsetSecret[:]); err != nil || n != len(tsOffsetSecret) { + panic(fmt.Sprintf("Read() failed: %v", err)) + } + p := protocol{ + stack: s, + sendBufferSize: tcpip.TCPSendBufferSizeRangeOption{ + Min: MinBufferSize, + Default: DefaultSendBufferSize, + Max: MaxBufferSize, + }, + recvBufferSize: tcpip.TCPReceiveBufferSizeRangeOption{ + Min: MinBufferSize, + Default: DefaultReceiveBufferSize, + Max: MaxBufferSize, + }, + sackEnabled: true, + congestionControl: cc, + availableCongestionControl: []string{ccReno, ccCubic}, + moderateReceiveBuffer: true, + lingerTimeout: DefaultTCPLingerTimeout, + timeWaitTimeout: DefaultTCPTimeWaitTimeout, + timeWaitReuse: tcpip.TCPTimeWaitReuseLoopbackOnly, + synRetries: DefaultSynRetries, + minRTO: MinRTO, + maxRTO: MaxRTO, + maxRetries: MaxRetries, + recovery: tcpip.TCPRACKLossDetection, + seqnumSecret: seqnumSecret, + tsOffsetSecret: tsOffsetSecret, + probe: probe, + } + p.dispatcher.init(s.InsecureRNG(), runtime.GOMAXPROCS(0)) + return &p +} + +// protocolFromStack retrieves the tcp.protocol instance from stack s. +func protocolFromStack(s *stack.Stack) *protocol { + return s.TransportProtocolInstance(ProtocolNumber).(*protocol) +} diff --git a/pkg/tcpip/transport/tcp/protocol_mutex.go b/pkg/tcpip/transport/tcp/protocol_mutex.go new file mode 100644 index 0000000..ad7f392 --- /dev/null +++ b/pkg/tcpip/transport/tcp/protocol_mutex.go @@ -0,0 +1,96 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// RWMutex is sync.RWMutex with the correctness validator. +type protocolRWMutex struct { + mu sync.RWMutex +} + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var protocollockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type protocollockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *protocolRWMutex) Lock() { + locking.AddGLock(protocolprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *protocolRWMutex) NestedLock(i protocollockNameIndex) { + locking.AddGLock(protocolprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *protocolRWMutex) Unlock() { + m.mu.Unlock() + locking.DelGLock(protocolprefixIndex, -1) +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *protocolRWMutex) NestedUnlock(i protocollockNameIndex) { + m.mu.Unlock() + locking.DelGLock(protocolprefixIndex, int(i)) +} + +// RLock locks m for reading. +// +checklocksignore +func (m *protocolRWMutex) RLock() { + locking.AddGLock(protocolprefixIndex, -1) + m.mu.RLock() +} + +// RUnlock undoes a single RLock call. +// +checklocksignore +func (m *protocolRWMutex) RUnlock() { + m.mu.RUnlock() + locking.DelGLock(protocolprefixIndex, -1) +} + +// RLockBypass locks m for reading without executing the validator. +// +checklocksignore +func (m *protocolRWMutex) RLockBypass() { + m.mu.RLock() +} + +// RUnlockBypass undoes a single RLockBypass call. +// +checklocksignore +func (m *protocolRWMutex) RUnlockBypass() { + m.mu.RUnlock() +} + +// DowngradeLock atomically unlocks rw for writing and locks it for reading. +// +checklocksignore +func (m *protocolRWMutex) DowngradeLock() { + m.mu.DowngradeLock() +} + +var protocolprefixIndex *locking.MutexClass + +// DO NOT REMOVE: The following function is automatically replaced. +func protocolinitLockNames() {} + +func init() { + protocolinitLockNames() + protocolprefixIndex = locking.NewMutexClass(reflect.TypeOf(protocolRWMutex{}), protocollockNames) +} diff --git a/pkg/tcpip/transport/tcp/rack.go b/pkg/tcpip/transport/tcp/rack.go new file mode 100644 index 0000000..a64cc83 --- /dev/null +++ b/pkg/tcpip/transport/tcp/rack.go @@ -0,0 +1,459 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +const ( + // wcDelayedACKTimeout is the recommended maximum delayed ACK timer + // value as defined in the RFC. It stands for worst case delayed ACK + // timer (WCDelAckT). When FlightSize is 1, PTO is inflated by + // WCDelAckT time to compensate for a potential long delayed ACK timer + // at the receiver. + // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5. + wcDelayedACKTimeout = 200 * time.Millisecond + + // tcpRACKRecoveryThreshold is the number of loss recoveries for which + // the reorder window is inflated and after that the reorder window is + // reset to its initial value of minRTT/4. + // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2. + tcpRACKRecoveryThreshold = 16 +) + +// RACK is a loss detection algorithm used in TCP to detect packet loss and +// reordering using transmission timestamp of the packets instead of packet or +// sequence counts. To use RACK, SACK should be enabled on the connection. + +// rackControl stores the rack related fields. +// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-6.1 +// +// +stateify savable +type rackControl struct { + TCPRACKState + + // exitedRecovery indicates if the connection is exiting loss recovery. + // This flag is set if the sender is leaving the recovery after + // receiving an ACK and is reset during updating of reorder window. + exitedRecovery bool + + // minRTT is the estimated minimum RTT of the connection. + minRTT time.Duration + + // tlpRxtOut indicates whether there is an unacknowledged + // TLP retransmission. + tlpRxtOut bool + + // tlpHighRxt the value of sender.sndNxt at the time of sending + // a TLP retransmission. + tlpHighRxt seqnum.Value + + // snd is a reference to the sender. + snd *sender +} + +// init initializes RACK specific fields. +func (rc *rackControl) init(snd *sender, iss seqnum.Value) { + rc.FACK = iss + rc.ReoWndIncr = 1 + rc.snd = snd +} + +// update will update the RACK related fields when an ACK has been received. +// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-09#section-6.2 +func (rc *rackControl) update(seg *segment, ackSeg *segment) { + rtt := rc.snd.ep.stack.Clock().NowMonotonic().Sub(seg.xmitTime) + + // If the ACK is for a retransmitted packet, do not update if it is a + // spurious inference which is determined by below checks: + // 1. When Timestamping option is available, if the TSVal is less than + // the transmit time of the most recent retransmitted packet. + // 2. When RTT calculated for the packet is less than the smoothed RTT + // for the connection. + // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 + // step 2 + if seg.xmitCount > 1 { + if ackSeg.parsedOptions.TS && ackSeg.parsedOptions.TSEcr != 0 { + if ackSeg.parsedOptions.TSEcr < rc.snd.ep.tsVal(seg.xmitTime) { + return + } + } + if rtt < rc.minRTT { + return + } + } + + rc.RTT = rtt + + // The sender can either track a simple global minimum of all RTT + // measurements from the connection, or a windowed min-filtered value + // of recent RTT measurements. This implementation keeps track of the + // simple global minimum of all RTTs for the connection. + if rtt < rc.minRTT || rc.minRTT == 0 { + rc.minRTT = rtt + } + + // Update rc.xmitTime and rc.endSequence to the transmit time and + // ending sequence number of the packet which has been acknowledged + // most recently. + endSeq := seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize())) + if rc.XmitTime.Before(seg.xmitTime) || (seg.xmitTime == rc.XmitTime && rc.EndSequence.LessThan(endSeq)) { + rc.XmitTime = seg.xmitTime + rc.EndSequence = endSeq + } +} + +// detectReorder detects if packet reordering has been observed. +// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 +// - Step 3: Detect data segment reordering. +// To detect reordering, the sender looks for original data segments being +// delivered out of order. To detect such cases, the sender tracks the +// highest sequence selectively or cumulatively acknowledged in the RACK.fack +// variable. The name "fack" stands for the most "Forward ACK" (this term is +// adopted from [FACK]). If a never retransmitted segment that's below +// RACK.fack is (selectively or cumulatively) acknowledged, it has been +// delivered out of order. The sender sets RACK.reord to TRUE if such segment +// is identified. +func (rc *rackControl) detectReorder(seg *segment) { + endSeq := seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize())) + if rc.FACK.LessThan(endSeq) { + rc.FACK = endSeq + return + } + + if endSeq.LessThan(rc.FACK) && seg.xmitCount == 1 { + rc.Reord = true + } +} + +func (rc *rackControl) setDSACKSeen(dsackSeen bool) { + rc.DSACKSeen = dsackSeen +} + +// shouldSchedulePTO dictates whether we should schedule a PTO or not. +// See https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.1. +func (s *sender) shouldSchedulePTO() bool { + // Schedule PTO only if RACK loss detection is enabled. + return s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 && + // The connection supports SACK. + s.ep.SACKPermitted && + // The connection is not in loss recovery. + (s.state != tcpip.RTORecovery && s.state != tcpip.SACKRecovery) && + // The connection has no SACKed sequences in the SACK scoreboard. + s.ep.scoreboard.Sacked() == 0 +} + +// schedulePTO schedules the probe timeout as defined in +// https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.1. +// +// +checklocks:s.ep.mu +func (s *sender) schedulePTO() { + pto := time.Second + s.rtt.Lock() + if s.rtt.TCPRTTState.SRTTInited && s.rtt.TCPRTTState.SRTT > 0 { + pto = s.rtt.TCPRTTState.SRTT * 2 + if s.Outstanding == 1 { + pto += wcDelayedACKTimeout + } + } + s.rtt.Unlock() + + now := s.ep.stack.Clock().NowMonotonic() + if s.resendTimer.enabled() { + if now.Add(pto).After(s.resendTimer.target) { + pto = s.resendTimer.target.Sub(now) + } + s.resendTimer.disable() + } + + s.probeTimer.enable(pto) +} + +// probeTimerExpired is the same as TLP_send_probe() as defined in +// https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.2. +// +// +checklocks:s.ep.mu +func (s *sender) probeTimerExpired() tcpip.Error { + if s.probeTimer.isUninitialized() || !s.probeTimer.checkExpiration() { + return nil + } + + var dataSent bool + if s.writeNext != nil && s.writeNext.xmitCount == 0 && s.Outstanding < s.SndCwnd { + dataSent = s.maybeSendSegment(s.writeNext, int(s.ep.scoreboard.SMSS()), s.SndUna.Add(s.SndWnd)) + if dataSent { + s.Outstanding += s.pCount(s.writeNext, s.MaxPayloadSize) + s.updateWriteNext(s.writeNext.Next()) + } + } + + if !dataSent && !s.rc.tlpRxtOut { + var highestSeqXmit *segment + for highestSeqXmit = s.writeList.Front(); highestSeqXmit != nil; highestSeqXmit = highestSeqXmit.Next() { + if highestSeqXmit.xmitCount == 0 { + // Nothing in writeList is transmitted, no need to send a probe. + highestSeqXmit = nil + break + } + if highestSeqXmit.Next() == nil || highestSeqXmit.Next().xmitCount == 0 { + // Either everything in writeList has been transmitted or the next + // sequence has not been transmitted. Either way this is the highest + // sequence segment that was transmitted. + break + } + } + + if highestSeqXmit != nil { + dataSent = s.maybeSendSegment(highestSeqXmit, int(s.ep.scoreboard.SMSS()), s.SndUna.Add(s.SndWnd)) + if dataSent { + s.rc.tlpRxtOut = true + s.rc.tlpHighRxt = s.SndNxt + } + } + } + + // Whether or not the probe was sent, the sender must arm the resend timer, + // not the probe timer. This ensures that the sender does not send repeated, + // back-to-back tail loss probes. + s.postXmit(dataSent, false /* shouldScheduleProbe */) + return nil +} + +// detectTLPRecovery detects if recovery was accomplished by the loss probes +// and updates TLP state accordingly. +// See https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.6.3. +// +// +checklocks:s.ep.mu +func (s *sender) detectTLPRecovery(ack seqnum.Value, rcvdSeg *segment) { + if !(s.ep.SACKPermitted && s.rc.tlpRxtOut) { + return + } + + // Step 1. + if s.isDupAck(rcvdSeg) && ack == s.rc.tlpHighRxt { + var sbAboveTLPHighRxt bool + for _, sb := range rcvdSeg.parsedOptions.SACKBlocks { + if s.rc.tlpHighRxt.LessThan(sb.End) { + sbAboveTLPHighRxt = true + break + } + } + if !sbAboveTLPHighRxt { + // TLP episode is complete. + s.rc.tlpRxtOut = false + } + } + + if s.rc.tlpRxtOut && s.rc.tlpHighRxt.LessThanEq(ack) { + // TLP episode is complete. + s.rc.tlpRxtOut = false + if !checkDSACK(rcvdSeg) { + // Step 2. Either the original packet or the retransmission (in the + // form of a probe) was lost. Invoke a congestion control response + // equivalent to fast recovery. + s.cc.HandleLossDetected() + s.enterRecovery() + s.leaveRecovery() + } + } +} + +// updateRACKReorderWindow updates the reorder window. +// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 +// - Step 4: Update RACK reordering window +// To handle the prevalent small degree of reordering, RACK.reo_wnd serves as +// an allowance for settling time before marking a packet lost. RACK starts +// initially with a conservative window of min_RTT/4. If no reordering has +// been observed RACK uses reo_wnd of zero during loss recovery, in order to +// retransmit quickly, or when the number of DUPACKs exceeds the classic +// DUPACKthreshold. +// +// +checklocks:rc.snd.ep.mu +func (rc *rackControl) updateRACKReorderWindow() { + dsackSeen := rc.DSACKSeen + snd := rc.snd + + // React to DSACK once per round trip. + // If SND.UNA < RACK.rtt_seq: + // RACK.dsack = false + if snd.SndUna.LessThan(rc.RTTSeq) { + dsackSeen = false + } + + // If RACK.dsack: + // RACK.reo_wnd_incr += 1 + // RACK.dsack = false + // RACK.rtt_seq = SND.NXT + // RACK.reo_wnd_persist = 16 + if dsackSeen { + rc.ReoWndIncr++ + dsackSeen = false + rc.RTTSeq = snd.SndNxt + rc.ReoWndPersist = tcpRACKRecoveryThreshold + } else if rc.exitedRecovery { + // Else if exiting loss recovery: + // RACK.reo_wnd_persist -= 1 + // If RACK.reo_wnd_persist <= 0: + // RACK.reo_wnd_incr = 1 + rc.ReoWndPersist-- + if rc.ReoWndPersist <= 0 { + rc.ReoWndIncr = 1 + } + rc.exitedRecovery = false + } + + // Reorder window is zero during loss recovery, or when the number of + // DUPACKs exceeds the classic DUPACKthreshold. + // If RACK.reord is FALSE: + // If in loss recovery: (If in fast or timeout recovery) + // RACK.reo_wnd = 0 + // Return + // Else if RACK.pkts_sacked >= RACK.dupthresh: + // RACK.reo_wnd = 0 + // return + if !rc.Reord { + if snd.state == tcpip.RTORecovery || snd.state == tcpip.SACKRecovery { + rc.ReoWnd = 0 + return + } + + if snd.SackedOut >= nDupAckThreshold { + rc.ReoWnd = 0 + return + } + } + + // Calculate reorder window. + // RACK.reo_wnd = RACK.min_RTT / 4 * RACK.reo_wnd_incr + // RACK.reo_wnd = min(RACK.reo_wnd, SRTT) + snd.rtt.Lock() + srtt := snd.rtt.TCPRTTState.SRTT + snd.rtt.Unlock() + rc.ReoWnd = time.Duration((int64(rc.minRTT) / 4) * int64(rc.ReoWndIncr)) + if srtt < rc.ReoWnd { + rc.ReoWnd = srtt + } +} + +func (rc *rackControl) exitRecovery() { + rc.exitedRecovery = true +} + +// detectLoss marks the segment as lost if the reordering window has elapsed +// and the ACK is not received. It will also arm the reorder timer. +// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 Step 5. +// +// +checklocks:rc.snd.ep.mu +func (rc *rackControl) detectLoss(rcvTime tcpip.MonotonicTime) int { + var timeout time.Duration + numLost := 0 + for seg := rc.snd.writeList.Front(); seg != nil && seg.xmitCount != 0; seg = seg.Next() { + if rc.snd.ep.scoreboard.IsSACKED(seg.sackBlock()) { + continue + } + + if seg.lost && seg.xmitCount == 1 { + numLost++ + continue + } + + endSeq := seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize())) + if seg.xmitTime.Before(rc.XmitTime) || (seg.xmitTime == rc.XmitTime && rc.EndSequence.LessThan(endSeq)) { + timeRemaining := seg.xmitTime.Sub(rcvTime) + rc.RTT + rc.ReoWnd + if timeRemaining <= 0 { + seg.lost = true + numLost++ + } else if timeRemaining > timeout { + timeout = timeRemaining + } + } + } + + if timeout != 0 && !rc.snd.reorderTimer.enabled() { + rc.snd.reorderTimer.enable(timeout) + } + return numLost +} + +// reorderTimerExpired will retransmit the segments which have not been acked +// before the reorder timer expired. +// +// +checklocks:rc.snd.ep.mu +func (rc *rackControl) reorderTimerExpired() tcpip.Error { + if rc.snd.reorderTimer.isUninitialized() || !rc.snd.reorderTimer.checkExpiration() { + return nil + } + + numLost := rc.detectLoss(rc.snd.ep.stack.Clock().NowMonotonic()) + if numLost == 0 { + return nil + } + + fastRetransmit := false + if !rc.snd.FastRecovery.Active { + rc.snd.cc.HandleLossDetected() + rc.snd.enterRecovery() + fastRetransmit = true + } + + rc.DoRecovery(nil, fastRetransmit) + return nil +} + +// DoRecovery implements lossRecovery.DoRecovery. +// +// +checklocks:rc.snd.ep.mu +func (rc *rackControl) DoRecovery(_ *segment, fastRetransmit bool) { + snd := rc.snd + if fastRetransmit { + snd.resendSegment() + } + + var dataSent bool + // Iterate the writeList and retransmit the segments which are marked + // as lost by RACK. + for seg := snd.writeList.Front(); seg != nil && seg.xmitCount > 0; seg = seg.Next() { + if seg == snd.writeNext { + break + } + + if !seg.lost { + continue + } + + // Reset seg.lost as it is already SACKed. + if snd.ep.scoreboard.IsSACKED(seg.sackBlock()) { + seg.lost = false + continue + } + + // Check the congestion window after entering recovery. + if snd.Outstanding >= snd.SndCwnd { + break + } + + if sent := snd.maybeSendSegment(seg, int(snd.ep.scoreboard.SMSS()), snd.SndUna.Add(snd.SndWnd)); !sent { + break + } + dataSent = true + snd.Outstanding += snd.pCount(seg, snd.MaxPayloadSize) + } + + snd.postXmit(dataSent, true /* shouldScheduleProbe */) +} diff --git a/pkg/tcpip/transport/tcp/rcv.go b/pkg/tcpip/transport/tcp/rcv.go new file mode 100644 index 0000000..31d36ff --- /dev/null +++ b/pkg/tcpip/transport/tcp/rcv.go @@ -0,0 +1,616 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "container/heap" + "math" + + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +// receiver holds the state necessary to receive TCP segments and turn them +// into a stream of bytes. +// +// +stateify savable +type receiver struct { + TCPReceiverState + ep *Endpoint + + // rcvWnd is the non-scaled receive window last advertised to the peer. + rcvWnd seqnum.Size + + // rcvWUP is the RcvNxt value at the last window update sent. + rcvWUP seqnum.Value + + // prevBufused is the snapshot of endpoint rcvBufUsed taken when we + // advertise a receive window. + prevBufUsed int + + closed bool + + // pendingRcvdSegments is bounded by the receive buffer size of the + // endpoint. + pendingRcvdSegments segmentHeap + + // Time when the last ack was received. + lastRcvdAckTime tcpip.MonotonicTime +} + +func newReceiver(ep *Endpoint, irs seqnum.Value, rcvWnd seqnum.Size, rcvWndScale uint8) *receiver { + return &receiver{ + ep: ep, + TCPReceiverState: TCPReceiverState{ + RcvNxt: irs + 1, + RcvAcc: irs.Add(rcvWnd + 1), + RcvWndScale: rcvWndScale, + }, + rcvWnd: rcvWnd, + rcvWUP: irs + 1, + lastRcvdAckTime: ep.stack.Clock().NowMonotonic(), + } +} + +// acceptable checks if the segment sequence number range is acceptable +// according to the table on page 26 of RFC 793. +func (r *receiver) acceptable(segSeq seqnum.Value, segLen seqnum.Size) bool { + // r.rcvWnd could be much larger than the window size we advertised in our + // outgoing packets, we should use what we have advertised for acceptability + // test. + scaledWindowSize := r.rcvWnd >> r.RcvWndScale + if scaledWindowSize > math.MaxUint16 { + // This is what we actually put in the Window field. + scaledWindowSize = math.MaxUint16 + } + advertisedWindowSize := scaledWindowSize << r.RcvWndScale + return header.Acceptable(segSeq, segLen, r.RcvNxt, r.RcvNxt.Add(advertisedWindowSize)) +} + +// currentWindow returns the available space in the window that was advertised +// last to our peer. +func (r *receiver) currentWindow() (curWnd seqnum.Size) { + endOfWnd := r.rcvWUP.Add(r.rcvWnd) + if endOfWnd.LessThan(r.RcvNxt) { + // return 0 if r.RcvNxt is past the end of the previously advertised window. + // This can happen because we accept a large segment completely even if + // accepting it causes it to partially exceed the advertised window. + return 0 + } + return r.RcvNxt.Size(endOfWnd) +} + +// getSendParams returns the parameters needed by the sender when building +// segments to send. +// +checklocks:r.ep.mu +// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu +func (r *receiver) getSendParams() (RcvNxt seqnum.Value, rcvWnd seqnum.Size) { + newWnd := r.ep.selectWindow() + curWnd := r.currentWindow() + unackLen := int(r.ep.snd.MaxSentAck.Size(r.RcvNxt)) + bufUsed := r.ep.receiveBufferUsed() + + // Grow the right edge of the window only for payloads larger than the + // the segment overhead OR if the application is actively consuming data. + // + // Avoiding growing the right edge otherwise, addresses a situation below: + // An application has been slow in reading data and we have burst of + // incoming segments lengths < segment overhead. Here, our available free + // memory would reduce drastically when compared to the advertised receive + // window. + // + // For example: With incoming 512 bytes segments, segment overhead of + // 552 bytes (at the time of writing this comment), with receive window + // starting from 1MB and with rcvAdvWndScale being 1, buffer would reach 0 + // when the curWnd is still 19436 bytes, because for every incoming segment + // newWnd would reduce by (552+512) >> rcvAdvWndScale (current value 1), + // while curWnd would reduce by 512 bytes. + // Such a situation causes us to keep tail dropping the incoming segments + // and never advertise zero receive window to the peer. + // + // Linux does a similar check for minimal sk_buff size (128): + // https://github.com/torvalds/linux/blob/d5beb3140f91b1c8a3d41b14d729aefa4dcc58bc/net/ipv4/tcp_input.c#L783 + // + // Also, if the application is reading the data, we keep growing the right + // edge, as we are still advertising a window that we think can be serviced. + toGrow := unackLen >= SegOverheadSize || bufUsed <= r.prevBufUsed + + // Update RcvAcc only if new window is > previously advertised window. We + // should never shrink the acceptable sequence space once it has been + // advertised the peer. If we shrink the acceptable sequence space then we + // would end up dropping bytes that might already be in flight. + // ==================================================== sequence space. + // ^ ^ ^ ^ + // rcvWUP RcvNxt RcvAcc new RcvAcc + // <=====curWnd ===> + // <========= newWnd > curWnd ========= > + if r.RcvNxt.Add(curWnd).LessThan(r.RcvNxt.Add(newWnd)) && toGrow { + // If the new window moves the right edge, then update RcvAcc. + r.RcvAcc = r.RcvNxt.Add(newWnd) + } else { + if newWnd == 0 { + // newWnd is zero but we can't advertise a zero as it would cause window + // to shrink so just increment a metric to record this event. + r.ep.stats.ReceiveErrors.WantZeroRcvWindow.Increment() + } + newWnd = curWnd + } + + // Apply silly-window avoidance when recovering from zero-window situation. + // Keep advertising zero receive window up until the new window reaches a + // threshold. + if r.rcvWnd == 0 && newWnd != 0 { + r.ep.rcvQueueMu.Lock() + if crossed, above := r.ep.windowCrossedACKThresholdLocked(int(newWnd), int(r.ep.ops.GetReceiveBufferSize())); !crossed && !above { + newWnd = 0 + } + r.ep.rcvQueueMu.Unlock() + } + + // Stash away the non-scaled receive window as we use it for measuring + // receiver's estimated RTT. + r.rcvWnd = newWnd + r.rcvWUP = r.RcvNxt + r.prevBufUsed = bufUsed + scaledWnd := r.rcvWnd >> r.RcvWndScale + if scaledWnd == 0 { + // Increment a metric if we are advertising an actual zero window. + r.ep.stats.ReceiveErrors.ZeroRcvWindowState.Increment() + } + + // If we started off with a window larger than what can he held in + // the 16bit window field, we ceil the value to the max value. + if scaledWnd > math.MaxUint16 { + scaledWnd = seqnum.Size(math.MaxUint16) + + // Ensure that the stashed receive window always reflects what + // is being advertised. + r.rcvWnd = scaledWnd << r.RcvWndScale + } + return r.RcvNxt, scaledWnd +} + +// nonZeroWindow is called when the receive window grows from zero to nonzero; +// in such cases we may need to send an ack to indicate to our peer that it can +// resume sending data. +// +checklocks:r.ep.mu +// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu +func (r *receiver) nonZeroWindow() { + // Immediately send an ack. + r.ep.snd.sendAck() +} + +// consumeSegment attempts to consume a segment that was received by r. The +// segment may have just been received or may have been received earlier but +// wasn't ready to be consumed then. +// +// Returns true if the segment was consumed, false if it cannot be consumed +// yet because of a missing segment. +// +checklocks:r.ep.mu +// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu +func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum.Size) bool { + if segLen > 0 { + // If the segment doesn't include the seqnum we're expecting to + // consume now, we're missing a segment. We cannot proceed until + // we receive that segment though. + if !r.RcvNxt.InWindow(segSeq, segLen) { + return false + } + + // Trim segment to eliminate already acknowledged data. + if segSeq.LessThan(r.RcvNxt) { + diff := segSeq.Size(r.RcvNxt) + segLen -= diff + segSeq.UpdateForward(diff) + s.sequenceNumber.UpdateForward(diff) + s.TrimFront(diff) + } + + // Move segment to ready-to-deliver list. Wakeup any waiters. + r.ep.readyToRead(s) + + } else if segSeq != r.RcvNxt { + return false + } + + // Update the segment that we're expecting to consume. + r.RcvNxt = segSeq.Add(segLen) + + // In cases of a misbehaving sender which could send more than the + // advertised window, we could end up in a situation where we get a + // segment that exceeds the window advertised. Instead of partially + // accepting the segment and discarding bytes beyond the advertised + // window, we accept the whole segment and make sure r.RcvAcc is moved + // forward to match r.RcvNxt to indicate that the window is now closed. + // + // In absence of this check the r.acceptable() check fails and accepts + // segments that should be dropped because rcvWnd is calculated as + // the size of the interval (RcvNxt, RcvAcc] which becomes extremely + // large if RcvAcc is ever less than RcvNxt. + if r.RcvAcc.LessThan(r.RcvNxt) { + r.RcvAcc = r.RcvNxt + } + + // Trim SACK Blocks to remove any SACK information that covers + // sequence numbers that have been consumed. + TrimSACKBlockList(&r.ep.sack, r.RcvNxt) + + // Handle FIN or FIN-ACK. + if s.flags.Contains(header.TCPFlagFin) { + r.RcvNxt++ + + // Send ACK immediately. + r.ep.snd.sendAck() + + // Tell any readers that no more data will come. + r.closed = true + r.ep.readyToRead(nil) + + // We just received a FIN, our next state depends on whether we sent a + // FIN already or not. + switch r.ep.EndpointState() { + case StateEstablished: + r.ep.setEndpointState(StateCloseWait) + case StateFinWait1: + if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt { + // FIN-ACK, transition to TIME-WAIT. + r.ep.setEndpointState(StateTimeWait) + } else { + // Simultaneous close, expecting a final ACK. + r.ep.setEndpointState(StateClosing) + } + case StateFinWait2: + r.ep.setEndpointState(StateTimeWait) + } + + // Flush out any pending segments, except the very first one if + // it happens to be the one we're handling now because the + // caller is using it. + first := 0 + if len(r.pendingRcvdSegments) != 0 && r.pendingRcvdSegments[0] == s { + first = 1 + } + + for i := first; i < len(r.pendingRcvdSegments); i++ { + r.PendingBufUsed -= r.pendingRcvdSegments[i].segMemSize() + r.pendingRcvdSegments[i].DecRef() + // Note that slice truncation does not allow garbage + // collection of truncated items, thus truncated items + // must be set to nil to avoid memory leaks. + r.pendingRcvdSegments[i] = nil + } + r.pendingRcvdSegments = r.pendingRcvdSegments[:first] + r.ep.updateConnDirectionState(connDirectionStateRcvClosed) + + return true + } + + // Handle ACK (not FIN-ACK, which we handled above) during one of the + // shutdown states. + if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt { + switch r.ep.EndpointState() { + case StateFinWait1: + r.ep.setEndpointState(StateFinWait2) + if e := r.ep; e.closed { + // The socket has been closed and we are in + // FIN-WAIT-2 so start the FIN-WAIT-2 timer. + e.finWait2Timer = e.stack.Clock().AfterFunc(e.tcpLingerTimeout, e.finWait2TimerExpired) + } + + case StateClosing: + r.ep.setEndpointState(StateTimeWait) + case StateLastAck: + r.ep.transitionToStateCloseLocked() + } + } + + return true +} + +// updateRTT updates the receiver RTT measurement based on the sequence number +// of the received segment. +func (r *receiver) updateRTT() { + // From: https://public.lanl.gov/radiant/pubs/drs/sc2001-poster.pdf + // + // A system that is only transmitting acknowledgements can still + // estimate the round-trip time by observing the time between when a byte + // is first acknowledged and the receipt of data that is at least one + // window beyond the sequence number that was acknowledged. + r.ep.rcvQueueMu.Lock() + if r.ep.RcvAutoParams.RTTMeasureTime == (tcpip.MonotonicTime{}) { + // New measurement. + r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic() + r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd) + r.ep.rcvQueueMu.Unlock() + return + } + if r.RcvNxt.LessThan(r.ep.RcvAutoParams.RTTMeasureSeqNumber) { + r.ep.rcvQueueMu.Unlock() + return + } + rtt := r.ep.stack.Clock().NowMonotonic().Sub(r.ep.RcvAutoParams.RTTMeasureTime) + // We only store the minimum observed RTT here as this is only used in + // absence of a SRTT available from either timestamps or a sender + // measurement of RTT. + if r.ep.RcvAutoParams.RTT == 0 || rtt < r.ep.RcvAutoParams.RTT { + r.ep.RcvAutoParams.RTT = rtt + } + r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic() + r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd) + r.ep.rcvQueueMu.Unlock() +} + +// +checklocks:r.ep.mu +// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu +func (r *receiver) handleRcvdSegmentClosing(s *segment, state EndpointState, closed bool) (drop bool, err tcpip.Error) { + r.ep.rcvQueueMu.Lock() + rcvClosed := r.ep.RcvClosed || r.closed + r.ep.rcvQueueMu.Unlock() + + // If we are in one of the shutdown states then we need to do + // additional checks before we try and process the segment. + switch state { + case StateCloseWait, StateClosing, StateLastAck: + if !s.sequenceNumber.LessThanEq(r.RcvNxt) { + // Just drop the segment as we have + // already received a FIN and this + // segment is after the sequence number + // for the FIN. + return true, nil + } + fallthrough + case StateFinWait1, StateFinWait2: + // If the ACK acks something not yet sent then we send an ACK. + // + // RFC793, page 37: If the connection is in a synchronized state, + // (ESTABLISHED, FIN-WAIT-1, FIN-WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK, + // TIME-WAIT), any unacceptable segment (out of window sequence number + // or unacceptable acknowledgment number) must elicit only an empty + // acknowledgment segment containing the current send-sequence number + // and an acknowledgment indicating the next sequence number expected + // to be received, and the connection remains in the same state. + // + // Just as on Linux, we do not apply this behavior when state is + // ESTABLISHED. + // Linux receive processing for all states except ESTABLISHED and + // TIME_WAIT is here where if the ACK check fails, we attempt to + // reply back with an ACK with correct seq/ack numbers. + // https://github.com/torvalds/linux/blob/v5.8/net/ipv4/tcp_input.c#L6186 + // The ESTABLISHED state processing is here where if the ACK check + // fails, we ignore the packet: + // https://github.com/torvalds/linux/blob/v5.8/net/ipv4/tcp_input.c#L5591 + if r.ep.snd.SndNxt.LessThan(s.ackNumber) { + r.ep.snd.maybeSendOutOfWindowAck(s) + return true, nil + } + + // If we are closed for reads (either due to an + // incoming FIN or the user calling shutdown(.., + // SHUT_RD) then any data past the RcvNxt should + // trigger a RST. + endDataSeq := s.sequenceNumber.Add(seqnum.Size(s.payloadSize())) + if state != StateCloseWait && rcvClosed && r.RcvNxt.LessThan(endDataSeq) { + return true, &tcpip.ErrConnectionAborted{} + } + if state == StateFinWait1 { + break + } + + // If it's a retransmission of an old data segment + // or a pure ACK then allow it. + if s.sequenceNumber.Add(s.logicalLen()).LessThanEq(r.RcvNxt) || + s.logicalLen() == 0 { + break + } + + // In FIN-WAIT2 if the socket is fully + // closed(not owned by application on our end + // then the only acceptable segment is a + // FIN. Since FIN can technically also carry + // data we verify that the segment carrying a + // FIN ends at exactly e.RcvNxt+1. + // + // From RFC793 page 25. + // + // For sequence number purposes, the SYN is + // considered to occur before the first actual + // data octet of the segment in which it occurs, + // while the FIN is considered to occur after + // the last actual data octet in a segment in + // which it occurs. + if closed && (!s.flags.Contains(header.TCPFlagFin) || s.sequenceNumber.Add(s.logicalLen()) != r.RcvNxt+1) { + return true, &tcpip.ErrConnectionAborted{} + } + } + + // We don't care about receive processing anymore if the receive side + // is closed. + // + // NOTE: We still want to permit a FIN as it's possible only our + // end has closed and the peer is yet to send a FIN. Hence we + // compare only the payload. + segEnd := s.sequenceNumber.Add(seqnum.Size(s.payloadSize())) + if rcvClosed && !segEnd.LessThanEq(r.RcvNxt) { + return true, nil + } + return false, nil +} + +// handleRcvdSegment handles TCP segments directed at the connection managed by +// r as they arrive. It is called by the protocol main loop. +// +checklocks:r.ep.mu +// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu +func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) { + state := r.ep.EndpointState() + closed := r.ep.closed + + segLen := seqnum.Size(s.payloadSize()) + segSeq := s.sequenceNumber + + // If the sequence number range is outside the acceptable range, just + // send an ACK and stop further processing of the segment. + // This is according to RFC 793, page 68. + if !r.acceptable(segSeq, segLen) { + r.ep.snd.maybeSendOutOfWindowAck(s) + return true, nil + } + + if state != StateEstablished { + drop, err := r.handleRcvdSegmentClosing(s, state, closed) + if drop || err != nil { + return drop, err + } + } + + // Store the time of the last ack. + r.lastRcvdAckTime = r.ep.stack.Clock().NowMonotonic() + + // Defer segment processing if it can't be consumed now. + if !r.consumeSegment(s, segSeq, segLen) { + if segLen > 0 || s.flags.Contains(header.TCPFlagFin) { + // We only store the segment if it's within our buffer + // size limit. + // + // Only use 75% of the receive buffer queue for + // out-of-order segments. This ensures that we always + // leave some space for the inorder segments to arrive + // allowing pending segments to be processed and + // delivered to the user. + // + // The ratio must be at least 50% (the size of rwnd) to + // leave space for retransmitted dropped packets. 51% + // would make recovery slow when there are multiple + // drops by necessitating multiple round trips. 100% + // would enable the buffer to be totally full of + // out-of-order data and stall the connection. + // + // An ideal solution is to ensure that there are at + // least N bytes free when N bytes are missing, but we + // don't have that computed at this point in the stack. + if rcvBufSize := r.ep.ops.GetReceiveBufferSize(); rcvBufSize > 0 && (r.PendingBufUsed+int(segLen)) < int(rcvBufSize-rcvBufSize/4) { + r.ep.rcvQueueMu.Lock() + r.PendingBufUsed += s.segMemSize() + r.ep.rcvQueueMu.Unlock() + s.IncRef() + heap.Push(&r.pendingRcvdSegments, s) + UpdateSACKBlocks(&r.ep.sack, segSeq, segSeq.Add(segLen), r.RcvNxt) + } + + // Immediately send an ack so that the peer knows it may + // have to retransmit. + r.ep.snd.sendAck() + } + return false, nil + } + + // Since we consumed a segment update the receiver's RTT estimate + // if required. + if segLen > 0 { + r.updateRTT() + } + + // By consuming the current segment, we may have filled a gap in the + // sequence number domain that allows pending segments to be consumed + // now. So try to do it. + for !r.closed && r.pendingRcvdSegments.Len() > 0 { + s := r.pendingRcvdSegments[0] + segLen := seqnum.Size(s.payloadSize()) + segSeq := s.sequenceNumber + + // Skip segment altogether if it has already been acknowledged. + if !segSeq.Add(segLen-1).LessThan(r.RcvNxt) && + !r.consumeSegment(s, segSeq, segLen) { + break + } + + heap.Pop(&r.pendingRcvdSegments) + r.ep.rcvQueueMu.Lock() + r.PendingBufUsed -= s.segMemSize() + r.ep.rcvQueueMu.Unlock() + s.DecRef() + } + return false, nil +} + +// handleTimeWaitSegment handles inbound segments received when the endpoint +// has entered the TIME_WAIT state. +// +checklocks:r.ep.mu +// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu +func (r *receiver) handleTimeWaitSegment(s *segment) (resetTimeWait bool, newSyn bool) { + segSeq := s.sequenceNumber + segLen := seqnum.Size(s.payloadSize()) + + // Just silently drop any RST packets in TIME_WAIT. We do not support + // TIME_WAIT assassination as a result we confirm w/ fix 1 as described + // in https://tools.ietf.org/html/rfc1337#section-3. + // + // This behavior overrides RFC793 page 70 where we transition to CLOSED + // on receiving RST, which is also default Linux behavior. + // On Linux the RST can be ignored by setting sysctl net.ipv4.tcp_rfc1337. + // + // As we do not yet support PAWS, we are being conservative in ignoring + // RSTs by default. + if s.flags.Contains(header.TCPFlagRst) { + return false, false + } + + // If it's a SYN and the sequence number is higher than any seen before + // for this connection then try and redirect it to a listening endpoint + // if available. + // + // RFC 1122: + // "When a connection is [...] on TIME-WAIT state [...] + // [a TCP] MAY accept a new SYN from the remote TCP to + // reopen the connection directly, if it: + + // (1) assigns its initial sequence number for the new + // connection to be larger than the largest sequence + // number it used on the previous connection incarnation, + // and + + // (2) returns to TIME-WAIT state if the SYN turns out + // to be an old duplicate". + if s.flags.Contains(header.TCPFlagSyn) && r.RcvNxt.LessThan(segSeq) { + return false, true + } + + // Drop the segment if it does not contain an ACK. + if !s.flags.Contains(header.TCPFlagAck) { + return false, false + } + + // Update Timestamp if required. See RFC7323, section-4.3. + if r.ep.SendTSOk && s.parsedOptions.TS { + r.ep.updateRecentTimestamp(s.parsedOptions.TSVal, r.ep.snd.MaxSentAck, segSeq) + } + + if segSeq.Add(1) == r.RcvNxt && s.flags.Contains(header.TCPFlagFin) { + // If it's a FIN-ACK then resetTimeWait and send an ACK, as it + // indicates our final ACK could have been lost. + r.ep.snd.sendAck() + return true, false + } + + // If the sequence number range is outside the acceptable range or + // carries data then just send an ACK. This is according to RFC 793, + // page 37. + // + // NOTE: In TIME_WAIT the only acceptable sequence number is RcvNxt. + if segSeq != r.RcvNxt || segLen != 0 { + r.ep.snd.sendAck() + } + return false, false +} diff --git a/pkg/tcpip/transport/tcp/rcv_queue_mutex.go b/pkg/tcpip/transport/tcp/rcv_queue_mutex.go new file mode 100644 index 0000000..e683804 --- /dev/null +++ b/pkg/tcpip/transport/tcp/rcv_queue_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type rcvQueueMutex struct { + mu sync.Mutex +} + +var rcvQueueprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var rcvQueuelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type rcvQueuelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *rcvQueueMutex) Lock() { + locking.AddGLock(rcvQueueprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *rcvQueueMutex) NestedLock(i rcvQueuelockNameIndex) { + locking.AddGLock(rcvQueueprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *rcvQueueMutex) Unlock() { + locking.DelGLock(rcvQueueprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *rcvQueueMutex) NestedUnlock(i rcvQueuelockNameIndex) { + locking.DelGLock(rcvQueueprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func rcvQueueinitLockNames() {} + +func init() { + rcvQueueinitLockNames() + rcvQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvQueueMutex{}), rcvQueuelockNames) +} diff --git a/pkg/tcpip/transport/tcp/reno.go b/pkg/tcpip/transport/tcp/reno.go new file mode 100644 index 0000000..4345376 --- /dev/null +++ b/pkg/tcpip/transport/tcp/reno.go @@ -0,0 +1,118 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "time" +) + +// renoState stores the variables related to TCP New Reno congestion +// control algorithm. +// +// +stateify savable +type renoState struct { + s *sender +} + +// newRenoCC initializes the state for the NewReno congestion control algorithm. +func newRenoCC(s *sender) *renoState { + return &renoState{s: s} +} + +// updateSlowStart will update the congestion window as per the slow-start +// algorithm used by NewReno. If after adjusting the congestion window +// we cross the SSthreshold then it will return the number of packets that +// must be consumed in congestion avoidance mode. +// +// +checklocks:r.s.ep.mu +func (r *renoState) updateSlowStart(packetsAcked int) int { + // Don't let the congestion window cross into the congestion + // avoidance range. + newcwnd := r.s.SndCwnd + packetsAcked + if newcwnd >= r.s.Ssthresh { + newcwnd = r.s.Ssthresh + r.s.SndCAAckCount = 0 + } + + packetsAcked -= newcwnd - r.s.SndCwnd + r.s.SndCwnd = newcwnd + return packetsAcked +} + +// updateCongestionAvoidance will update congestion window in congestion +// avoidance mode as described in RFC5681 section 3.1 +// +// +checklocks:r.s.ep.mu +func (r *renoState) updateCongestionAvoidance(packetsAcked int) { + // Consume the packets in congestion avoidance mode. + r.s.SndCAAckCount += packetsAcked + if r.s.SndCAAckCount >= r.s.SndCwnd { + r.s.SndCwnd += r.s.SndCAAckCount / r.s.SndCwnd + r.s.SndCAAckCount = r.s.SndCAAckCount % r.s.SndCwnd + } +} + +// reduceSlowStartThreshold reduces the slow-start threshold per RFC 5681, +// page 6, eq. 4. It is called when we detect congestion in the network. +// +// +checklocks:r.s.ep.mu +func (r *renoState) reduceSlowStartThreshold() { + r.s.Ssthresh = r.s.Outstanding / 2 + if r.s.Ssthresh < 2 { + r.s.Ssthresh = 2 + } +} + +// Update updates the congestion state based on the number of packets that +// were acknowledged. +// Update implements congestionControl.Update. +// +// +checklocks:r.s.ep.mu +func (r *renoState) Update(packetsAcked int, _ time.Duration) { + if r.s.SndCwnd < r.s.Ssthresh { + packetsAcked = r.updateSlowStart(packetsAcked) + if packetsAcked == 0 { + return + } + } + r.updateCongestionAvoidance(packetsAcked) +} + +// HandleLossDetected implements congestionControl.HandleLossDetected. +// +// +checklocks:r.s.ep.mu +func (r *renoState) HandleLossDetected() { + // A retransmit was triggered due to nDupAckThreshold or when RACK + // detected loss. Reduce our slow start threshold. + r.reduceSlowStartThreshold() +} + +// HandleRTOExpired implements congestionControl.HandleRTOExpired. +// +// +checklocks:r.s.ep.mu +func (r *renoState) HandleRTOExpired() { + // We lost a packet, so reduce ssthresh. + r.reduceSlowStartThreshold() + + // Reduce the congestion window to 1, i.e., enter slow-start. Per + // RFC 5681, page 7, we must use 1 regardless of the value of the + // initial congestion window. + r.s.SndCwnd = 1 +} + +// PostRecovery implements congestionControl.PostRecovery. +func (r *renoState) PostRecovery() { + // noop. +} diff --git a/pkg/tcpip/transport/tcp/reno_recovery.go b/pkg/tcpip/transport/tcp/reno_recovery.go new file mode 100644 index 0000000..e387dfa --- /dev/null +++ b/pkg/tcpip/transport/tcp/reno_recovery.go @@ -0,0 +1,68 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +// renoRecovery stores the variables related to TCP Reno loss recovery +// algorithm. +// +// +stateify savable +type renoRecovery struct { + s *sender +} + +func newRenoRecovery(s *sender) *renoRecovery { + return &renoRecovery{s: s} +} + +// +checklocks:rr.s.ep.mu +func (rr *renoRecovery) DoRecovery(rcvdSeg *segment, fastRetransmit bool) { + ack := rcvdSeg.ackNumber + snd := rr.s + + // We are in fast recovery mode. Ignore the ack if it's out of range. + if !ack.InRange(snd.SndUna, snd.SndNxt+1) { + return + } + + // Don't count this as a duplicate if it is carrying data or + // updating the window. + if rcvdSeg.logicalLen() != 0 || snd.SndWnd != rcvdSeg.window { + return + } + + // Inflate the congestion window if we're getting duplicate acks + // for the packet we retransmitted. + if !fastRetransmit && ack == snd.FastRecovery.First { + // We received a dup, inflate the congestion window by 1 packet + // if we're not at the max yet. Only inflate the window if + // regular FastRecovery is in use, RFC6675 does not require + // inflating cwnd on duplicate ACKs. + if snd.SndCwnd < snd.FastRecovery.MaxCwnd { + snd.SndCwnd++ + } + return + } + + // A partial ack was received. Retransmit this packet and remember it + // so that we don't retransmit it again. + // + // We don't inflate the window because we're putting the same packet + // back onto the wire. + // + // N.B. The retransmit timer will be reset by the caller. + snd.FastRecovery.First = ack + snd.DupAckCount = 0 + snd.resendSegment() +} diff --git a/pkg/tcpip/transport/tcp/rtt_mutex.go b/pkg/tcpip/transport/tcp/rtt_mutex.go new file mode 100644 index 0000000..4ce9ee8 --- /dev/null +++ b/pkg/tcpip/transport/tcp/rtt_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type rttMutex struct { + mu sync.Mutex +} + +var rttprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var rttlockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type rttlockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *rttMutex) Lock() { + locking.AddGLock(rttprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *rttMutex) NestedLock(i rttlockNameIndex) { + locking.AddGLock(rttprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *rttMutex) Unlock() { + locking.DelGLock(rttprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *rttMutex) NestedUnlock(i rttlockNameIndex) { + locking.DelGLock(rttprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func rttinitLockNames() {} + +func init() { + rttinitLockNames() + rttprefixIndex = locking.NewMutexClass(reflect.TypeOf(rttMutex{}), rttlockNames) +} diff --git a/pkg/tcpip/transport/tcp/sack.go b/pkg/tcpip/transport/tcp/sack.go new file mode 100644 index 0000000..a257af1 --- /dev/null +++ b/pkg/tcpip/transport/tcp/sack.go @@ -0,0 +1,105 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +const ( + // MaxSACKBlocks is the maximum number of SACK blocks stored + // at receiver side. + MaxSACKBlocks = 6 +) + +// UpdateSACKBlocks updates the list of SACK blocks to include the segment +// specified by segStart->segEnd. If the segment happens to be an out of order +// delivery then the first block in the sack.blocks always includes the +// segment identified by segStart->segEnd. +func UpdateSACKBlocks(sack *SACKInfo, segStart seqnum.Value, segEnd seqnum.Value, rcvNxt seqnum.Value) { + newSB := header.SACKBlock{Start: segStart, End: segEnd} + + // Ignore any invalid SACK blocks or blocks that are before rcvNxt as + // those bytes have already been acked. + if newSB.End.LessThanEq(newSB.Start) || newSB.End.LessThan(rcvNxt) { + return + } + + if sack.NumBlocks == 0 { + sack.Blocks[0] = newSB + sack.NumBlocks = 1 + return + } + n := 0 + for i := 0; i < sack.NumBlocks; i++ { + start, end := sack.Blocks[i].Start, sack.Blocks[i].End + if end.LessThanEq(rcvNxt) { + // Discard any sack blocks that are before rcvNxt as + // those have already been acked. + continue + } + if newSB.Start.LessThanEq(end) && start.LessThanEq(newSB.End) { + // Merge this SACK block into newSB and discard this SACK + // block. + if start.LessThan(newSB.Start) { + newSB.Start = start + } + if newSB.End.LessThan(end) { + newSB.End = end + } + } else { + // Save this block. + sack.Blocks[n] = sack.Blocks[i] + n++ + } + } + if rcvNxt.LessThan(newSB.Start) { + // If this was an out of order segment then make sure that the + // first SACK block is the one that includes the segment. + // + // See the first bullet point in + // https://tools.ietf.org/html/rfc2018#section-4 + if n == MaxSACKBlocks { + // If the number of SACK blocks is equal to + // MaxSACKBlocks then discard the last SACK block. + n-- + } + for i := n - 1; i >= 0; i-- { + sack.Blocks[i+1] = sack.Blocks[i] + } + sack.Blocks[0] = newSB + n++ + } + sack.NumBlocks = n +} + +// TrimSACKBlockList updates the sack block list by removing/modifying any block +// where start is < rcvNxt. +func TrimSACKBlockList(sack *SACKInfo, rcvNxt seqnum.Value) { + n := 0 + for i := 0; i < sack.NumBlocks; i++ { + if sack.Blocks[i].End.LessThanEq(rcvNxt) { + continue + } + if sack.Blocks[i].Start.LessThan(rcvNxt) { + // Shrink this SACK block. + sack.Blocks[i].Start = rcvNxt + } + sack.Blocks[n] = sack.Blocks[i] + n++ + } + sack.NumBlocks = n +} diff --git a/pkg/tcpip/transport/tcp/sack_recovery.go b/pkg/tcpip/transport/tcp/sack_recovery.go new file mode 100644 index 0000000..4a73cae --- /dev/null +++ b/pkg/tcpip/transport/tcp/sack_recovery.go @@ -0,0 +1,122 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + +// sackRecovery stores the variables related to TCP SACK loss recovery +// algorithm. +// +// +stateify savable +type sackRecovery struct { + s *sender +} + +func newSACKRecovery(s *sender) *sackRecovery { + return &sackRecovery{s: s} +} + +// handleSACKRecovery implements the loss recovery phase as described in RFC6675 +// section 5, step C. +// +checklocks:sr.s.ep.mu +func (sr *sackRecovery) handleSACKRecovery(limit int, end seqnum.Value) (dataSent bool) { + snd := sr.s + snd.SetPipe() + + if smss := int(snd.ep.scoreboard.SMSS()); limit > smss { + // Cap segment size limit to s.smss as SACK recovery requires + // that all retransmissions or new segments send during recovery + // be of <= SMSS. + limit = smss + } + + nextSegHint := snd.writeList.Front() + for snd.Outstanding < snd.SndCwnd { + var nextSeg *segment + var rescueRtx bool + nextSeg, nextSegHint, rescueRtx = snd.NextSeg(nextSegHint) + if nextSeg == nil { + return dataSent + } + if !snd.isAssignedSequenceNumber(nextSeg) || snd.SndNxt.LessThanEq(nextSeg.sequenceNumber) { + // New data being sent. + + // Step C.3 described below is handled by + // maybeSendSegment which increments sndNxt when + // a segment is transmitted. + // + // Step C.3 "If any of the data octets sent in + // (C.1) are above HighData, HighData must be + // updated to reflect the transmission of + // previously unsent data." + // + // We pass s.smss as the limit as the Step 2) requires that + // new data sent should be of size s.smss or less. + if sent := snd.maybeSendSegment(nextSeg, limit, end); !sent { + return dataSent + } + dataSent = true + snd.Outstanding++ + snd.updateWriteNext(nextSeg.Next()) + continue + } + + // Now handle the retransmission case where we matched either step 1,3 or 4 + // of the NextSeg algorithm. + // RFC 6675, Step C.4. + // + // "The estimate of the amount of data outstanding in the network + // must be updated by incrementing pipe by the number of octets + // transmitted in (C.1)." + snd.Outstanding++ + dataSent = true + snd.sendSegment(nextSeg) + + segEnd := nextSeg.sequenceNumber.Add(nextSeg.logicalLen()) + if rescueRtx { + // We do the last part of rule (4) of NextSeg here to update + // RescueRxt as until this point we don't know if we are going + // to use the rescue transmission. + snd.FastRecovery.RescueRxt = snd.FastRecovery.Last + } else { + // RFC 6675, Step C.2 + // + // "If any of the data octets sent in (C.1) are below + // HighData, HighRxt MUST be set to the highest sequence + // number of the retransmitted segment unless NextSeg () + // rule (4) was invoked for this retransmission." + snd.FastRecovery.HighRxt = segEnd - 1 + } + } + return dataSent +} + +// +checklocks:sr.s.ep.mu +func (sr *sackRecovery) DoRecovery(rcvdSeg *segment, fastRetransmit bool) { + snd := sr.s + if fastRetransmit { + snd.resendSegment() + } + + // We are in fast recovery mode. Ignore the ack if it's out of range. + if ack := rcvdSeg.ackNumber; !ack.InRange(snd.SndUna, snd.SndNxt+1) { + return + } + + // RFC 6675 recovery algorithm step C 1-5. + end := snd.SndUna.Add(snd.SndWnd) + dataSent := sr.handleSACKRecovery(snd.MaxPayloadSize, end) + snd.postXmit(dataSent, true /* shouldScheduleProbe */) +} diff --git a/pkg/tcpip/transport/tcp/sack_scoreboard.go b/pkg/tcpip/transport/tcp/sack_scoreboard.go new file mode 100644 index 0000000..fa306a7 --- /dev/null +++ b/pkg/tcpip/transport/tcp/sack_scoreboard.go @@ -0,0 +1,306 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "fmt" + "strings" + + "github.com/google/btree" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +const ( + // maxSACKBlocks is the maximum number of distinct SACKBlocks the + // scoreboard will track. Once there are 100 distinct blocks, new + // insertions will fail. + maxSACKBlocks = 100 + + // defaultBtreeDegree is set to 2 as btree.New(2) results in a 2-3-4 + // tree. + defaultBtreeDegree = 2 +) + +// SACKScoreboard stores a set of disjoint SACK ranges. +// +// +stateify savable +type SACKScoreboard struct { + // smss is defined in RFC5681 as following: + // + // The SMSS is the size of the largest segment that the sender can + // transmit. This value can be based on the maximum transmission unit + // of the network, the path MTU discovery [RFC1191, RFC4821] algorithm, + // RMSS (see next item), or other factors. The size does not include + // the TCP/IP headers and options. + smss uint16 + maxSACKED seqnum.Value + sacked seqnum.Size `state:"nosave"` + ranges *btree.BTree `state:"nosave"` +} + +// NewSACKScoreboard returns a new SACK Scoreboard. +func NewSACKScoreboard(smss uint16, iss seqnum.Value) *SACKScoreboard { + return &SACKScoreboard{ + smss: smss, + ranges: btree.New(defaultBtreeDegree), + maxSACKED: iss, + } +} + +// Reset erases all known range information from the SACK scoreboard. +func (s *SACKScoreboard) Reset() { + s.ranges = btree.New(defaultBtreeDegree) + s.sacked = 0 +} + +// Insert inserts/merges the provided SACKBlock into the scoreboard. +func (s *SACKScoreboard) Insert(r header.SACKBlock) { + if s.ranges.Len() >= maxSACKBlocks { + return + } + + // Check if we can merge the new range with a range before or after it. + var toDelete []btree.Item + if s.maxSACKED.LessThan(r.End - 1) { + s.maxSACKED = r.End - 1 + } + s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool { + if i == r { + return true + } + sacked := i.(header.SACKBlock) + // There is a hole between these two SACK blocks, so we can't + // merge anymore. + if r.End.LessThan(sacked.Start) { + return false + } + // There is some overlap at this point, merge the blocks and + // delete the other one. + // + // ----sS--------sE + // r.S---------------rE + // -------sE + if sacked.End.LessThan(r.End) { + // sacked is contained in the newly inserted range. + // Delete this block. + toDelete = append(toDelete, i) + return true + } + // sacked covers a range past end of the newly inserted + // block. + r.End = sacked.End + toDelete = append(toDelete, i) + return true + }) + + s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { + if i == r { + return true + } + sacked := i.(header.SACKBlock) + // sA------sE + // rA----rE + if sacked.End.LessThan(r.Start) { + return false + } + // The previous range extends into the current block. Merge it + // into the newly inserted range and delete the other one. + // + // <-rA---rE----<---rE---> + // sA--------------sE + r.Start = sacked.Start + // Extend r to cover sacked if sacked extends past r. + if r.End.LessThan(sacked.End) { + r.End = sacked.End + } + toDelete = append(toDelete, i) + return true + }) + for _, i := range toDelete { + if sb := s.ranges.Delete(i); sb != nil { + sb := i.(header.SACKBlock) + s.sacked -= sb.Start.Size(sb.End) + } + } + + replaced := s.ranges.ReplaceOrInsert(r) + if replaced == nil { + s.sacked += r.Start.Size(r.End) + } +} + +// IsSACKED returns true if the a given range of sequence numbers denoted by r +// are already covered by SACK information in the scoreboard. +func (s *SACKScoreboard) IsSACKED(r header.SACKBlock) bool { + if s.Empty() { + return false + } + + found := false + s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { + sacked := i.(header.SACKBlock) + if sacked.End.LessThan(r.Start) { + return false + } + if sacked.Contains(r) { + found = true + return false + } + return true + }) + return found +} + +// String returns human-readable state of the scoreboard structure. +func (s *SACKScoreboard) String() string { + var str strings.Builder + str.WriteString("SACKScoreboard: {") + s.ranges.Ascend(func(i btree.Item) bool { + str.WriteString(fmt.Sprintf("%v,", i)) + return true + }) + str.WriteString("}\n") + return str.String() +} + +// Delete removes all SACK information prior to seq. +func (s *SACKScoreboard) Delete(seq seqnum.Value) { + if s.Empty() { + return + } + toDelete := []btree.Item{} + toInsert := []btree.Item{} + r := header.SACKBlock{seq, seq.Add(1)} + s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { + if i == r { + return true + } + sb := i.(header.SACKBlock) + toDelete = append(toDelete, i) + if sb.End.LessThanEq(seq) { + s.sacked -= sb.Start.Size(sb.End) + } else { + newSB := header.SACKBlock{seq, sb.End} + toInsert = append(toInsert, newSB) + s.sacked -= sb.Start.Size(seq) + } + return true + }) + for _, sb := range toDelete { + s.ranges.Delete(sb) + } + for _, sb := range toInsert { + s.ranges.ReplaceOrInsert(sb) + } +} + +// Copy provides a copy of the SACK scoreboard. +func (s *SACKScoreboard) Copy() (sackBlocks []header.SACKBlock, maxSACKED seqnum.Value) { + s.ranges.Ascend(func(i btree.Item) bool { + sackBlocks = append(sackBlocks, i.(header.SACKBlock)) + return true + }) + return sackBlocks, s.maxSACKED +} + +// IsRangeLost implements the IsLost(SeqNum) operation defined in RFC 6675 +// section 4 but operates on a range of sequence numbers and returns true if +// there are at least nDupAckThreshold SACK blocks greater than the range being +// checked or if at least (nDupAckThreshold-1)*s.smss bytes have been SACKED +// with sequence numbers greater than the block being checked. +func (s *SACKScoreboard) IsRangeLost(r header.SACKBlock) bool { + if s.Empty() { + return false + } + nDupSACK := 0 + nDupSACKBytes := seqnum.Size(0) + isLost := false + + // We need to check if the immediate lower (if any) sacked + // range contains or partially overlaps with r. + searchMore := true + s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { + sacked := i.(header.SACKBlock) + if sacked.Contains(r) { + searchMore = false + return false + } + if sacked.End.LessThanEq(r.Start) { + // all sequence numbers covered by sacked are below + // r so we continue searching. + return false + } + // There is a partial overlap. In this case we r.Start is + // between sacked.Start & sacked.End and r.End extends beyond + // sacked.End. + // Move r.Start to sacked.End and continuing searching blocks + // above r.Start. + r.Start = sacked.End + return false + }) + + if !searchMore { + return isLost + } + + s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool { + sacked := i.(header.SACKBlock) + if sacked.Contains(r) { + return false + } + nDupSACKBytes += sacked.Start.Size(sacked.End) + nDupSACK++ + if nDupSACK >= nDupAckThreshold || nDupSACKBytes >= seqnum.Size((nDupAckThreshold-1)*s.smss) { + isLost = true + return false + } + return true + }) + return isLost +} + +// IsLost implements the IsLost(SeqNum) operation defined in RFC3517 section +// 4. +// +// This routine returns whether the given sequence number is considered to be +// lost. The routine returns true when either nDupAckThreshold discontiguous +// SACKed sequences have arrived above 'SeqNum' or (nDupAckThreshold * SMSS) +// bytes with sequence numbers greater than 'SeqNum' have been SACKed. +// Otherwise, the routine returns false. +func (s *SACKScoreboard) IsLost(seq seqnum.Value) bool { + return s.IsRangeLost(header.SACKBlock{seq, seq.Add(1)}) +} + +// Empty returns true if the SACK scoreboard has no entries, false otherwise. +func (s *SACKScoreboard) Empty() bool { + return s.ranges.Len() == 0 +} + +// Sacked returns the current number of bytes held in the SACK scoreboard. +func (s *SACKScoreboard) Sacked() seqnum.Size { + return s.sacked +} + +// MaxSACKED returns the highest sequence number ever inserted in the SACK +// scoreboard. +func (s *SACKScoreboard) MaxSACKED() seqnum.Value { + return s.maxSACKED +} + +// SMSS returns the sender's MSS as held by the SACK scoreboard. +func (s *SACKScoreboard) SMSS() uint16 { + return s.smss +} diff --git a/pkg/tcpip/transport/tcp/segment.go b/pkg/tcpip/transport/tcp/segment.go new file mode 100644 index 0000000..863b1e3 --- /dev/null +++ b/pkg/tcpip/transport/tcp/segment.go @@ -0,0 +1,251 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "fmt" + "io" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// queueFlags are used to indicate which queue of an endpoint a particular segment +// belongs to. This is used to track memory accounting correctly. +type queueFlags uint8 + +const ( + // SegOverheadSize is the size of an empty seg in memory including packet + // buffer overhead. It is advised to use SegOverheadSize instead of segSize + // in all cases where accounting for segment memory overhead is important. + SegOverheadSize = segSize + stack.PacketBufferStructSize + header.IPv4MaximumHeaderSize + + recvQ queueFlags = 1 << iota + sendQ +) + +var segmentPool = sync.Pool{ + New: func() any { + return &segment{} + }, +} + +// segment represents a TCP segment. It holds the payload and parsed TCP segment +// information, and can be added to intrusive lists. +// segment is mostly immutable, the only field allowed to change is data. +// +// +stateify savable +type segment struct { + segmentEntry + segmentRefs + + ep *Endpoint + qFlags queueFlags + id stack.TransportEndpointID `state:"manual"` + + pkt *stack.PacketBuffer + + sequenceNumber seqnum.Value + ackNumber seqnum.Value + flags header.TCPFlags + window seqnum.Size + // csum is only populated for received segments. + csum uint16 + // csumValid is true if the csum in the received segment is valid. + csumValid bool + + // parsedOptions stores the parsed values from the options in the segment. + parsedOptions header.TCPOptions + options []byte `state:".([]byte)"` + hasNewSACKInfo bool + rcvdTime tcpip.MonotonicTime + // xmitTime is the last transmit time of this segment. + xmitTime tcpip.MonotonicTime + xmitCount uint32 + + // acked indicates if the segment has already been SACKed. + acked bool + + // dataMemSize is the memory used by pkt initially. The value is used for + // memory accounting in the receive buffer instead of pkt.MemSize() because + // packet contents can be modified, so relying on the computed memory size + // to "free" reserved bytes could leak memory in the receiver. + dataMemSize int + + // lost indicates if the segment is marked as lost by RACK. + lost bool +} + +func newIncomingSegment(id stack.TransportEndpointID, clock tcpip.Clock, pkt *stack.PacketBuffer) (*segment, error) { + hdr := header.TCP(pkt.TransportHeader().Slice()) + var srcAddr tcpip.Address + var dstAddr tcpip.Address + switch netProto := pkt.NetworkProtocolNumber; netProto { + case header.IPv4ProtocolNumber: + hdr := header.IPv4(pkt.NetworkHeader().Slice()) + srcAddr = hdr.SourceAddress() + dstAddr = hdr.DestinationAddress() + case header.IPv6ProtocolNumber: + hdr := header.IPv6(pkt.NetworkHeader().Slice()) + srcAddr = hdr.SourceAddress() + dstAddr = hdr.DestinationAddress() + default: + panic(fmt.Sprintf("unknown network protocol number %d", netProto)) + } + + csum, csumValid, ok := header.TCPValid( + hdr, + func() uint16 { return pkt.Data().Checksum() }, + uint16(pkt.Data().Size()), + srcAddr, + dstAddr, + pkt.RXChecksumValidated) + if !ok { + return nil, fmt.Errorf("header data offset does not respect size constraints: %d < offset < %d, got offset=%d", header.TCPMinimumSize, len(hdr), hdr.DataOffset()) + } + + s := newSegment() + s.id = id + s.options = hdr[header.TCPMinimumSize:] + s.parsedOptions = header.ParseTCPOptions(hdr[header.TCPMinimumSize:]) + s.sequenceNumber = seqnum.Value(hdr.SequenceNumber()) + s.ackNumber = seqnum.Value(hdr.AckNumber()) + s.flags = hdr.Flags() + s.window = seqnum.Size(hdr.WindowSize()) + s.rcvdTime = clock.NowMonotonic() + s.dataMemSize = pkt.MemSize() + s.pkt = pkt.Clone() + s.csumValid = csumValid + + if !s.pkt.RXChecksumValidated { + s.csum = csum + } + return s, nil +} + +func newOutgoingSegment(id stack.TransportEndpointID, clock tcpip.Clock, buf buffer.Buffer) *segment { + s := newSegment() + s.id = id + s.rcvdTime = clock.NowMonotonic() + s.pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf}) + s.dataMemSize = s.pkt.MemSize() + return s +} + +func (s *segment) clone() *segment { + t := newSegment() + t.id = s.id + t.sequenceNumber = s.sequenceNumber + t.ackNumber = s.ackNumber + t.flags = s.flags + t.window = s.window + t.rcvdTime = s.rcvdTime + t.xmitTime = s.xmitTime + t.xmitCount = s.xmitCount + t.ep = s.ep + t.qFlags = s.qFlags + t.dataMemSize = s.dataMemSize + t.pkt = s.pkt.Clone() + return t +} + +func newSegment() *segment { + s := segmentPool.Get().(*segment) + *s = segment{} + s.InitRefs() + return s +} + +// merge merges data in oth and clears oth. +func (s *segment) merge(oth *segment) { + s.pkt.Data().Merge(oth.pkt.Data()) + s.dataMemSize = s.pkt.MemSize() + oth.dataMemSize = oth.pkt.MemSize() +} + +// setOwner sets the owning endpoint for this segment. Its required +// to be called to ensure memory accounting for receive/send buffer +// queues is done properly. +func (s *segment) setOwner(ep *Endpoint, qFlags queueFlags) { + switch qFlags { + case recvQ: + ep.updateReceiveMemUsed(s.segMemSize()) + case sendQ: + // no memory account for sendQ yet. + default: + panic(fmt.Sprintf("unexpected queue flag %b", qFlags)) + } + s.ep = ep + s.qFlags = qFlags +} + +func (s *segment) DecRef() { + s.segmentRefs.DecRef(func() { + if s.ep != nil { + switch s.qFlags { + case recvQ: + s.ep.updateReceiveMemUsed(-s.segMemSize()) + case sendQ: + // no memory accounting for sendQ yet. + default: + panic(fmt.Sprintf("unexpected queue flag %b set for segment", s.qFlags)) + } + } + s.pkt.DecRef() + s.pkt = nil + segmentPool.Put(s) + }) +} + +// logicalLen is the segment length in the sequence number space. It's defined +// as the data length plus one for each of the SYN and FIN bits set. +func (s *segment) logicalLen() seqnum.Size { + l := seqnum.Size(s.payloadSize()) + if s.flags.Contains(header.TCPFlagSyn) { + l++ + } + if s.flags.Contains(header.TCPFlagFin) { + l++ + } + return l +} + +// payloadSize is the size of s.data. +func (s *segment) payloadSize() int { + return s.pkt.Data().Size() +} + +// segMemSize is the amount of memory used to hold the segment data and +// the associated metadata. +func (s *segment) segMemSize() int { + return segSize + s.dataMemSize +} + +// sackBlock returns a header.SACKBlock that represents this segment. +func (s *segment) sackBlock() header.SACKBlock { + return header.SACKBlock{Start: s.sequenceNumber, End: s.sequenceNumber.Add(s.logicalLen())} +} + +func (s *segment) TrimFront(ackLeft seqnum.Size) { + s.pkt.Data().TrimFront(int(ackLeft)) +} + +func (s *segment) ReadTo(dst io.Writer, peek bool) (int, error) { + return s.pkt.Data().ReadTo(dst, peek) +} diff --git a/pkg/tcpip/transport/tcp/segment_heap.go b/pkg/tcpip/transport/tcp/segment_heap.go new file mode 100644 index 0000000..33dcc09 --- /dev/null +++ b/pkg/tcpip/transport/tcp/segment_heap.go @@ -0,0 +1,51 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import "container/heap" + +type segmentHeap []*segment + +var _ heap.Interface = (*segmentHeap)(nil) + +// Len returns the length of h. +func (h *segmentHeap) Len() int { + return len(*h) +} + +// Less determines whether the i-th element of h is less than the j-th element. +func (h *segmentHeap) Less(i, j int) bool { + return (*h)[i].sequenceNumber.LessThan((*h)[j].sequenceNumber) +} + +// Swap swaps the i-th and j-th elements of h. +func (h *segmentHeap) Swap(i, j int) { + (*h)[i], (*h)[j] = (*h)[j], (*h)[i] +} + +// Push adds x as the last element of h. +func (h *segmentHeap) Push(x any) { + *h = append(*h, x.(*segment)) +} + +// Pop removes the last element of h and returns it. +func (h *segmentHeap) Pop() any { + old := *h + n := len(old) + x := old[n-1] + old[n-1] = nil + *h = old[:n-1] + return x +} diff --git a/pkg/tcpip/transport/tcp/segment_queue.go b/pkg/tcpip/transport/tcp/segment_queue.go new file mode 100644 index 0000000..c7669b0 --- /dev/null +++ b/pkg/tcpip/transport/tcp/segment_queue.go @@ -0,0 +1,99 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +// segmentQueue is a bounded, thread-safe queue of TCP segments. +// +// +stateify savable +type segmentQueue struct { + mu segmentQueueMutex `state:"nosave"` + list segmentList `state:"wait"` + ep *Endpoint + frozen bool +} + +// emptyLocked determines if the queue is empty. +// Preconditions: q.mu must be held. +func (q *segmentQueue) emptyLocked() bool { + return q.list.Empty() +} + +// empty determines if the queue is empty. +func (q *segmentQueue) empty() bool { + q.mu.Lock() + defer q.mu.Unlock() + return q.emptyLocked() +} + +// enqueue adds the given segment to the queue. +// +// Returns true when the segment is successfully added to the queue, in which +// case ownership of the reference is transferred to the queue. And returns +// false if the queue is full, in which case ownership is retained by the +// caller. +func (q *segmentQueue) enqueue(s *segment) bool { + // q.ep.receiveBufferParams() must be called without holding q.mu to + // avoid lock order inversion. + bufSz := q.ep.ops.GetReceiveBufferSize() + used := q.ep.receiveMemUsed() + + q.mu.Lock() + defer q.mu.Unlock() + + // Allow zero sized segments (ACK/FIN/RSTs etc even if the segment queue + // is currently full). + allow := (used <= int(bufSz) || s.payloadSize() == 0) && !q.frozen + + if allow { + s.IncRef() + q.list.PushBack(s) + // Set the owner now that the endpoint owns the segment. + s.setOwner(q.ep, recvQ) + } + + return allow +} + +// dequeue removes and returns the next segment from queue, if one exists. +// Ownership is transferred to the caller, who is responsible for decrementing +// the ref count when done. +func (q *segmentQueue) dequeue() *segment { + q.mu.Lock() + defer q.mu.Unlock() + + s := q.list.Front() + if s != nil { + q.list.Remove(s) + } + + return s +} + +// freeze prevents any more segments from being added to the queue. i.e all +// future segmentQueue.enqueue will return false and not add the segment to the +// queue till the queue is unfroze with a corresponding segmentQueue.thaw call. +func (q *segmentQueue) freeze() { + q.mu.Lock() + defer q.mu.Unlock() + q.frozen = true +} + +// thaw unfreezes a previously frozen queue using segmentQueue.freeze() and +// allows new segments to be queued again. +func (q *segmentQueue) thaw() { + q.mu.Lock() + defer q.mu.Unlock() + q.frozen = false +} diff --git a/pkg/tcpip/transport/tcp/segment_queue_mutex.go b/pkg/tcpip/transport/tcp/segment_queue_mutex.go new file mode 100644 index 0000000..e1e5dd4 --- /dev/null +++ b/pkg/tcpip/transport/tcp/segment_queue_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type segmentQueueMutex struct { + mu sync.Mutex +} + +var segmentQueueprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var segmentQueuelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type segmentQueuelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *segmentQueueMutex) Lock() { + locking.AddGLock(segmentQueueprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *segmentQueueMutex) NestedLock(i segmentQueuelockNameIndex) { + locking.AddGLock(segmentQueueprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *segmentQueueMutex) Unlock() { + locking.DelGLock(segmentQueueprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *segmentQueueMutex) NestedUnlock(i segmentQueuelockNameIndex) { + locking.DelGLock(segmentQueueprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func segmentQueueinitLockNames() {} + +func init() { + segmentQueueinitLockNames() + segmentQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(segmentQueueMutex{}), segmentQueuelockNames) +} diff --git a/pkg/tcpip/transport/tcp/segment_state.go b/pkg/tcpip/transport/tcp/segment_state.go new file mode 100644 index 0000000..76ab562 --- /dev/null +++ b/pkg/tcpip/transport/tcp/segment_state.go @@ -0,0 +1,35 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "context" +) + +// saveOptions is invoked by stateify. +func (s *segment) saveOptions() []byte { + // We cannot save s.options directly as it may point to s.data's trimmed + // tail, which is not allowed by state framework (in-struct pointer). + b := make([]byte, 0, cap(s.options)) + return append(b, s.options...) +} + +// loadOptions is invoked by stateify. +func (s *segment) loadOptions(_ context.Context, options []byte) { + // NOTE: We cannot point s.options back into s.data's trimmed tail. But + // it is OK as they do not need to aliased. Plus, options is already + // allocated so there is no cost here. + s.options = options +} diff --git a/pkg/tcpip/transport/tcp/segment_unsafe.go b/pkg/tcpip/transport/tcp/segment_unsafe.go new file mode 100644 index 0000000..0ab7b8f --- /dev/null +++ b/pkg/tcpip/transport/tcp/segment_unsafe.go @@ -0,0 +1,23 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "unsafe" +) + +const ( + segSize = int(unsafe.Sizeof(segment{})) +) diff --git a/pkg/tcpip/transport/tcp/snd.go b/pkg/tcpip/transport/tcp/snd.go new file mode 100644 index 0000000..ee29b87 --- /dev/null +++ b/pkg/tcpip/transport/tcp/snd.go @@ -0,0 +1,1905 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "fmt" + "math" + "sort" + "time" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // MinRTO is the minimum allowed value for the retransmit timeout. + MinRTO = 200 * time.Millisecond + + // MaxRTO is the maximum allowed value for the retransmit timeout. + MaxRTO = 120 * time.Second + + // MinSRTT is the minimum allowed value for smoothed RTT. + MinSRTT = 1 * time.Millisecond + + // InitialCwnd is the initial congestion window. + InitialCwnd = 10 + + // nDupAckThreshold is the number of duplicate ACK's required + // before fast-retransmit is entered. + nDupAckThreshold = 3 + + // MaxRetries is the maximum number of probe retries sender does + // before timing out the connection. + // Linux default TCP_RETR2, net.ipv4.tcp_retries2. + MaxRetries = 15 + + // InitialSsthresh is the the maximum int value, which depends on the + // platform. + InitialSsthresh = math.MaxInt + + // unknownRTT is used to indicate to congestion control algorithms that we + // were unable to measure the round-trip time when processing ACKs. + // Algorithms (such as HyStart) that use the round-trip time should ignore + // such Updates. + unknownRTT = time.Duration(-1) +) + +// congestionControl is an interface that must be implemented by any supported +// congestion control algorithm. +type congestionControl interface { + // HandleLossDetected is invoked when the loss is detected by RACK or + // sender.dupAckCount >= nDupAckThreshold just before entering fast + // retransmit. + HandleLossDetected() + + // HandleRTOExpired is invoked when the retransmit timer expires. + HandleRTOExpired() + + // Update is invoked when processing inbound acks. It's passed the + // number of packet's that were acked by the most recent cumulative + // acknowledgement. rtt is the round-trip time, or is set to unknownRTT + // (above) to indicate the time is unknown. + Update(packetsAcked int, rtt time.Duration) + + // PostRecovery is invoked when the sender is exiting a fast retransmit/ + // recovery phase. This provides congestion control algorithms a way + // to adjust their state when exiting recovery. + PostRecovery() +} + +// lossRecovery is an interface that must be implemented by any supported +// loss recovery algorithm. +type lossRecovery interface { + // DoRecovery is invoked when loss is detected and segments need + // to be retransmitted. The cumulative or selective ACK is passed along + // with the flag which identifies whether the connection entered fast + // retransmit with this ACK and to retransmit the first unacknowledged + // segment. + DoRecovery(rcvdSeg *segment, fastRetransmit bool) +} + +// sender holds the state necessary to send TCP segments. +// +// +stateify savable +type sender struct { + // +checklocks:ep.mu + TCPSenderState + + ep *Endpoint + + // lr is the loss recovery algorithm used by the sender. + lr lossRecovery + + // firstRetransmittedSegXmitTime is the original transmit time of + // the first segment that was retransmitted due to RTO expiration. + firstRetransmittedSegXmitTime tcpip.MonotonicTime + + // zeroWindowProbing is set if the sender is currently probing + // for zero receive window. + zeroWindowProbing bool `state:"nosave"` + + // unackZeroWindowProbes is the number of unacknowledged zero + // window probes. + unackZeroWindowProbes uint32 `state:"nosave"` + + // writeNext is the next segment to write that hasn't already been + // written, i.e. the first payload starting at SND.NXT. + writeNext *segment + + // writeList holds all writable data: both unsent data and + // sent-but-unacknowledged data. Alternatively: it holds all bytes + // starting from SND.UNA. + // + // +checklocks:ep.mu + writeList protectedWriteList + + // resendTimer is used for RTOs. + resendTimer timer `state:"nosave"` + + // rtt.TCPRTTState.SRTT and rtt.TCPRTTState.RTTVar are the "smoothed + // round-trip time", and "round-trip time variation", as defined in + // section 2 of RFC 6298. + rtt rtt + + // minRTO is the minimum permitted value for sender.rto. + minRTO time.Duration + + // maxRTO is the maximum permitted value for sender.rto. + maxRTO time.Duration + + // maxRetries is the maximum permitted retransmissions. + maxRetries uint32 + + // gso is set if generic segmentation offload is enabled. + gso bool + + // state is the current state of congestion control for this endpoint. + state tcpip.CongestionControlState + + // cc is the congestion control algorithm in use for this sender. + cc congestionControl + + // rc has the fields needed for implementing RACK loss detection + // algorithm. + rc rackControl + + // reorderTimer is the timer used to retransmit the segments after RACK + // detects them as lost. + reorderTimer timer `state:"nosave"` + + // probeTimer is used to schedule PTO for RACK TLP algorithm. + probeTimer timer `state:"nosave"` + + // spuriousRecovery indicates whether the sender entered recovery + // spuriously as described in RFC3522 Section 3.2. + spuriousRecovery bool + + // retransmitTS is the timestamp at which the sender sends retransmitted + // segment after entering an RTO for the first time as described in + // RFC3522 Section 3.2. + retransmitTS uint32 + + // startCork start corking the segments. + startCork bool + + // corkTimer is used to drain the segments which are held when TCP_CORK + // option is enabled. + corkTimer timer `state:"nosave"` +} + +// protectedWriteList wraps the write list, checking for invalid state when +// segments are added or removed. +// +// TODO(b/339664055): Revert once bug is fixed. +// +// +stateify savable +type protectedWriteList struct { + writeList segmentList + set map[*segment]struct{} +} + +// Front returns the front of the write list. +func (wl *protectedWriteList) Front() *segment { + return wl.writeList.Front() +} + +// Back returns the back of the write list. +func (wl *protectedWriteList) Back() *segment { + return wl.writeList.Back() +} + +// Remove removes seg from the write list. +func (wl *protectedWriteList) Remove(seg *segment) { + if _, ok := wl.set[seg]; !ok { + panic("segment not found write list") + } + wl.writeList.Remove(seg) + delete(wl.set, seg) +} + +// PushBack pushes seg onto the back of the write list. +func (wl *protectedWriteList) PushBack(seg *segment) { + if _, ok := wl.set[seg]; ok { + panic("segment already in write list") + } + wl.writeList.PushBack(seg) + wl.set[seg] = struct{}{} +} + +// InsertAfter inserts seg after before. +func (wl *protectedWriteList) InsertAfter(before, seg *segment) { + if _, ok := wl.set[seg]; ok { + panic("segment already in write list") + } + wl.writeList.InsertAfter(before, seg) + wl.set[seg] = struct{}{} +} + +// rtt is a synchronization wrapper used to appease stateify. See the comment +// in sender, where it is used. +// +// +stateify savable +type rtt struct { + rttMutex `state:"nosave"` + + TCPRTTState +} + +// +checklocks:ep.mu +func newSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int) *sender { + // The sender MUST reduce the TCP data length to account for any IP or + // TCP options that it is including in the packets that it sends. + // See: https://tools.ietf.org/html/rfc6691#section-2 + maxPayloadSize := int(mss) - ep.maxOptionSize() + + s := &sender{ + ep: ep, + TCPSenderState: TCPSenderState{ + SndWnd: sndWnd, + SndUna: iss + 1, + SndNxt: iss + 1, + RTTMeasureSeqNum: iss + 1, + LastSendTime: ep.stack.Clock().NowMonotonic(), + MaxPayloadSize: maxPayloadSize, + MaxSentAck: irs + 1, + FastRecovery: TCPFastRecoveryState{ + // See: https://tools.ietf.org/html/rfc6582#section-3.2 Step 1. + Last: iss, + HighRxt: iss, + RescueRxt: iss, + }, + RTO: 1 * time.Second, + }, + gso: ep.gso.Type != stack.GSONone, + writeList: protectedWriteList{ + set: make(map[*segment]struct{}), + }, + } + return newSenderHelper(ep, iss, irs, sndWnd, mss, sndWndScale, maxPayloadSize, s) +} + +// newSenderHelper exists to sate checklocks. +// +// +checklocks:ep.mu +// +checklocksalias:s.ep.mu=ep.mu +func newSenderHelper(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int, maxPayloadSize int, s *sender) *sender { + if s.gso { + s.ep.gso.MSS = uint16(maxPayloadSize) + } + + s.cc = s.initCongestionControl(ep.cc) + s.lr = s.initLossRecovery() + s.rc.init(s, iss) + + // A negative sndWndScale means that no scaling is in use, otherwise we + // store the scaling value. + if sndWndScale > 0 { + s.SndWndScale = uint8(sndWndScale) + } + + s.resendTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.retransmitTimerExpired)) + s.reorderTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.rc.reorderTimerExpired)) + s.probeTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.probeTimerExpired)) + s.corkTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.corkTimerExpired)) + + s.updateMaxPayloadSize(int(ep.route.MTU()), 0) + // Initialize SACK Scoreboard after updating max payload size as we use + // the maxPayloadSize as the smss when determining if a segment is lost + // etc. + s.ep.scoreboard = NewSACKScoreboard(uint16(s.MaxPayloadSize), iss) + + // Get Stack wide config. + var minRTO tcpip.TCPMinRTOOption + if err := ep.stack.TransportProtocolOption(ProtocolNumber, &minRTO); err != nil { + panic(fmt.Sprintf("unable to get minRTO from stack: %s", err)) + } + s.minRTO = time.Duration(minRTO) + + var maxRTO tcpip.TCPMaxRTOOption + if err := ep.stack.TransportProtocolOption(ProtocolNumber, &maxRTO); err != nil { + panic(fmt.Sprintf("unable to get maxRTO from stack: %s", err)) + } + s.maxRTO = time.Duration(maxRTO) + + var maxRetries tcpip.TCPMaxRetriesOption + if err := ep.stack.TransportProtocolOption(ProtocolNumber, &maxRetries); err != nil { + panic(fmt.Sprintf("unable to get maxRetries from stack: %s", err)) + } + s.maxRetries = uint32(maxRetries) + + return s +} + +// initCongestionControl initializes the specified congestion control module and +// returns a handle to it. It also initializes the sndCwnd and sndSsThresh to +// their initial values. +// +// +checklocks:s.ep.mu +func (s *sender) initCongestionControl(congestionControlName tcpip.CongestionControlOption) congestionControl { + s.SndCwnd = InitialCwnd + s.Ssthresh = InitialSsthresh + + switch congestionControlName { + case ccCubic: + return newCubicCC(s) + case ccReno: + fallthrough + default: + return newRenoCC(s) + } +} + +// initLossRecovery initiates the loss recovery algorithm for the sender. +func (s *sender) initLossRecovery() lossRecovery { + if s.ep.SACKPermitted { + return newSACKRecovery(s) + } + return newRenoRecovery(s) +} + +// updateMaxPayloadSize updates the maximum payload size based on the given +// MTU. If this is in response to "packet too big" control packets (indicated +// by the count argument), it also reduces the number of outstanding packets and +// attempts to retransmit the first packet above the MTU size. +// +checklocks:s.ep.mu +func (s *sender) updateMaxPayloadSize(mtu, count int) { + m := mtu - header.TCPMinimumSize + + m -= s.ep.maxOptionSize() + + // We don't adjust up for now. + if m >= s.MaxPayloadSize { + return + } + + // Make sure we can transmit at least one byte. + if m <= 0 { + m = 1 + } + + oldMSS := s.MaxPayloadSize + s.MaxPayloadSize = m + if s.gso { + s.ep.gso.MSS = uint16(m) + } + + if count == 0 { + // updateMaxPayloadSize is also called when the sender is created. + // and there is no data to send in such cases. Return immediately. + return + } + + // Update the scoreboard's smss to reflect the new lowered + // maxPayloadSize. + s.ep.scoreboard.smss = uint16(m) + + s.Outstanding -= count + if s.Outstanding < 0 { + s.Outstanding = 0 + } + + // Rewind writeNext to the first segment exceeding the MTU. Do nothing + // if it is already before such a packet. + nextSeg := s.writeNext + for seg := s.writeList.Front(); seg != nil; seg = seg.Next() { + if seg == s.writeNext { + // We got to writeNext before we could find a segment + // exceeding the MTU. + break + } + + if nextSeg == s.writeNext && seg.payloadSize() > m { + // We found a segment exceeding the MTU. Rewind + // writeNext and try to retransmit it. + nextSeg = seg + } + + if s.ep.SACKPermitted && s.ep.scoreboard.IsSACKED(seg.sackBlock()) { + // Update sackedOut for new maximum payload size. + s.SackedOut -= s.pCount(seg, oldMSS) + s.SackedOut += s.pCount(seg, s.MaxPayloadSize) + } + } + + // Since we likely reduced the number of outstanding packets, we may be + // ready to send some more. + s.updateWriteNext(nextSeg) + s.sendData() +} + +// sendAck sends an ACK segment. +// +checklocks:s.ep.mu +func (s *sender) sendAck() { + s.sendEmptySegment(header.TCPFlagAck, s.SndNxt) +} + +// updateRTO updates the retransmit timeout when a new roud-trip time is +// available. This is done in accordance with section 2 of RFC 6298. +// +// +checklocks:s.ep.mu +func (s *sender) updateRTO(rtt time.Duration) { + s.rtt.Lock() + if !s.rtt.TCPRTTState.SRTTInited { + s.rtt.TCPRTTState.RTTVar = rtt / 2 + s.rtt.TCPRTTState.SRTT = rtt + s.rtt.TCPRTTState.SRTTInited = true + } else { + diff := s.rtt.TCPRTTState.SRTT - rtt + if diff < 0 { + diff = -diff + } + // Use RFC6298 standard algorithm to update TCPRTTState.RTTVar and TCPRTTState.SRTT when + // no timestamps are available. + if !s.ep.SendTSOk { + s.rtt.TCPRTTState.RTTVar = (3*s.rtt.TCPRTTState.RTTVar + diff) / 4 + s.rtt.TCPRTTState.SRTT = (7*s.rtt.TCPRTTState.SRTT + rtt) / 8 + } else { + // When we are taking RTT measurements of every ACK then + // we need to use a modified method as specified in + // https://tools.ietf.org/html/rfc7323#appendix-G + if s.Outstanding == 0 { + s.rtt.Unlock() + return + } + // Netstack measures congestion window/inflight all in + // terms of packets and not bytes. This is similar to + // how linux also does cwnd and inflight. In practice + // this approximation works as expected. + expectedSamples := math.Ceil(float64(s.Outstanding) / 2) + + // alpha & beta values are the original values as recommended in + // https://tools.ietf.org/html/rfc6298#section-2.3. + const alpha = 0.125 + const beta = 0.25 + + alphaPrime := alpha / expectedSamples + betaPrime := beta / expectedSamples + rttVar := (1-betaPrime)*s.rtt.TCPRTTState.RTTVar.Seconds() + betaPrime*diff.Seconds() + srtt := (1-alphaPrime)*s.rtt.TCPRTTState.SRTT.Seconds() + alphaPrime*rtt.Seconds() + s.rtt.TCPRTTState.RTTVar = time.Duration(rttVar * float64(time.Second)) + s.rtt.TCPRTTState.SRTT = time.Duration(srtt * float64(time.Second)) + } + } + + if s.rtt.TCPRTTState.SRTT < MinSRTT { + s.rtt.TCPRTTState.SRTT = MinSRTT + } + + s.RTO = s.rtt.TCPRTTState.SRTT + 4*s.rtt.TCPRTTState.RTTVar + s.RTTState = s.rtt.TCPRTTState + s.rtt.Unlock() + if s.RTO < s.minRTO { + s.RTO = s.minRTO + } + if s.RTO > s.maxRTO { + s.RTO = s.maxRTO + } +} + +// resendSegment resends the first unacknowledged segment. +// +checklocks:s.ep.mu +func (s *sender) resendSegment() { + // Don't use any segments we already sent to measure RTT as they may + // have been affected by packets being lost. + s.RTTMeasureSeqNum = s.SndNxt + + // Resend the segment. + if seg := s.writeList.Front(); seg != nil { + if seg.payloadSize() > s.MaxPayloadSize { + s.splitSeg(seg, s.MaxPayloadSize) + } + + // See: RFC 6675 section 5 Step 4.3 + // + // To prevent retransmission, set both the HighRXT and RescueRXT + // to the highest sequence number in the retransmitted segment. + s.FastRecovery.HighRxt = seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize())) - 1 + s.FastRecovery.RescueRxt = seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize())) - 1 + s.sendSegment(seg) + s.ep.stack.Stats().TCP.FastRetransmit.Increment() + s.ep.stats.SendErrors.FastRetransmit.Increment() + + // Run SetPipe() as per RFC 6675 section 5 Step 4.4 + s.SetPipe() + } +} + +// retransmitTimerExpired is called when the retransmit timer expires, and +// unacknowledged segments are assumed lost, and thus need to be resent. +// Returns true if the connection is still usable, or false if the connection +// is deemed lost. +// +checklocks:s.ep.mu +func (s *sender) retransmitTimerExpired() tcpip.Error { + // Check if the timer actually expired or if it's a spurious wake due + // to a previously orphaned runtime timer. + if s.resendTimer.isUninitialized() || !s.resendTimer.checkExpiration() { + return nil + } + + // Initialize the variables used to detect spurious recovery after + // entering RTO. + // + // See: https://www.rfc-editor.org/rfc/rfc3522.html#section-3.2 Step 1. + s.spuriousRecovery = false + s.retransmitTS = 0 + + // TODO(b/147297758): Band-aid fix, retransmitTimer can fire in some edge cases + // when writeList is empty. Remove this once we have a proper fix for this + // issue. + if s.writeList.Front() == nil { + return nil + } + + s.ep.stack.Stats().TCP.Timeouts.Increment() + s.ep.stats.SendErrors.Timeouts.Increment() + + // Set TLPRxtOut to false according to + // https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.6.1. + s.rc.tlpRxtOut = false + + // Give up if we've waited more than a minute since the last resend or + // if a user time out is set and we have exceeded the user specified + // timeout since the first retransmission. + uto := s.ep.userTimeout + + if s.firstRetransmittedSegXmitTime == (tcpip.MonotonicTime{}) { + // We store the original xmitTime of the segment that we are + // about to retransmit as the retransmission time. This is + // required as by the time the retransmitTimer has expired the + // segment has already been sent and unacked for the RTO at the + // time the segment was sent. + s.firstRetransmittedSegXmitTime = s.writeList.Front().xmitTime + } + + elapsed := s.ep.stack.Clock().NowMonotonic().Sub(s.firstRetransmittedSegXmitTime) + remaining := s.maxRTO + if uto != 0 { + // Cap to the user specified timeout if one is specified. + remaining = uto - elapsed + } + + // Always honor the user-timeout irrespective of whether the zero + // window probes were acknowledged. + // net/ipv4/tcp_timer.c::tcp_probe_timer() + if remaining <= 0 || s.unackZeroWindowProbes >= s.maxRetries { + s.ep.stack.Stats().TCP.EstablishedTimedout.Increment() + return &tcpip.ErrTimeout{} + } + + // Set new timeout. The timer will be restarted by the call to sendData + // below. + s.RTO *= 2 + // Cap the RTO as per RFC 1122 4.2.3.1, RFC 6298 5.5 + if s.RTO > s.maxRTO { + s.RTO = s.maxRTO + } + + // Cap RTO to remaining time. + if s.RTO > remaining { + s.RTO = remaining + } + + // See: https://tools.ietf.org/html/rfc6582#section-3.2 Step 4. + // + // Retransmit timeouts: + // After a retransmit timeout, record the highest sequence number + // transmitted in the variable recover, and exit the fast recovery + // procedure if applicable. + s.FastRecovery.Last = s.SndNxt - 1 + + if s.FastRecovery.Active { + // We were attempting fast recovery but were not successful. + // Leave the state. We don't need to update ssthresh because it + // has already been updated when entered fast-recovery. + s.leaveRecovery() + } + + // Record retransmitTS if the sender is not in recovery as per: + // https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 2 + s.recordRetransmitTS() + + s.state = tcpip.RTORecovery + s.cc.HandleRTOExpired() + + // Mark the next segment to be sent as the first unacknowledged one and + // start sending again. Set the number of outstanding packets to 0 so + // that we'll be able to retransmit. + // + // We'll keep on transmitting (or retransmitting) as we get acks for + // the data we transmit. + s.Outstanding = 0 + + // Expunge all SACK information as per https://tools.ietf.org/html/rfc6675#section-5.1 + // + // In order to avoid memory deadlocks, the TCP receiver is allowed to + // discard data that has already been selectively acknowledged. As a + // result, [RFC2018] suggests that a TCP sender SHOULD expunge the SACK + // information gathered from a receiver upon a retransmission timeout + // (RTO) "since the timeout might indicate that the data receiver has + // reneged." Additionally, a TCP sender MUST "ignore prior SACK + // information in determining which data to retransmit." + // + // NOTE: We take the stricter interpretation and just expunge all + // information as we lack more rigorous checks to validate if the SACK + // information is usable after an RTO. + s.ep.scoreboard.Reset() + s.updateWriteNext(s.writeList.Front()) + + // RFC 1122 4.2.2.17: Start sending zero window probes when we still see a + // zero receive window after retransmission interval and we have data to + // send. + if s.zeroWindowProbing { + s.sendZeroWindowProbe() + // RFC 1122 4.2.2.17: A TCP MAY keep its offered receive window closed + // indefinitely. As long as the receiving TCP continues to send + // acknowledgments in response to the probe segments, the sending TCP + // MUST allow the connection to stay open. + return nil + } + + seg := s.writeNext + // RFC 1122 4.2.3.5: Close the connection when the number of + // retransmissions for this segment is beyond a limit. + if seg != nil && seg.xmitCount > s.maxRetries { + s.ep.stack.Stats().TCP.EstablishedTimedout.Increment() + return &tcpip.ErrTimeout{} + } + + s.sendData() + + return nil +} + +// pCount returns the number of packets in the segment. Due to GSO, a segment +// can be composed of multiple packets. +func (s *sender) pCount(seg *segment, maxPayloadSize int) int { + size := seg.payloadSize() + if size == 0 { + return 1 + } + + return (size-1)/maxPayloadSize + 1 +} + +// splitSeg splits a given segment at the size specified and inserts the +// remainder as a new segment after the current one in the write list. +// +// +checklocks:s.ep.mu +func (s *sender) splitSeg(seg *segment, size int) { + if seg.payloadSize() <= size { + return + } + // Split this segment up. + nSeg := seg.clone() + nSeg.pkt.Data().TrimFront(size) + nSeg.sequenceNumber.UpdateForward(seqnum.Size(size)) + s.writeList.InsertAfter(seg, nSeg) + + // The segment being split does not carry PUSH flag because it is + // followed by the newly split segment. + // RFC1122 section 4.2.2.2: MUST set the PSH bit in the last buffered + // segment (i.e., when there is no more queued data to be sent). + // Linux removes PSH flag only when the segment is being split over MSS + // and retains it when we are splitting the segment over lack of sender + // window space. + // ref: net/ipv4/tcp_output.c::tcp_write_xmit(), tcp_mss_split_point() + // ref: net/ipv4/tcp_output.c::tcp_write_wakeup(), tcp_snd_wnd_test() + if seg.payloadSize() > s.MaxPayloadSize { + seg.flags ^= header.TCPFlagPsh + } + seg.pkt.Data().CapLength(size) +} + +// NextSeg implements the RFC6675 NextSeg() operation. +// +// NextSeg starts scanning the writeList starting from nextSegHint and returns +// the hint to be passed on the next call to NextSeg. This is required to avoid +// iterating the write list repeatedly when NextSeg is invoked in a loop during +// recovery. The returned hint will be nil if there are no more segments that +// can match rules defined by NextSeg operation in RFC6675. +// +// rescueRtx will be true only if nextSeg is a rescue retransmission as +// described by Step 4) of the NextSeg algorithm. +// +// +checklocks:s.ep.mu +func (s *sender) NextSeg(nextSegHint *segment) (nextSeg, hint *segment, rescueRtx bool) { + var s3 *segment + var s4 *segment + // Step 1. + for seg := nextSegHint; seg != nil; seg = seg.Next() { + // Stop iteration if we hit a segment that has never been + // transmitted (i.e. either it has no assigned sequence number + // or if it does have one, it's >= the next sequence number + // to be sent [i.e. >= s.sndNxt]). + if !s.isAssignedSequenceNumber(seg) || s.SndNxt.LessThanEq(seg.sequenceNumber) { + hint = nil + break + } + segSeq := seg.sequenceNumber + if smss := s.ep.scoreboard.SMSS(); seg.payloadSize() > int(smss) { + s.splitSeg(seg, int(smss)) + } + + // See RFC 6675 Section 4 + // + // 1. If there exists a smallest unSACKED sequence number + // 'S2' that meets the following 3 criteria for determinig + // loss, the sequence range of one segment of up to SMSS + // octets starting with S2 MUST be returned. + if !s.ep.scoreboard.IsSACKED(header.SACKBlock{Start: segSeq, End: segSeq.Add(1)}) { + // NextSeg(): + // + // (1.a) S2 is greater than HighRxt + // (1.b) S2 is less than highest octet covered by + // any received SACK. + if s.FastRecovery.HighRxt.LessThan(segSeq) && segSeq.LessThan(s.ep.scoreboard.maxSACKED) { + // NextSeg(): + // (1.c) IsLost(S2) returns true. + if s.ep.scoreboard.IsLost(segSeq) { + return seg, seg.Next(), false + } + + // NextSeg(): + // + // (3): If the conditions for rules (1) and (2) + // fail, but there exists an unSACKed sequence + // number S3 that meets the criteria for + // detecting loss given in steps 1.a and 1.b + // above (specifically excluding (1.c)) then one + // segment of upto SMSS octets starting with S3 + // SHOULD be returned. + if s3 == nil { + s3 = seg + hint = seg.Next() + } + } + // NextSeg(): + // + // (4) If the conditions for (1), (2) and (3) fail, + // but there exists outstanding unSACKED data, we + // provide the opportunity for a single "rescue" + // retransmission per entry into loss recovery. If + // HighACK is greater than RescueRxt (or RescueRxt + // is undefined), then one segment of upto SMSS + // octets that MUST include the highest outstanding + // unSACKed sequence number SHOULD be returned, and + // RescueRxt set to RecoveryPoint. HighRxt MUST NOT + // be updated. + if s.FastRecovery.RescueRxt.LessThan(s.SndUna - 1) { + if s4 != nil { + if s4.sequenceNumber.LessThan(segSeq) { + s4 = seg + } + } else { + s4 = seg + } + } + } + } + + // If we got here then no segment matched step (1). + // Step (2): "If no sequence number 'S2' per rule (1) + // exists but there exists available unsent data and the + // receiver's advertised window allows, the sequence + // range of one segment of up to SMSS octets of + // previously unsent data starting with sequence number + // HighData+1 MUST be returned." + for seg := s.writeNext; seg != nil; seg = seg.Next() { + if s.isAssignedSequenceNumber(seg) && seg.sequenceNumber.LessThan(s.SndNxt) { + continue + } + // We do not split the segment here to <= smss as it has + // potentially not been assigned a sequence number yet. + return seg, nil, false + } + + if s3 != nil { + return s3, hint, false + } + + return s4, nil, true +} + +// maybeSendSegment tries to send the specified segment and either coalesces +// other segments into this one or splits the specified segment based on the +// lower of the specified limit value or the receivers window size specified by +// end. +// +checklocks:s.ep.mu +func (s *sender) maybeSendSegment(seg *segment, limit int, end seqnum.Value) (sent bool) { + // We abuse the flags field to determine if we have already + // assigned a sequence number to this segment. + if !s.isAssignedSequenceNumber(seg) { + // Merge segments if allowed. + if seg.payloadSize() != 0 { + available := int(s.SndNxt.Size(end)) + if available > limit { + available = limit + } + + // nextTooBig indicates that the next segment was too + // large to entirely fit in the current segment. It + // would be possible to split the next segment and merge + // the portion that fits, but unexpectedly splitting + // segments can have user visible side-effects which can + // break applications. For example, RFC 7766 section 8 + // says that the length and data of a DNS response + // should be sent in the same TCP segment to avoid + // triggering bugs in poorly written DNS + // implementations. + var nextTooBig bool + for nSeg := seg.Next(); nSeg != nil && nSeg.payloadSize() != 0; nSeg = seg.Next() { + if seg.payloadSize()+nSeg.payloadSize() > available { + nextTooBig = true + break + } + seg.merge(nSeg) + s.writeList.Remove(nSeg) + nSeg.DecRef() + } + if !nextTooBig && seg.payloadSize() < available { + // Segment is not full. + if s.Outstanding > 0 && s.ep.ops.GetDelayOption() { + // Nagle's algorithm. From Wikipedia: + // Nagle's algorithm works by + // combining a number of small + // outgoing messages and sending them + // all at once. Specifically, as long + // as there is a sent packet for which + // the sender has received no + // acknowledgment, the sender should + // keep buffering its output until it + // has a full packet's worth of + // output, thus allowing output to be + // sent all at once. + return false + } + // With TCP_CORK, hold back until minimum of the available + // send space and MSS. + if s.ep.ops.GetCorkOption() { + if seg.payloadSize() < s.MaxPayloadSize { + if !s.startCork { + s.startCork = true + // Enable the timer for + // 200ms, after which + // the segments are drained. + s.corkTimer.enable(MinRTO) + } + return false + } + // Disable the TCP_CORK timer. + s.startCork = false + s.corkTimer.disable() + } + } + } + + // Assign flags. We don't do it above so that we can merge + // additional data if Nagle holds the segment. + seg.sequenceNumber = s.SndNxt + seg.flags = header.TCPFlagAck | header.TCPFlagPsh + } + + var segEnd seqnum.Value + if seg.payloadSize() == 0 { + if s.writeList.Back() != seg { + panic("FIN segments must be the final segment in the write list.") + } + seg.flags = header.TCPFlagAck | header.TCPFlagFin + segEnd = seg.sequenceNumber.Add(1) + // Update the state to reflect that we have now + // queued a FIN. + s.ep.updateConnDirectionState(connDirectionStateSndClosed) + switch s.ep.EndpointState() { + case StateCloseWait: + s.ep.setEndpointState(StateLastAck) + default: + s.ep.setEndpointState(StateFinWait1) + } + } else { + // We're sending a non-FIN segment. + if seg.flags&header.TCPFlagFin != 0 { + panic("Netstack queues FIN segments without data.") + } + + if !seg.sequenceNumber.LessThan(end) { + return false + } + + available := int(seg.sequenceNumber.Size(end)) + if available == 0 { + return false + } + + // If the whole segment or at least 1MSS sized segment cannot + // be accommodated in the receiver advertised window, skip + // splitting and sending of the segment. ref: + // net/ipv4/tcp_output.c::tcp_snd_wnd_test() + // + // Linux checks this for all segment transmits not triggered by + // a probe timer. On this condition, it defers the segment split + // and transmit to a short probe timer. + // + // ref: include/net/tcp.h::tcp_check_probe_timer() + // ref: net/ipv4/tcp_output.c::tcp_write_wakeup() + // + // Instead of defining a new transmit timer, we attempt to split + // the segment right here if there are no pending segments. If + // there are pending segments, segment transmits are deferred to + // the retransmit timer handler. + if s.SndUna != s.SndNxt { + switch { + case available >= seg.payloadSize(): + // OK to send, the whole segments fits in the + // receiver's advertised window. + case available >= s.MaxPayloadSize: + // OK to send, at least 1 MSS sized segment fits + // in the receiver's advertised window. + default: + return false + } + } + + // The segment size limit is computed as a function of sender + // congestion window and MSS. When sender congestion window is > + // 1, this limit can be larger than MSS. Ensure that the + // currently available send space is not greater than minimum of + // this limit and MSS. + if available > limit { + available = limit + } + + // If GSO is not in use then cap available to + // maxPayloadSize. When GSO is in use the gVisor GSO logic or + // the host GSO logic will cap the segment to the correct size. + if s.ep.gso.Type == stack.GSONone && available > s.MaxPayloadSize { + available = s.MaxPayloadSize + } + + if seg.payloadSize() > available { + // A negative value causes splitSeg to panic anyways, so just panic + // earlier to get more information about the cause. + s.splitSeg(seg, available) + } + + segEnd = seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize())) + } + + // TODO(b/379932042): Below is the only place we update SND.NXT besides + // initialization. It's possible that we're increasing SND.NXT by + // trying to write a segment that isn't in the write list. + if _, ok := s.writeList.set[seg]; !ok { + panic("attempted to send segment not in write list") + } + + s.sendSegment(seg) + + // Update sndNxt if we actually sent new data (as opposed to + // retransmitting some previously sent data). + if s.SndNxt.LessThan(segEnd) { + s.SndNxt = segEnd + } + + return true +} + +// zeroProbeJunk is data sent during zero window probes. Its value is +// irrelevant; since the sequence number has already been acknowledged it will +// be discarded. It's only here to avoid allocating. +var zeroProbeJunk = []byte{0} + +// +checklocks:s.ep.mu +func (s *sender) sendZeroWindowProbe() { + s.unackZeroWindowProbes++ + + // Send a zero window probe with sequence number pointing to the last + // acknowledged byte. Note that, like Linux, this isn't quite what RFC + // 9293 3.8.6.1 describes: we don't send the next byte in the stream, + // we re-send an ACKed byte to goad the receiver into responding. + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithData(zeroProbeJunk), + }) + defer pkt.DecRef() + s.sendSegmentFromPacketBuffer(pkt, header.TCPFlagAck, s.SndUna-1) + + // Rearm the timer to continue probing. + s.resendTimer.enable(s.RTO) +} + +// +checklocks:s.ep.mu +func (s *sender) enableZeroWindowProbing() { + s.zeroWindowProbing = true + // We piggyback the probing on the retransmit timer with the + // current retranmission interval, as we may start probing while + // segment retransmissions. + if s.firstRetransmittedSegXmitTime == (tcpip.MonotonicTime{}) { + s.firstRetransmittedSegXmitTime = s.ep.stack.Clock().NowMonotonic() + } + s.resendTimer.enable(s.RTO) +} + +func (s *sender) disableZeroWindowProbing() { + s.zeroWindowProbing = false + s.unackZeroWindowProbes = 0 + s.firstRetransmittedSegXmitTime = tcpip.MonotonicTime{} + s.resendTimer.disable() +} + +// +checklocks:s.ep.mu +func (s *sender) postXmit(dataSent bool, shouldScheduleProbe bool) { + if dataSent { + // We sent data, so we should stop the keepalive timer to ensure + // that no keepalives are sent while there is pending data. + s.ep.disableKeepaliveTimer() + } + + // If the sender has advertised zero receive window and we have + // data to be sent out, start zero window probing to query the + // the remote for it's receive window size. + if s.writeNext != nil && s.SndWnd == 0 { + s.enableZeroWindowProbing() + } + + // If we have no more pending data, start the keepalive timer. + if s.SndUna == s.SndNxt { + s.ep.resetKeepaliveTimer(false) + } else { + // Enable timers if we have pending data. + if shouldScheduleProbe && s.shouldSchedulePTO() { + // Schedule PTO after transmitting new data that wasn't itself a TLP probe. + s.schedulePTO() + } else if !s.resendTimer.enabled() { + s.probeTimer.disable() + if s.Outstanding > 0 { + // Enable the resend timer if it's not enabled yet and there is + // outstanding data. + s.resendTimer.enable(s.RTO) + } + } + } +} + +// sendData sends new data segments. It is called when data becomes available or +// when the send window opens up. +// +checklocks:s.ep.mu +func (s *sender) sendData() { + limit := s.MaxPayloadSize + if s.gso { + limit = int(s.ep.gso.MaxSize - header.TCPTotalHeaderMaximumSize - 1) + } + end := s.SndUna.Add(s.SndWnd) + + // Reduce the congestion window to min(IW, cwnd) per RFC 5681, page 10. + // "A TCP SHOULD set cwnd to no more than RW before beginning + // transmission if the TCP has not sent data in the interval exceeding + // the retrasmission timeout." + if !s.FastRecovery.Active && s.state != tcpip.RTORecovery && s.ep.stack.Clock().NowMonotonic().Sub(s.LastSendTime) > s.RTO { + if s.SndCwnd > InitialCwnd { + s.SndCwnd = InitialCwnd + } + } + + var dataSent bool + for seg := s.writeNext; seg != nil && s.Outstanding < s.SndCwnd; seg = seg.Next() { + // NOTE(gvisor.dev/issue/11632): Use uint64 to avoid overflow. + cwndLimit := uint64(s.SndCwnd-s.Outstanding) * uint64(s.MaxPayloadSize) + if cwndLimit > 0 && cwndLimit < uint64(limit) { + limit = int(cwndLimit) + } + if s.isAssignedSequenceNumber(seg) && s.ep.SACKPermitted && s.ep.scoreboard.IsSACKED(seg.sackBlock()) { + // Move writeNext along so that we don't try and scan data that + // has already been SACKED. + s.updateWriteNext(seg.Next()) + continue + } + if sent := s.maybeSendSegment(seg, limit, end); !sent { + break + } + dataSent = true + s.Outstanding += s.pCount(seg, s.MaxPayloadSize) + s.updateWriteNext(seg.Next()) + } + + s.postXmit(dataSent, true /* shouldScheduleProbe */) +} + +// +checklocks:s.ep.mu +func (s *sender) enterRecovery() { + // Initialize the variables used to detect spurious recovery after + // entering recovery. + // + // See: https://www.rfc-editor.org/rfc/rfc3522.html#section-3.2 Step 1. + s.spuriousRecovery = false + s.retransmitTS = 0 + + s.FastRecovery.Active = true + // Save state to reflect we're now in fast recovery. + // + // See : https://tools.ietf.org/html/rfc5681#section-3.2 Step 3. + // We inflate the cwnd by 3 to account for the 3 packets which triggered + // the 3 duplicate ACKs and are now not in flight. + s.SndCwnd = s.Ssthresh + 3 + s.SackedOut = 0 + s.DupAckCount = 0 + s.FastRecovery.First = s.SndUna + s.FastRecovery.Last = s.SndNxt - 1 + s.FastRecovery.MaxCwnd = s.SndCwnd + s.Outstanding + s.FastRecovery.HighRxt = s.SndUna + s.FastRecovery.RescueRxt = s.SndUna + + // Record retransmitTS if the sender is not in recovery as per: + // https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 2 + s.recordRetransmitTS() + + if s.ep.SACKPermitted { + s.state = tcpip.SACKRecovery + s.ep.stack.Stats().TCP.SACKRecovery.Increment() + // Set TLPRxtOut to false according to + // https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.6.1. + if s.rc.tlpRxtOut { + // The tail loss probe triggered recovery. + s.ep.stack.Stats().TCP.TLPRecovery.Increment() + } + s.rc.tlpRxtOut = false + return + } + s.state = tcpip.FastRecovery + s.ep.stack.Stats().TCP.FastRecovery.Increment() +} + +// +checklocks:s.ep.mu +func (s *sender) leaveRecovery() { + s.FastRecovery.Active = false + s.FastRecovery.MaxCwnd = 0 + s.DupAckCount = 0 + + // Deflate cwnd. It had been artificially inflated when new dups arrived. + s.SndCwnd = s.Ssthresh + s.cc.PostRecovery() +} + +// isAssignedSequenceNumber relies on the fact that we only set flags once a +// sequencenumber is assigned and that is only done right before we send the +// segment. As a result any segment that has a non-zero flag has a valid +// sequence number assigned to it. +func (s *sender) isAssignedSequenceNumber(seg *segment) bool { + return seg.flags != 0 +} + +// SetPipe implements the SetPipe() function described in RFC6675. Netstack +// maintains the congestion window in number of packets and not bytes, so +// SetPipe() here measures number of outstanding packets rather than actual +// outstanding bytes in the network. +// +// +checklocks:s.ep.mu +func (s *sender) SetPipe() { + // If SACK isn't permitted or it is permitted but recovery is not active + // then ignore pipe calculations. + if !s.ep.SACKPermitted || !s.FastRecovery.Active { + return + } + pipe := 0 + smss := seqnum.Size(s.ep.scoreboard.SMSS()) + for s1 := s.writeList.Front(); s1 != nil && s1.payloadSize() != 0 && s.isAssignedSequenceNumber(s1); s1 = s1.Next() { + // With GSO each segment can be much larger than SMSS. So check the segment + // in SMSS sized ranges. + segEnd := s1.sequenceNumber.Add(seqnum.Size(s1.payloadSize())) + for startSeq := s1.sequenceNumber; startSeq.LessThan(segEnd); startSeq = startSeq.Add(smss) { + endSeq := startSeq.Add(smss) + if segEnd.LessThan(endSeq) { + endSeq = segEnd + } + sb := header.SACKBlock{Start: startSeq, End: endSeq} + // SetPipe(): + // + // After initializing pipe to zero, the following steps are + // taken for each octet 'S1' in the sequence space between + // HighACK and HighData that has not been SACKed: + if !s1.sequenceNumber.LessThan(s.SndNxt) { + break + } + if s.ep.scoreboard.IsSACKED(sb) { + continue + } + + // SetPipe(): + // + // (a) If IsLost(S1) returns false, Pipe is incremened by 1. + // + // NOTE: here we mark the whole segment as lost. We do not try + // and test every byte in our write buffer as we maintain our + // pipe in terms of outstanding packets and not bytes. + if !s.ep.scoreboard.IsRangeLost(sb) { + pipe++ + } + // SetPipe(): + // (b) If S1 <= HighRxt, Pipe is incremented by 1. + if s1.sequenceNumber.LessThanEq(s.FastRecovery.HighRxt) { + pipe++ + } + } + } + s.Outstanding = pipe +} + +// shouldEnterRecovery returns true if the sender should enter fast recovery +// based on dupAck count and sack scoreboard. +// See RFC 6675 section 5. +// +// +checklocks:s.ep.mu +func (s *sender) shouldEnterRecovery() bool { + return s.DupAckCount >= nDupAckThreshold || + (s.ep.SACKPermitted && s.ep.tcpRecovery&tcpip.TCPRACKLossDetection == 0 && s.ep.scoreboard.IsLost(s.SndUna)) +} + +// detectLoss is called when an ack is received and returns whether a loss is +// detected. It manages the state related to duplicate acks and determines if +// a retransmit is needed according to the rules in RFC 6582 (NewReno). +// +// +checklocks:s.ep.mu +func (s *sender) detectLoss(seg *segment) (fastRetransmit bool) { + // We're not in fast recovery yet. + + // If RACK is enabled and there is no reordering we should honor the + // three duplicate ACK rule to enter recovery. + // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-4 + if s.ep.SACKPermitted && s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 { + if s.rc.Reord { + return false + } + } + + if !s.isDupAck(seg) { + s.DupAckCount = 0 + return false + } + + s.DupAckCount++ + + // Do not enter fast recovery until we reach nDupAckThreshold or the + // first unacknowledged byte is considered lost as per SACK scoreboard. + if !s.shouldEnterRecovery() { + // RFC 6675 Step 3. + s.FastRecovery.HighRxt = s.SndUna - 1 + // Do run SetPipe() to calculate the outstanding segments. + s.SetPipe() + s.state = tcpip.Disorder + return false + } + + // See: https://tools.ietf.org/html/rfc6582#section-3.2 Step 2 + // + // We only do the check here, the incrementing of last to the highest + // sequence number transmitted till now is done when enterRecovery + // is invoked. + // + // Note that we only enter recovery when at least one more byte of data + // beyond s.fr.last (the highest byte that was outstanding when fast + // retransmit was last entered) is acked. + if !s.FastRecovery.Last.LessThan(seg.ackNumber - 1) { + s.DupAckCount = 0 + return false + } + s.cc.HandleLossDetected() + s.enterRecovery() + return true +} + +// isDupAck determines if seg is a duplicate ack as defined in +// https://tools.ietf.org/html/rfc5681#section-2. +// +// +checklocks:s.ep.mu +func (s *sender) isDupAck(seg *segment) bool { + // A TCP that utilizes selective acknowledgments (SACKs) [RFC2018, RFC2883] + // can leverage the SACK information to determine when an incoming ACK is a + // "duplicate" (e.g., if the ACK contains previously unknown SACK + // information). + if s.ep.SACKPermitted && !seg.hasNewSACKInfo { + return false + } + + // (a) The receiver of the ACK has outstanding data. + return s.SndUna != s.SndNxt && + // (b) The incoming acknowledgment carries no data. + seg.logicalLen() == 0 && + // (c) The SYN and FIN bits are both off. + !seg.flags.Intersects(header.TCPFlagFin|header.TCPFlagSyn) && + // (d) the ACK number is equal to the greatest acknowledgment received on + // the given connection (TCP.UNA from RFC793). + seg.ackNumber == s.SndUna && + // (e) the advertised window in the incoming acknowledgment equals the + // advertised window in the last incoming acknowledgment. + s.SndWnd == seg.window +} + +// Iterate the writeList and update RACK for each segment which is newly acked +// either cumulatively or selectively. Loop through the segments which are +// sacked, and update the RACK related variables and check for reordering. +// Returns true when the DSACK block has been detected in the received ACK. +// +// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 +// steps 2 and 3. +// +// +checklocks:s.ep.mu +func (s *sender) walkSACK(rcvdSeg *segment) bool { + s.rc.setDSACKSeen(false) + + // Look for DSACK block. + hasDSACK := false + idx := 0 + n := len(rcvdSeg.parsedOptions.SACKBlocks) + if checkDSACK(rcvdSeg) { + dsackBlock := rcvdSeg.parsedOptions.SACKBlocks[0] + numDSACK := uint64(dsackBlock.End-dsackBlock.Start) / uint64(s.MaxPayloadSize) + // numDSACK can be zero when DSACK is sent for subsegments. + if numDSACK < 1 { + numDSACK = 1 + } + s.ep.stack.Stats().TCP.SegmentsAckedWithDSACK.IncrementBy(numDSACK) + s.rc.setDSACKSeen(true) + idx = 1 + n-- + hasDSACK = true + } + + if n == 0 { + return hasDSACK + } + + // Sort the SACK blocks. The first block is the most recent unacked + // block. The following blocks can be in arbitrary order. + sackBlocks := make([]header.SACKBlock, n) + copy(sackBlocks, rcvdSeg.parsedOptions.SACKBlocks[idx:]) + sort.Slice(sackBlocks, func(i, j int) bool { + return sackBlocks[j].Start.LessThan(sackBlocks[i].Start) + }) + + seg := s.writeList.Front() + for _, sb := range sackBlocks { + for seg != nil && seg.sequenceNumber.LessThan(sb.End) && seg.xmitCount != 0 { + if sb.Start.LessThanEq(seg.sequenceNumber) && !seg.acked { + s.rc.update(seg, rcvdSeg) + s.rc.detectReorder(seg) + seg.acked = true + s.SackedOut += s.pCount(seg, s.MaxPayloadSize) + } + seg = seg.Next() + } + } + return hasDSACK +} + +// checkDSACK checks if a DSACK is reported. +func checkDSACK(rcvdSeg *segment) bool { + n := len(rcvdSeg.parsedOptions.SACKBlocks) + if n == 0 { + return false + } + + sb := rcvdSeg.parsedOptions.SACKBlocks[0] + // Check if SACK block is invalid. + if sb.End.LessThan(sb.Start) { + return false + } + + // See: https://tools.ietf.org/html/rfc2883#section-5 DSACK is sent in + // at most one SACK block. DSACK is detected in the below two cases: + // * If the SACK sequence space is less than this cumulative ACK, it is + // an indication that the segment identified by the SACK block has + // been received more than once by the receiver. + // * If the sequence space in the first SACK block is greater than the + // cumulative ACK, then the sender next compares the sequence space + // in the first SACK block with the sequence space in the second SACK + // block, if there is one. This comparison can determine if the first + // SACK block is reporting duplicate data that lies above the + // cumulative ACK. + if sb.Start.LessThan(rcvdSeg.ackNumber) { + return true + } + + if n > 1 { + sb1 := rcvdSeg.parsedOptions.SACKBlocks[1] + if sb1.End.LessThan(sb1.Start) { + return false + } + + // If the first SACK block is fully covered by second SACK + // block, then the first block is a DSACK block. + if sb.End.LessThanEq(sb1.End) && sb1.Start.LessThanEq(sb.Start) { + return true + } + } + + return false +} + +func (s *sender) recordRetransmitTS() { + // See: https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 + // + // The Eifel detection algorithm is used, only upon initiation of loss + // recovery, i.e., when either the timeout-based retransmit or the fast + // retransmit is sent. The Eifel detection algorithm MUST NOT be + // reinitiated after loss recovery has already started. In particular, + // it must not be reinitiated upon subsequent timeouts for the same + // segment, and not upon retransmitting segments other than the oldest + // outstanding segment, e.g., during selective loss recovery. + if s.inRecovery() { + return + } + + // See: https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 2 + // + // Set a "RetransmitTS" variable to the value of the Timestamp Value + // field of the Timestamps option included in the retransmit sent when + // loss recovery is initiated. A TCP sender must ensure that + // RetransmitTS does not get overwritten as loss recovery progresses, + // e.g., in case of a second timeout and subsequent second retransmit of + // the same octet. + s.retransmitTS = s.ep.tsValNow() +} + +// +checklocks:s.ep.mu +func (s *sender) detectSpuriousRecovery(hasDSACK bool, tsEchoReply uint32) { + // Return if the sender has already detected spurious recovery. + if s.spuriousRecovery { + return + } + + // See: https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 4 + // + // If the value of the Timestamp Echo Reply field of the acceptable ACK's + // Timestamps option is smaller than the value of RetransmitTS, then + // proceed to next step, else return. + if tsEchoReply >= s.retransmitTS { + return + } + + // See: https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 5 + // + // If the acceptable ACK carries a DSACK option [RFC2883], then return. + if hasDSACK { + return + } + + // See: https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 5 + // + // If during the lifetime of the TCP connection the TCP sender has + // previously received an ACK with a DSACK option, or the acceptable ACK + // does not acknowledge all outstanding data, then proceed to next step, + // else return. + numDSACK := s.ep.stack.Stats().TCP.SegmentsAckedWithDSACK.Value() + if numDSACK == 0 && s.SndUna == s.SndNxt { + return + } + + // See: https://datatracker.ietf.org/doc/html/rfc3522#section-3.2 Step 6 + // + // If the loss recovery has been initiated with a timeout-based + // retransmit, then set + // SpuriousRecovery <- SPUR_TO (equal 1), + // else set + // SpuriousRecovery <- dupacks+1 + // Set the spurious recovery variable to true as we do not differentiate + // between fast, SACK or RTO recovery. + s.spuriousRecovery = true + s.ep.stack.Stats().TCP.SpuriousRecovery.Increment() + + // RFC 3522 will detect all kinds of spurious recoveries (fast, SACK and + // timeout). Increment the metric for RTO only as we want to track the + // number of timeout recoveries. + if s.state == tcpip.RTORecovery { + s.ep.stack.Stats().TCP.SpuriousRTORecovery.Increment() + } +} + +// Check if the sender is in RTORecovery, FastRecovery or SACKRecovery state. +func (s *sender) inRecovery() bool { + if s.state == tcpip.RTORecovery || s.state == tcpip.FastRecovery || s.state == tcpip.SACKRecovery { + return true + } + return false +} + +// handleRcvdSegment is called when a segment is received; it is responsible for +// updating the send-related state. +// +checklocks:s.ep.mu +// +checklocksalias:s.rc.snd.ep.mu=s.ep.mu +func (s *sender) handleRcvdSegment(rcvdSeg *segment) { + bestRTT := unknownRTT + + // Check if we can extract an RTT measurement from this ack. + if !rcvdSeg.parsedOptions.TS && s.RTTMeasureSeqNum.LessThan(rcvdSeg.ackNumber) { + bestRTT = s.ep.stack.Clock().NowMonotonic().Sub(s.RTTMeasureTime) + s.updateRTO(bestRTT) + s.RTTMeasureSeqNum = s.SndNxt + } + + // Update Timestamp if required. See RFC7323, section-4.3. + if s.ep.SendTSOk && rcvdSeg.parsedOptions.TS { + s.ep.updateRecentTimestamp(rcvdSeg.parsedOptions.TSVal, s.MaxSentAck, rcvdSeg.sequenceNumber) + } + + // Insert SACKBlock information into our scoreboard. + hasDSACK := false + if s.ep.SACKPermitted { + for _, sb := range rcvdSeg.parsedOptions.SACKBlocks { + // Only insert the SACK block if the following holds + // true: + // * SACK block acks data after the ack number in the + // current segment. + // * SACK block represents a sequence + // between sndUna and sndNxt (i.e. data that is + // currently unacked and in-flight). + // * SACK block that has not been SACKed already. + // + // NOTE: This check specifically excludes DSACK blocks + // which have start/end before sndUna and are used to + // indicate spurious retransmissions. + if rcvdSeg.ackNumber.LessThan(sb.Start) && s.SndUna.LessThan(sb.Start) && sb.End.LessThanEq(s.SndNxt) && !s.ep.scoreboard.IsSACKED(sb) { + s.ep.scoreboard.Insert(sb) + rcvdSeg.hasNewSACKInfo = true + } + } + + // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08 + // section-7.2 + // * Step 2: Update RACK stats. + // If the ACK is not ignored as invalid, update the RACK.rtt + // to be the RTT sample calculated using this ACK, and + // continue. If this ACK or SACK was for the most recently + // sent packet, then record the RACK.xmit_ts timestamp and + // RACK.end_seq sequence implied by this ACK. + // * Step 3: Detect packet reordering. + // If the ACK selectively or cumulatively acknowledges an + // unacknowledged and also never retransmitted sequence below + // RACK.fack, then the corresponding packet has been + // reordered and RACK.reord is set to TRUE. + if s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 { + hasDSACK = s.walkSACK(rcvdSeg) + } + s.SetPipe() + } + + ack := rcvdSeg.ackNumber + fastRetransmit := false + // Do not leave fast recovery, if the ACK is out of range. + if s.FastRecovery.Active { + // Leave fast recovery if it acknowledges all the data covered by + // this fast recovery session. + if (ack-1).InRange(s.SndUna, s.SndNxt) && s.FastRecovery.Last.LessThan(ack) { + s.leaveRecovery() + } + } else { + // Detect loss by counting the duplicates and enter recovery. + fastRetransmit = s.detectLoss(rcvdSeg) + } + + // See if TLP based recovery was successful. + if s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 { + s.detectTLPRecovery(ack, rcvdSeg) + } + + // Stash away the current window size. + s.SndWnd = rcvdSeg.window + + // Disable zero window probing if remote advertises a non-zero receive + // window. This can be with an ACK to the zero window probe (where the + // acknumber refers to the already acknowledged byte) OR to any previously + // unacknowledged segment. + if s.zeroWindowProbing && rcvdSeg.window > 0 && + (ack == s.SndUna || (ack-1).InRange(s.SndUna, s.SndNxt)) { + s.disableZeroWindowProbing() + } + + // On receiving the ACK for the zero window probe, account for it and + // skip trying to send any segment as we are still probing for + // receive window to become non-zero. + if s.zeroWindowProbing && s.unackZeroWindowProbes > 0 && ack == s.SndUna { + s.unackZeroWindowProbes-- + return + } + + // Ignore ack if it doesn't acknowledge any new data. + if (ack - 1).InRange(s.SndUna, s.SndNxt) { + s.DupAckCount = 0 + + // See : https://tools.ietf.org/html/rfc1323#section-3.3. + // Specifically we should only update the RTO using TSEcr if the + // following condition holds: + // + // A TSecr value received in a segment is used to update the + // averaged RTT measurement only if the segment acknowledges + // some new data, i.e., only if it advances the left edge of + // the send window. + if s.ep.SendTSOk && rcvdSeg.parsedOptions.TSEcr != 0 { + tsRTT := s.ep.elapsed(s.ep.stack.Clock().NowMonotonic(), rcvdSeg.parsedOptions.TSEcr) + s.updateRTO(tsRTT) + // Following Linux, prefer RTT computed from ACKs to TSEcr because, + // "broken middle-boxes or peers may corrupt TS-ECR fields" + // https://github.com/torvalds/linux/blob/39cd87c4eb2b893354f3b850f916353f2658ae6f/net/ipv4/tcp_input.c#L3141C1-L3144C24 + if bestRTT == unknownRTT { + bestRTT = tsRTT + } + } + + if s.shouldSchedulePTO() { + // Schedule PTO upon receiving an ACK that cumulatively acknowledges data. + // See https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.1. + s.schedulePTO() + } else { + // When an ack is received we must rearm the timer. + // RFC 6298 5.3 + s.probeTimer.disable() + s.resendTimer.enable(s.RTO) + } + + // Remove all acknowledged data from the write list. + acked := s.SndUna.Size(ack) + s.SndUna = ack + ackLeft := acked + originalOutstanding := s.Outstanding + for ackLeft > 0 { + // We use logicalLen here because we can have FIN + // segments (which are always at the end of list) that + // have no data, but do consume a sequence number. + seg := s.writeList.Front() + if seg == nil { + panic(fmt.Sprintf("invalid state: there are %d unacknowledged bytes left, but the write list is empty:\n"+ + "TCPSenderState: %+v\nsender: %+v\nendpoint: %+v", ackLeft, s.TCPSenderState, s, s.ep)) + } + + datalen := seg.logicalLen() + if datalen > ackLeft { + prevCount := s.pCount(seg, s.MaxPayloadSize) + seg.TrimFront(ackLeft) + seg.sequenceNumber.UpdateForward(ackLeft) + s.Outstanding -= prevCount - s.pCount(seg, s.MaxPayloadSize) + break + } + + if s.writeNext == seg { + s.updateWriteNext(seg.Next()) + } + + // Update the RACK fields if SACK is enabled. + if s.ep.SACKPermitted && !seg.acked && s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 { + s.rc.update(seg, rcvdSeg) + s.rc.detectReorder(seg) + } + + s.writeList.Remove(seg) + + // If SACK is enabled then only reduce outstanding if + // the segment was not previously SACKED as these have + // already been accounted for in SetPipe(). + if !s.ep.SACKPermitted || !s.ep.scoreboard.IsSACKED(seg.sackBlock()) { + s.Outstanding -= s.pCount(seg, s.MaxPayloadSize) + } else { + s.SackedOut -= s.pCount(seg, s.MaxPayloadSize) + } + seg.DecRef() + ackLeft -= datalen + } + + // Clear SACK information for all acked data. + s.ep.scoreboard.Delete(s.SndUna) + + // Detect if the sender entered recovery spuriously. + if s.inRecovery() { + s.detectSpuriousRecovery(hasDSACK, rcvdSeg.parsedOptions.TSEcr) + } + + // If we are not in fast recovery then update the congestion + // window based on the number of acknowledged packets. + if !s.FastRecovery.Active { + s.cc.Update(originalOutstanding-s.Outstanding, bestRTT) + if s.FastRecovery.Last.LessThan(s.SndUna) { + s.state = tcpip.Open + // Update RACK when we are exiting fast or RTO + // recovery as described in the RFC + // draft-ietf-tcpm-rack-08 Section-7.2 Step 4. + if s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 { + s.rc.exitRecovery() + } + s.reorderTimer.disable() + } + } + + // Update the send buffer usage and notify potential waiters. + s.ep.updateSndBufferUsage(int(acked)) + + // It is possible for s.outstanding to drop below zero if we get + // a retransmit timeout, reset outstanding to zero but later + // get an ack that cover previously sent data. + if s.Outstanding < 0 { + s.Outstanding = 0 + } + + s.SetPipe() + + // If all outstanding data was acknowledged the disable the timer. + // RFC 6298 Rule 5.3 + if s.SndUna == s.SndNxt { + s.Outstanding = 0 + // Reset firstRetransmittedSegXmitTime to the zero value. + s.firstRetransmittedSegXmitTime = tcpip.MonotonicTime{} + s.resendTimer.disable() + s.probeTimer.disable() + } + } + + if s.ep.SACKPermitted && s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 { + // Update RACK reorder window. + // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 + // * Upon receiving an ACK: + // * Step 4: Update RACK reordering window + s.rc.updateRACKReorderWindow() + + // After the reorder window is calculated, detect any loss by checking + // if the time elapsed after the segments are sent is greater than the + // reorder window. + if numLost := s.rc.detectLoss(rcvdSeg.rcvdTime); numLost > 0 && !s.FastRecovery.Active { + // If any segment is marked as lost by + // RACK, enter recovery and retransmit + // the lost segments. + s.cc.HandleLossDetected() + s.enterRecovery() + fastRetransmit = true + } + + if s.FastRecovery.Active { + s.rc.DoRecovery(nil, fastRetransmit) + } + } + + // Now that we've popped all acknowledged data from the retransmit + // queue, retransmit if needed. + if s.FastRecovery.Active && s.ep.tcpRecovery&tcpip.TCPRACKLossDetection == 0 { + s.lr.DoRecovery(rcvdSeg, fastRetransmit) + // When SACK is enabled data sending is governed by steps in + // RFC 6675 Section 5 recovery steps A-C. + // See: https://tools.ietf.org/html/rfc6675#section-5. + if s.ep.SACKPermitted { + return + } + } + + // Send more data now that some of the pending data has been ack'd, or + // that the window opened up, or the congestion window was inflated due + // to a duplicate ack during fast recovery. This will also re-enable + // the retransmit timer if needed. + s.sendData() +} + +// sendSegment sends the specified segment. +// +checklocks:s.ep.mu +func (s *sender) sendSegment(seg *segment) tcpip.Error { + if seg.xmitCount > 0 { + s.ep.stack.Stats().TCP.Retransmits.Increment() + s.ep.stats.SendErrors.Retransmits.Increment() + if s.SndCwnd < s.Ssthresh { + s.ep.stack.Stats().TCP.SlowStartRetransmits.Increment() + } + } + seg.xmitTime = s.ep.stack.Clock().NowMonotonic() + seg.xmitCount++ + seg.lost = false + + err := s.sendSegmentFromPacketBuffer(seg.pkt, seg.flags, seg.sequenceNumber) + + // Every time a packet containing data is sent (including a + // retransmission), if SACK is enabled and we are retransmitting data + // then use the conservative timer described in RFC6675 Section 6.0, + // otherwise follow the standard time described in RFC6298 Section 5.1. + if err != nil && seg.payloadSize() != 0 { + if s.FastRecovery.Active && seg.xmitCount > 1 && s.ep.SACKPermitted { + s.resendTimer.enable(s.RTO) + } else { + if !s.resendTimer.enabled() { + s.resendTimer.enable(s.RTO) + } + } + } + + return err +} + +// sendSegmentFromPacketBuffer sends a new segment containing the given payload, +// flags and sequence number. +// +checklocks:s.ep.mu +// +checklocksalias:s.ep.rcv.ep.mu=s.ep.mu +// +checklocksalias:s.ep.rcv.ep.snd.ep.mu=s.ep.mu +func (s *sender) sendSegmentFromPacketBuffer(pkt *stack.PacketBuffer, flags header.TCPFlags, seq seqnum.Value) tcpip.Error { + s.LastSendTime = s.ep.stack.Clock().NowMonotonic() + if seq == s.RTTMeasureSeqNum { + s.RTTMeasureTime = s.LastSendTime + } + + rcvNxt, rcvWnd := s.ep.rcv.getSendParams() + + // Remember the max sent ack. + s.MaxSentAck = rcvNxt + + // We need to clone the packet because sendRaw takes ownership of pkt, + // and pkt could be reprocessed later on (i.e retrasmission). + pkt = pkt.Clone() + defer pkt.DecRef() + + return s.ep.sendRaw(pkt, flags, seq, rcvNxt, rcvWnd) +} + +// sendEmptySegment sends a new empty segment, flags and sequence number. +// +checklocks:s.ep.mu +// +checklocksalias:s.ep.rcv.ep.snd.ep.mu=s.ep.mu +// +checklocksalias:s.ep.rcv.ep.mu=s.ep.mu +// +checklocksalias:s.ep.snd.ep.mu=s.ep.mu +func (s *sender) sendEmptySegment(flags header.TCPFlags, seq seqnum.Value) tcpip.Error { + s.LastSendTime = s.ep.stack.Clock().NowMonotonic() + if seq == s.RTTMeasureSeqNum { + s.RTTMeasureTime = s.LastSendTime + } + + rcvNxt, rcvWnd := s.ep.rcv.getSendParams() + + // Remember the max sent ack. + s.MaxSentAck = rcvNxt + + return s.ep.sendEmptyRaw(flags, seq, rcvNxt, rcvWnd) +} + +// maybeSendOutOfWindowAck sends an ACK if we are not being rate limited +// currently. +// +checklocks:s.ep.mu +func (s *sender) maybeSendOutOfWindowAck(seg *segment) { + // Data packets are unlikely to be part of an ACK loop. So always send + // an ACK for a packet w/ data. + if seg.payloadSize() > 0 || s.ep.allowOutOfWindowAck() { + s.sendAck() + } +} + +func (s *sender) updateWriteNext(seg *segment) { + if s.writeNext != nil { + s.writeNext.DecRef() + } + if seg != nil { + seg.IncRef() + } + s.writeNext = seg +} + +// corkTimerExpired drains all the segments when TCP_CORK is enabled. +// +checklocks:s.ep.mu +func (s *sender) corkTimerExpired() tcpip.Error { + // Check if the timer actually expired or if it's a spurious wake due + // to a previously orphaned runtime timer. + if s.corkTimer.isUninitialized() || !s.corkTimer.checkExpiration() { + return nil + } + + // Assign sequence number and flags to the segment. + seg := s.writeNext + if seg == nil { + return nil + } + seg.sequenceNumber = s.SndNxt + seg.flags = header.TCPFlagAck | header.TCPFlagPsh + // Drain all the segments. + s.sendData() + return nil +} diff --git a/pkg/tcpip/transport/tcp/snd_queue_mutex.go b/pkg/tcpip/transport/tcp/snd_queue_mutex.go new file mode 100644 index 0000000..7984134 --- /dev/null +++ b/pkg/tcpip/transport/tcp/snd_queue_mutex.go @@ -0,0 +1,64 @@ +package tcp + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type sndQueueMutex struct { + mu sync.Mutex +} + +var sndQueueprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var sndQueuelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type sndQueuelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *sndQueueMutex) Lock() { + locking.AddGLock(sndQueueprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *sndQueueMutex) NestedLock(i sndQueuelockNameIndex) { + locking.AddGLock(sndQueueprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *sndQueueMutex) Unlock() { + locking.DelGLock(sndQueueprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *sndQueueMutex) NestedUnlock(i sndQueuelockNameIndex) { + locking.DelGLock(sndQueueprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func sndQueueinitLockNames() {} + +func init() { + sndQueueinitLockNames() + sndQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(sndQueueMutex{}), sndQueuelockNames) +} diff --git a/pkg/tcpip/transport/tcp/state.go b/pkg/tcpip/transport/tcp/state.go new file mode 100644 index 0000000..90ef115 --- /dev/null +++ b/pkg/tcpip/transport/tcp/state.go @@ -0,0 +1,480 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/internal/tcp" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +// TCPProbeFunc is the expected function type for a TCP probe function to be +// passed to stack.AddTCPProbe. +type TCPProbeFunc func(s *TCPEndpointState) + +// TCPCubicState is used to hold a copy of the internal cubic state when the +// TCPProbeFunc is invoked. +// +// +stateify savable +type TCPCubicState struct { + // WLastMax is the previous wMax value. + WLastMax float64 + + // WMax is the value of the congestion window at the time of the last + // congestion event. + WMax float64 + + // T is the time when the current congestion avoidance was entered. + T tcpip.MonotonicTime + + // TimeSinceLastCongestion denotes the time since the current + // congestion avoidance was entered. + TimeSinceLastCongestion time.Duration + + // C is the cubic constant as specified in RFC8312, page 11. + C float64 + + // K is the time period (in seconds) that the above function takes to + // increase the current window size to WMax if there are no further + // congestion events and is calculated using the following equation: + // + // K = cubic_root(WMax*(1-beta_cubic)/C) (Eq. 2, page 5) + K float64 + + // Beta is the CUBIC multiplication decrease factor. That is, when a + // congestion event is detected, CUBIC reduces its cwnd to + // WC(0)=WMax*beta_cubic. + Beta float64 + + // WC is window computed by CUBIC at time TimeSinceLastCongestion. It's + // calculated using the formula: + // + // WC(TimeSinceLastCongestion) = C*(t-K)^3 + WMax (Eq. 1) + WC float64 + + // WEst is the window computed by CUBIC at time + // TimeSinceLastCongestion+RTT i.e WC(TimeSinceLastCongestion+RTT). + WEst float64 + + // EndSeq is the sequence number that, when cumulatively ACK'd, ends the + // HyStart round. + EndSeq seqnum.Value + + // CurrRTT is the minimum round-trip time from the current round. + CurrRTT time.Duration + + // LastRTT is the minimum round-trip time from the previous round. + LastRTT time.Duration + + // SampleCount is the number of samples from the current round. + SampleCount uint + + // LastAck is the time we received the most recent ACK (or start of round if + // more recent). + LastAck tcpip.MonotonicTime + + // RoundStart is the time we started the most recent HyStart round. + RoundStart tcpip.MonotonicTime +} + +// TCPRACKState is used to hold a copy of the internal RACK state when the +// TCPProbeFunc is invoked. +// +// +stateify savable +type TCPRACKState struct { + // XmitTime is the transmission timestamp of the most recent + // acknowledged segment. + XmitTime tcpip.MonotonicTime + + // EndSequence is the ending TCP sequence number of the most recent + // acknowledged segment. + EndSequence seqnum.Value + + // FACK is the highest selectively or cumulatively acknowledged + // sequence. + FACK seqnum.Value + + // RTT is the round trip time of the most recently delivered packet on + // the connection (either cumulatively acknowledged or selectively + // acknowledged) that was not marked invalid as a possible spurious + // retransmission. + RTT time.Duration + + // Reord is true iff reordering has been detected on this connection. + Reord bool + + // DSACKSeen is true iff the connection has seen a DSACK. + DSACKSeen bool + + // ReoWnd is the reordering window time used for recording packet + // transmission times. It is used to defer the moment at which RACK + // marks a packet lost. + ReoWnd time.Duration + + // ReoWndIncr is the multiplier applied to adjust reorder window. + ReoWndIncr uint8 + + // ReoWndPersist is the number of loss recoveries before resetting + // reorder window. + ReoWndPersist int8 + + // RTTSeq is the SND.NXT when RTT is updated. + RTTSeq seqnum.Value +} + +// TCPEndpointID is the unique 4 tuple that identifies a given endpoint. +// +// +stateify savable +type TCPEndpointID struct { + // LocalPort is the local port associated with the endpoint. + LocalPort uint16 + + // LocalAddress is the local [network layer] address associated with + // the endpoint. + LocalAddress tcpip.Address + + // RemotePort is the remote port associated with the endpoint. + RemotePort uint16 + + // RemoteAddress it the remote [network layer] address associated with + // the endpoint. + RemoteAddress tcpip.Address +} + +// TCPFastRecoveryState holds a copy of the internal fast recovery state of a +// TCP endpoint. +// +// +stateify savable +type TCPFastRecoveryState struct { + // Active if true indicates the endpoint is in fast recovery. The + // following fields are only meaningful when Active is true. + Active bool + + // First is the first unacknowledged sequence number being recovered. + First seqnum.Value + + // Last is the 'recover' sequence number that indicates the point at + // which we should exit recovery barring any timeouts etc. + Last seqnum.Value + + // MaxCwnd is the maximum value we are permitted to grow the congestion + // window during recovery. This is set at the time we enter recovery. + // It exists to avoid attacks where the receiver intentionally sends + // duplicate acks to artificially inflate the sender's cwnd. + MaxCwnd int + + // HighRxt is the highest sequence number which has been retransmitted + // during the current loss recovery phase. See: RFC 6675 Section 2 for + // details. + HighRxt seqnum.Value + + // RescueRxt is the highest sequence number which has been + // optimistically retransmitted to prevent stalling of the ACK clock + // when there is loss at the end of the window and no new data is + // available for transmission. See: RFC 6675 Section 2 for details. + RescueRxt seqnum.Value +} + +// TCPReceiverState holds a copy of the internal state of the receiver for a +// given TCP endpoint. +// +// +stateify savable +type TCPReceiverState struct { + // RcvNxt is the TCP variable RCV.NXT. + RcvNxt seqnum.Value + + // RcvAcc is one beyond the last acceptable sequence number. That is, + // the "largest" sequence value that the receiver has announced to its + // peer that it's willing to accept. This may be different than RcvNxt + // + (last advertised receive window) if the receive window is reduced; + // in that case we have to reduce the window as we receive more data + // instead of shrinking it. + RcvAcc seqnum.Value + + // RcvWndScale is the window scaling to use for inbound segments. + RcvWndScale uint8 + + // PendingBufUsed is the number of bytes pending in the receive queue. + PendingBufUsed int +} + +// TCPRTTState holds a copy of information about the endpoint's round trip +// time. +// +// +stateify savable +type TCPRTTState struct { + // SRTT is the smoothed round trip time defined in section 2 of RFC + // 6298. + SRTT time.Duration + + // RTTVar is the round-trip time variation as defined in section 2 of + // RFC 6298. + RTTVar time.Duration + + // SRTTInited if true indicates that a valid RTT measurement has been + // completed. + SRTTInited bool +} + +// TCPSenderState holds a copy of the internal state of the sender for a given +// TCP Endpoint. +// +// +stateify savable +type TCPSenderState struct { + // LastSendTime is the timestamp at which we sent the last segment. + LastSendTime tcpip.MonotonicTime + + // DupAckCount is the number of Duplicate ACKs received. It is used for + // fast retransmit. + DupAckCount int + + // SndCwnd is the size of the sending congestion window in packets. + SndCwnd int + + // Ssthresh is the threshold between slow start and congestion + // avoidance. + Ssthresh int + + // SndCAAckCount is the number of packets acknowledged during + // congestion avoidance. When enough packets have been ack'd (typically + // cwnd packets), the congestion window is incremented by one. + SndCAAckCount int + + // Outstanding is the number of packets that have been sent but not yet + // acknowledged. + Outstanding int + + // SackedOut is the number of packets which have been selectively + // acked. + SackedOut int + + // SndWnd is the send window size in bytes. + SndWnd seqnum.Size + + // SndUna is the next unacknowledged sequence number. + SndUna seqnum.Value + + // SndNxt is the sequence number of the next segment to be sent. + SndNxt seqnum.Value + + // RTTMeasureSeqNum is the sequence number being used for the latest + // RTT measurement. + RTTMeasureSeqNum seqnum.Value + + // RTTMeasureTime is the time when the RTTMeasureSeqNum was sent. + RTTMeasureTime tcpip.MonotonicTime + + // Closed indicates that the caller has closed the endpoint for + // sending. + Closed bool + + // RTO is the retransmit timeout as defined in section of 2 of RFC + // 6298. + RTO time.Duration + + // RTTState holds information about the endpoint's round trip time. + RTTState TCPRTTState + + // MaxPayloadSize is the maximum size of the payload of a given + // segment. It is initialized on demand. + MaxPayloadSize int + + // SndWndScale is the number of bits to shift left when reading the + // send window size from a segment. + SndWndScale uint8 + + // MaxSentAck is the highest acknowledgement number sent till now. + MaxSentAck seqnum.Value + + // FastRecovery holds the fast recovery state for the endpoint. + FastRecovery TCPFastRecoveryState + + // Cubic holds the state related to CUBIC congestion control. + Cubic TCPCubicState + + // RACKState holds the state related to RACK loss detection algorithm. + RACKState TCPRACKState + + // RetransmitTS records the timestamp used to detect spurious recovery. + RetransmitTS uint32 + + // SpuriousRecovery indicates if the sender entered recovery spuriously. + SpuriousRecovery bool +} + +// TCPSACKInfo holds TCP SACK related information for a given TCP endpoint. +// +// +stateify savable +type TCPSACKInfo struct { + // Blocks is the list of SACK Blocks that identify the out of order + // segments held by a given TCP endpoint. + Blocks []header.SACKBlock + + // ReceivedBlocks are the SACK blocks received by this endpoint from + // the peer endpoint. + ReceivedBlocks []header.SACKBlock + + // MaxSACKED is the highest sequence number that has been SACKED by the + // peer. + MaxSACKED seqnum.Value +} + +// RcvBufAutoTuneParams holds state related to TCP receive buffer auto-tuning. +// +// +stateify savable +type RcvBufAutoTuneParams struct { + // MeasureTime is the time at which the current measurement was + // started. + MeasureTime tcpip.MonotonicTime + + // CopiedBytes is the number of bytes copied to user space since this + // measure began. + CopiedBytes int + + // PrevCopiedBytes is the number of bytes copied to userspace in the + // previous RTT period. + PrevCopiedBytes int + + // RcvBufSize is the auto tuned receive buffer size. + RcvBufSize int + + // RTT is the smoothed RTT as measured by observing the time between + // when a byte is first acknowledged and the receipt of data that is at + // least one window beyond the sequence number that was acknowledged. + RTT time.Duration + + // RTTVar is the "round-trip time variation" as defined in section 2 of + // RFC6298. + RTTVar time.Duration + + // RTTMeasureSeqNumber is the highest acceptable sequence number at the + // time this RTT measurement period began. + RTTMeasureSeqNumber seqnum.Value + + // RTTMeasureTime is the absolute time at which the current RTT + // measurement period began. + RTTMeasureTime tcpip.MonotonicTime + + // Disabled is true if an explicit receive buffer is set for the + // endpoint. + Disabled bool +} + +// TCPRcvBufState contains information about the state of an endpoint's receive +// socket buffer. +// +// +stateify savable +type TCPRcvBufState struct { + // RcvBufUsed is the amount of bytes actually held in the receive + // socket buffer for the endpoint. + RcvBufUsed int + + // RcvBufAutoTuneParams is used to hold state variables to compute the + // auto tuned receive buffer size. + RcvAutoParams RcvBufAutoTuneParams + + // RcvClosed if true, indicates the endpoint has been closed for + // reading. + RcvClosed bool +} + +// TCPSndBufState contains information about the state of an endpoint's send +// socket buffer. +// +// +stateify savable +type TCPSndBufState struct { + // SndBufSize is the size of the socket send buffer. + SndBufSize int + + // SndBufUsed is the number of bytes held in the socket send buffer. + SndBufUsed int + + // SndClosed indicates that the endpoint has been closed for sends. + SndClosed bool + + // PacketTooBigCount is used to notify the main protocol routine how + // many times a "packet too big" control packet is received. + PacketTooBigCount int + + // SndMTU is the smallest MTU seen in the control packets received. + SndMTU int + + // AutoTuneSndBufDisabled indicates that the auto tuning of send buffer + // is disabled. + AutoTuneSndBufDisabled atomicbitops.Uint32 +} + +// TCPEndpointStateInner contains the members of TCPEndpointState used directly +// (that is, not within another containing struct) within the endpoint's +// internal implementation. +// +// +stateify savable +type TCPEndpointStateInner struct { + // TSOffset is a randomized offset added to the value of the TSVal + // field in the timestamp option. + TSOffset tcp.TSOffset + + // SACKPermitted is set to true if the peer sends the TCPSACKPermitted + // option in the SYN/SYN-ACK. + SACKPermitted bool + + // SendTSOk is used to indicate when the TS Option has been negotiated. + // When sendTSOk is true every non-RST segment should carry a TS as per + // RFC7323#section-1.1. + SendTSOk bool + + // RecentTS is the timestamp that should be sent in the TSEcr field of + // the timestamp for future segments sent by the endpoint. This field + // is updated if required when a new segment is received by this + // endpoint. + RecentTS uint32 +} + +// TCPEndpointState is a copy of the internal state of a TCP endpoint. +// +// +stateify savable +type TCPEndpointState struct { + // TCPEndpointStateInner contains the members of TCPEndpointState used + // by the endpoint's internal implementation. + TCPEndpointStateInner + + // ID is a copy of the TransportEndpointID for the endpoint. + ID TCPEndpointID + + // SegTime denotes the absolute time when this segment was received. + SegTime tcpip.MonotonicTime + + // RcvBufState contains information about the state of the endpoint's + // receive socket buffer. + RcvBufState TCPRcvBufState + + // SndBufState contains information about the state of the endpoint's + // send socket buffer. + SndBufState TCPSndBufState + + // SACK holds TCP SACK related information for this endpoint. + SACK TCPSACKInfo + + // Receiver holds variables related to the TCP receiver for the + // endpoint. + Receiver TCPReceiverState + + // Sender holds state related to the TCP Sender for the endpoint. + Sender TCPSenderState +} diff --git a/pkg/tcpip/transport/tcp/tcp_endpoint_list.go b/pkg/tcpip/transport/tcp/tcp_endpoint_list.go new file mode 100644 index 0000000..67bfa99 --- /dev/null +++ b/pkg/tcpip/transport/tcp/tcp_endpoint_list.go @@ -0,0 +1,239 @@ +package tcp + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type endpointElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (endpointElementMapper) linkerFor(elem *Endpoint) *Endpoint { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type endpointList struct { + head *Endpoint + tail *Endpoint +} + +// Reset resets list l to the empty state. +func (l *endpointList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *endpointList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *endpointList) Front() *Endpoint { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *endpointList) Back() *Endpoint { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *endpointList) Len() (count int) { + for e := l.Front(); e != nil; e = (endpointElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *endpointList) PushFront(e *Endpoint) { + linker := endpointElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + endpointElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *endpointList) PushFrontList(m *endpointList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + endpointElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + endpointElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *endpointList) PushBack(e *Endpoint) { + linker := endpointElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + endpointElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *endpointList) PushBackList(m *endpointList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + endpointElementMapper{}.linkerFor(l.tail).SetNext(m.head) + endpointElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *endpointList) InsertAfter(b, e *Endpoint) { + bLinker := endpointElementMapper{}.linkerFor(b) + eLinker := endpointElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + endpointElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *endpointList) InsertBefore(a, e *Endpoint) { + aLinker := endpointElementMapper{}.linkerFor(a) + eLinker := endpointElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + endpointElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *endpointList) Remove(e *Endpoint) { + linker := endpointElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + endpointElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + endpointElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type endpointEntry struct { + next *Endpoint + prev *Endpoint +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *endpointEntry) Next() *Endpoint { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *endpointEntry) Prev() *Endpoint { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *endpointEntry) SetNext(elem *Endpoint) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *endpointEntry) SetPrev(elem *Endpoint) { + e.prev = elem +} diff --git a/pkg/tcpip/transport/tcp/tcp_segment_list.go b/pkg/tcpip/transport/tcp/tcp_segment_list.go new file mode 100644 index 0000000..770adb4 --- /dev/null +++ b/pkg/tcpip/transport/tcp/tcp_segment_list.go @@ -0,0 +1,239 @@ +package tcp + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type segmentElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (segmentElementMapper) linkerFor(elem *segment) *segment { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type segmentList struct { + head *segment + tail *segment +} + +// Reset resets list l to the empty state. +func (l *segmentList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *segmentList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *segmentList) Front() *segment { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *segmentList) Back() *segment { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *segmentList) Len() (count int) { + for e := l.Front(); e != nil; e = (segmentElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *segmentList) PushFront(e *segment) { + linker := segmentElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + segmentElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *segmentList) PushFrontList(m *segmentList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + segmentElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + segmentElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *segmentList) PushBack(e *segment) { + linker := segmentElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + segmentElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *segmentList) PushBackList(m *segmentList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + segmentElementMapper{}.linkerFor(l.tail).SetNext(m.head) + segmentElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *segmentList) InsertAfter(b, e *segment) { + bLinker := segmentElementMapper{}.linkerFor(b) + eLinker := segmentElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + segmentElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *segmentList) InsertBefore(a, e *segment) { + aLinker := segmentElementMapper{}.linkerFor(a) + eLinker := segmentElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + segmentElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *segmentList) Remove(e *segment) { + linker := segmentElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + segmentElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + segmentElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type segmentEntry struct { + next *segment + prev *segment +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *segmentEntry) Next() *segment { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *segmentEntry) Prev() *segment { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *segmentEntry) SetNext(elem *segment) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *segmentEntry) SetPrev(elem *segment) { + e.prev = elem +} diff --git a/pkg/tcpip/transport/tcp/tcp_segment_refs.go b/pkg/tcpip/transport/tcp/tcp_segment_refs.go new file mode 100644 index 0000000..9ad0201 --- /dev/null +++ b/pkg/tcpip/transport/tcp/tcp_segment_refs.go @@ -0,0 +1,141 @@ +package tcp + +import ( + "context" + "fmt" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/refs" +) + +// enableLogging indicates whether reference-related events should be logged (with +// stack traces). This is false by default and should only be set to true for +// debugging purposes, as it can generate an extremely large amount of output +// and drastically degrade performance. +const segmentenableLogging = false + +// obj is used to customize logging. Note that we use a pointer to T so that +// we do not copy the entire object when passed as a format parameter. +var segmentobj *segment + +// Refs implements refs.RefCounter. It keeps a reference count using atomic +// operations and calls the destructor when the count reaches zero. +// +// NOTE: Do not introduce additional fields to the Refs struct. It is used by +// many filesystem objects, and we want to keep it as small as possible (i.e., +// the same size as using an int64 directly) to avoid taking up extra cache +// space. In general, this template should not be extended at the cost of +// performance. If it does not offer enough flexibility for a particular object +// (example: b/187877947), we should implement the RefCounter/CheckedObject +// interfaces manually. +// +// +stateify savable +type segmentRefs struct { + // refCount is composed of two fields: + // + // [32-bit speculative references]:[32-bit real references] + // + // Speculative references are used for TryIncRef, to avoid a CompareAndSwap + // loop. See IncRef, DecRef and TryIncRef for details of how these fields are + // used. + refCount atomicbitops.Int64 +} + +// InitRefs initializes r with one reference and, if enabled, activates leak +// checking. +func (r *segmentRefs) InitRefs() { + r.refCount.RacyStore(1) + refs.Register(r) +} + +// RefType implements refs.CheckedObject.RefType. +func (r *segmentRefs) RefType() string { + return fmt.Sprintf("%T", segmentobj)[1:] +} + +// LeakMessage implements refs.CheckedObject.LeakMessage. +func (r *segmentRefs) LeakMessage() string { + return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs()) +} + +// LogRefs implements refs.CheckedObject.LogRefs. +func (r *segmentRefs) LogRefs() bool { + return segmentenableLogging +} + +// ReadRefs returns the current number of references. The returned count is +// inherently racy and is unsafe to use without external synchronization. +func (r *segmentRefs) ReadRefs() int64 { + return r.refCount.Load() +} + +// IncRef implements refs.RefCounter.IncRef. +// +//go:nosplit +func (r *segmentRefs) IncRef() { + v := r.refCount.Add(1) + if segmentenableLogging { + refs.LogIncRef(r, v) + } + if v <= 1 { + panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType())) + } +} + +// TryIncRef implements refs.TryRefCounter.TryIncRef. +// +// To do this safely without a loop, a speculative reference is first acquired +// on the object. This allows multiple concurrent TryIncRef calls to distinguish +// other TryIncRef calls from genuine references held. +// +//go:nosplit +func (r *segmentRefs) TryIncRef() bool { + const speculativeRef = 1 << 32 + if v := r.refCount.Add(speculativeRef); int32(v) == 0 { + + r.refCount.Add(-speculativeRef) + return false + } + + v := r.refCount.Add(-speculativeRef + 1) + if segmentenableLogging { + refs.LogTryIncRef(r, v) + } + return true +} + +// DecRef implements refs.RefCounter.DecRef. +// +// Note that speculative references are counted here. Since they were added +// prior to real references reaching zero, they will successfully convert to +// real references. In other words, we see speculative references only in the +// following case: +// +// A: TryIncRef [speculative increase => sees non-negative references] +// B: DecRef [real decrease] +// A: TryIncRef [transform speculative to real] +// +//go:nosplit +func (r *segmentRefs) DecRef(destroy func()) { + v := r.refCount.Add(-1) + if segmentenableLogging { + refs.LogDecRef(r, v) + } + switch { + case v < 0: + panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType())) + + case v == 0: + refs.Unregister(r) + + if destroy != nil { + destroy() + } + } +} + +func (r *segmentRefs) afterLoad(context.Context) { + if r.ReadRefs() > 0 { + refs.Register(r) + } +} diff --git a/pkg/tcpip/transport/tcp/tcp_state_autogen.go b/pkg/tcpip/transport/tcp/tcp_state_autogen.go new file mode 100644 index 0000000..2dda131 --- /dev/null +++ b/pkg/tcpip/transport/tcp/tcp_state_autogen.go @@ -0,0 +1,1935 @@ +// automatically generated by stateify. + +package tcp + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (a *acceptQueue) StateTypeName() string { + return "pkg/tcpip/transport/tcp.acceptQueue" +} + +func (a *acceptQueue) StateFields() []string { + return []string{ + "endpoints", + "pendingEndpoints", + "capacity", + } +} + +func (a *acceptQueue) beforeSave() {} + +// +checklocksignore +func (a *acceptQueue) StateSave(stateSinkObject state.Sink) { + a.beforeSave() + var endpointsValue []*Endpoint + endpointsValue = a.saveEndpoints() + stateSinkObject.SaveValue(0, endpointsValue) + stateSinkObject.Save(1, &a.pendingEndpoints) + stateSinkObject.Save(2, &a.capacity) +} + +func (a *acceptQueue) afterLoad(context.Context) {} + +// +checklocksignore +func (a *acceptQueue) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(1, &a.pendingEndpoints) + stateSourceObject.Load(2, &a.capacity) + stateSourceObject.LoadValue(0, new([]*Endpoint), func(y any) { a.loadEndpoints(ctx, y.([]*Endpoint)) }) +} + +func (h *handshake) StateTypeName() string { + return "pkg/tcpip/transport/tcp.handshake" +} + +func (h *handshake) StateFields() []string { + return []string{ + "ep", + "listenEP", + "state", + "active", + "flags", + "ackNum", + "iss", + "rcvWnd", + "sndWnd", + "mss", + "sndWndScale", + "rcvWndScale", + "startTime", + "deferAccept", + "acked", + "sendSYNOpts", + "sampleRTTWithTSOnly", + } +} + +func (h *handshake) beforeSave() {} + +// +checklocksignore +func (h *handshake) StateSave(stateSinkObject state.Sink) { + h.beforeSave() + stateSinkObject.Save(0, &h.ep) + stateSinkObject.Save(1, &h.listenEP) + stateSinkObject.Save(2, &h.state) + stateSinkObject.Save(3, &h.active) + stateSinkObject.Save(4, &h.flags) + stateSinkObject.Save(5, &h.ackNum) + stateSinkObject.Save(6, &h.iss) + stateSinkObject.Save(7, &h.rcvWnd) + stateSinkObject.Save(8, &h.sndWnd) + stateSinkObject.Save(9, &h.mss) + stateSinkObject.Save(10, &h.sndWndScale) + stateSinkObject.Save(11, &h.rcvWndScale) + stateSinkObject.Save(12, &h.startTime) + stateSinkObject.Save(13, &h.deferAccept) + stateSinkObject.Save(14, &h.acked) + stateSinkObject.Save(15, &h.sendSYNOpts) + stateSinkObject.Save(16, &h.sampleRTTWithTSOnly) +} + +func (h *handshake) afterLoad(context.Context) {} + +// +checklocksignore +func (h *handshake) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &h.ep) + stateSourceObject.Load(1, &h.listenEP) + stateSourceObject.Load(2, &h.state) + stateSourceObject.Load(3, &h.active) + stateSourceObject.Load(4, &h.flags) + stateSourceObject.Load(5, &h.ackNum) + stateSourceObject.Load(6, &h.iss) + stateSourceObject.Load(7, &h.rcvWnd) + stateSourceObject.Load(8, &h.sndWnd) + stateSourceObject.Load(9, &h.mss) + stateSourceObject.Load(10, &h.sndWndScale) + stateSourceObject.Load(11, &h.rcvWndScale) + stateSourceObject.Load(12, &h.startTime) + stateSourceObject.Load(13, &h.deferAccept) + stateSourceObject.Load(14, &h.acked) + stateSourceObject.Load(15, &h.sendSYNOpts) + stateSourceObject.Load(16, &h.sampleRTTWithTSOnly) +} + +func (c *cubicState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.cubicState" +} + +func (c *cubicState) StateFields() []string { + return []string{ + "TCPCubicState", + "numCongestionEvents", + "s", + } +} + +func (c *cubicState) beforeSave() {} + +// +checklocksignore +func (c *cubicState) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.TCPCubicState) + stateSinkObject.Save(1, &c.numCongestionEvents) + stateSinkObject.Save(2, &c.s) +} + +func (c *cubicState) afterLoad(context.Context) {} + +// +checklocksignore +func (c *cubicState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.TCPCubicState) + stateSourceObject.Load(1, &c.numCongestionEvents) + stateSourceObject.Load(2, &c.s) +} + +func (q *epQueue) StateTypeName() string { + return "pkg/tcpip/transport/tcp.epQueue" +} + +func (q *epQueue) StateFields() []string { + return []string{ + "list", + } +} + +func (q *epQueue) beforeSave() {} + +// +checklocksignore +func (q *epQueue) StateSave(stateSinkObject state.Sink) { + q.beforeSave() + stateSinkObject.Save(0, &q.list) +} + +func (q *epQueue) afterLoad(context.Context) {} + +// +checklocksignore +func (q *epQueue) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &q.list) +} + +func (p *processor) StateTypeName() string { + return "pkg/tcpip/transport/tcp.processor" +} + +func (p *processor) StateFields() []string { + return []string{ + "epQ", + } +} + +func (p *processor) beforeSave() {} + +// +checklocksignore +func (p *processor) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.epQ) +} + +func (p *processor) afterLoad(context.Context) {} + +// +checklocksignore +func (p *processor) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.epQ) +} + +func (d *dispatcher) StateTypeName() string { + return "pkg/tcpip/transport/tcp.dispatcher" +} + +func (d *dispatcher) StateFields() []string { + return []string{ + "processors", + "hasher", + "paused", + "closed", + } +} + +func (d *dispatcher) beforeSave() {} + +// +checklocksignore +func (d *dispatcher) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.processors) + stateSinkObject.Save(1, &d.hasher) + stateSinkObject.Save(2, &d.paused) + stateSinkObject.Save(3, &d.closed) +} + +func (d *dispatcher) afterLoad(context.Context) {} + +// +checklocksignore +func (d *dispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.processors) + stateSourceObject.Load(1, &d.hasher) + stateSourceObject.Load(2, &d.paused) + stateSourceObject.Load(3, &d.closed) +} + +func (j *jenkinsHasher) StateTypeName() string { + return "pkg/tcpip/transport/tcp.jenkinsHasher" +} + +func (j *jenkinsHasher) StateFields() []string { + return []string{ + "seed", + } +} + +func (j *jenkinsHasher) beforeSave() {} + +// +checklocksignore +func (j *jenkinsHasher) StateSave(stateSinkObject state.Sink) { + j.beforeSave() + stateSinkObject.Save(0, &j.seed) +} + +func (j *jenkinsHasher) afterLoad(context.Context) {} + +// +checklocksignore +func (j *jenkinsHasher) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &j.seed) +} + +func (s *SACKInfo) StateTypeName() string { + return "pkg/tcpip/transport/tcp.SACKInfo" +} + +func (s *SACKInfo) StateFields() []string { + return []string{ + "Blocks", + "NumBlocks", + } +} + +func (s *SACKInfo) beforeSave() {} + +// +checklocksignore +func (s *SACKInfo) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.Blocks) + stateSinkObject.Save(1, &s.NumBlocks) +} + +func (s *SACKInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SACKInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.Blocks) + stateSourceObject.Load(1, &s.NumBlocks) +} + +func (r *ReceiveErrors) StateTypeName() string { + return "pkg/tcpip/transport/tcp.ReceiveErrors" +} + +func (r *ReceiveErrors) StateFields() []string { + return []string{ + "ReceiveErrors", + "SegmentQueueDropped", + "ChecksumErrors", + "ListenOverflowSynDrop", + "ListenOverflowAckDrop", + "ZeroRcvWindowState", + "WantZeroRcvWindow", + } +} + +func (r *ReceiveErrors) beforeSave() {} + +// +checklocksignore +func (r *ReceiveErrors) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.ReceiveErrors) + stateSinkObject.Save(1, &r.SegmentQueueDropped) + stateSinkObject.Save(2, &r.ChecksumErrors) + stateSinkObject.Save(3, &r.ListenOverflowSynDrop) + stateSinkObject.Save(4, &r.ListenOverflowAckDrop) + stateSinkObject.Save(5, &r.ZeroRcvWindowState) + stateSinkObject.Save(6, &r.WantZeroRcvWindow) +} + +func (r *ReceiveErrors) afterLoad(context.Context) {} + +// +checklocksignore +func (r *ReceiveErrors) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.ReceiveErrors) + stateSourceObject.Load(1, &r.SegmentQueueDropped) + stateSourceObject.Load(2, &r.ChecksumErrors) + stateSourceObject.Load(3, &r.ListenOverflowSynDrop) + stateSourceObject.Load(4, &r.ListenOverflowAckDrop) + stateSourceObject.Load(5, &r.ZeroRcvWindowState) + stateSourceObject.Load(6, &r.WantZeroRcvWindow) +} + +func (s *SendErrors) StateTypeName() string { + return "pkg/tcpip/transport/tcp.SendErrors" +} + +func (s *SendErrors) StateFields() []string { + return []string{ + "SendErrors", + "SegmentSendToNetworkFailed", + "SynSendToNetworkFailed", + "Retransmits", + "FastRetransmit", + "Timeouts", + } +} + +func (s *SendErrors) beforeSave() {} + +// +checklocksignore +func (s *SendErrors) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.SendErrors) + stateSinkObject.Save(1, &s.SegmentSendToNetworkFailed) + stateSinkObject.Save(2, &s.SynSendToNetworkFailed) + stateSinkObject.Save(3, &s.Retransmits) + stateSinkObject.Save(4, &s.FastRetransmit) + stateSinkObject.Save(5, &s.Timeouts) +} + +func (s *SendErrors) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SendErrors) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.SendErrors) + stateSourceObject.Load(1, &s.SegmentSendToNetworkFailed) + stateSourceObject.Load(2, &s.SynSendToNetworkFailed) + stateSourceObject.Load(3, &s.Retransmits) + stateSourceObject.Load(4, &s.FastRetransmit) + stateSourceObject.Load(5, &s.Timeouts) +} + +func (s *Stats) StateTypeName() string { + return "pkg/tcpip/transport/tcp.Stats" +} + +func (s *Stats) StateFields() []string { + return []string{ + "SegmentsReceived", + "SegmentsSent", + "FailedConnectionAttempts", + "ReceiveErrors", + "ReadErrors", + "SendErrors", + "WriteErrors", + } +} + +func (s *Stats) beforeSave() {} + +// +checklocksignore +func (s *Stats) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.SegmentsReceived) + stateSinkObject.Save(1, &s.SegmentsSent) + stateSinkObject.Save(2, &s.FailedConnectionAttempts) + stateSinkObject.Save(3, &s.ReceiveErrors) + stateSinkObject.Save(4, &s.ReadErrors) + stateSinkObject.Save(5, &s.SendErrors) + stateSinkObject.Save(6, &s.WriteErrors) +} + +func (s *Stats) afterLoad(context.Context) {} + +// +checklocksignore +func (s *Stats) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.SegmentsReceived) + stateSourceObject.Load(1, &s.SegmentsSent) + stateSourceObject.Load(2, &s.FailedConnectionAttempts) + stateSourceObject.Load(3, &s.ReceiveErrors) + stateSourceObject.Load(4, &s.ReadErrors) + stateSourceObject.Load(5, &s.SendErrors) + stateSourceObject.Load(6, &s.WriteErrors) +} + +func (sq *sndQueueInfo) StateTypeName() string { + return "pkg/tcpip/transport/tcp.sndQueueInfo" +} + +func (sq *sndQueueInfo) StateFields() []string { + return []string{ + "TCPSndBufState", + } +} + +func (sq *sndQueueInfo) beforeSave() {} + +// +checklocksignore +func (sq *sndQueueInfo) StateSave(stateSinkObject state.Sink) { + sq.beforeSave() + stateSinkObject.Save(0, &sq.TCPSndBufState) +} + +func (sq *sndQueueInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (sq *sndQueueInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &sq.TCPSndBufState) +} + +func (e *Endpoint) StateTypeName() string { + return "pkg/tcpip/transport/tcp.Endpoint" +} + +func (e *Endpoint) StateFields() []string { + return []string{ + "TCPEndpointStateInner", + "TransportEndpointInfo", + "DefaultSocketOptionsHandler", + "stack", + "protocol", + "waiterQueue", + "hardError", + "lastError", + "TCPRcvBufState", + "rcvMemUsed", + "ownedByUser", + "rcvQueue", + "state", + "connectionDirectionState", + "isPortReserved", + "isRegistered", + "boundNICID", + "ipv4TTL", + "ipv6HopLimit", + "isConnectNotified", + "h", + "portFlags", + "boundBindToDevice", + "boundPortFlags", + "boundDest", + "effectiveNetProtos", + "recentTSTime", + "shutdownFlags", + "tcpRecovery", + "sack", + "delay", + "scoreboard", + "segmentQueue", + "userMSS", + "maxSynRetries", + "windowClamp", + "sndQueueInfo", + "cc", + "keepalive", + "userTimeout", + "deferAccept", + "acceptQueue", + "rcv", + "snd", + "connectingAddress", + "amss", + "sendTOS", + "gso", + "stats", + "tcpLingerTimeout", + "closed", + "txHash", + "owner", + "ops", + "lastOutOfWindowAckTime", + "pmtud", + "alsoBindToV4", + } +} + +// +checklocksignore +func (e *Endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + var stateValue EndpointState + stateValue = e.saveState() + stateSinkObject.SaveValue(12, stateValue) + stateSinkObject.Save(0, &e.TCPEndpointStateInner) + stateSinkObject.Save(1, &e.TransportEndpointInfo) + stateSinkObject.Save(2, &e.DefaultSocketOptionsHandler) + stateSinkObject.Save(3, &e.stack) + stateSinkObject.Save(4, &e.protocol) + stateSinkObject.Save(5, &e.waiterQueue) + stateSinkObject.Save(6, &e.hardError) + stateSinkObject.Save(7, &e.lastError) + stateSinkObject.Save(8, &e.TCPRcvBufState) + stateSinkObject.Save(9, &e.rcvMemUsed) + stateSinkObject.Save(10, &e.ownedByUser) + stateSinkObject.Save(11, &e.rcvQueue) + stateSinkObject.Save(13, &e.connectionDirectionState) + stateSinkObject.Save(14, &e.isPortReserved) + stateSinkObject.Save(15, &e.isRegistered) + stateSinkObject.Save(16, &e.boundNICID) + stateSinkObject.Save(17, &e.ipv4TTL) + stateSinkObject.Save(18, &e.ipv6HopLimit) + stateSinkObject.Save(19, &e.isConnectNotified) + stateSinkObject.Save(20, &e.h) + stateSinkObject.Save(21, &e.portFlags) + stateSinkObject.Save(22, &e.boundBindToDevice) + stateSinkObject.Save(23, &e.boundPortFlags) + stateSinkObject.Save(24, &e.boundDest) + stateSinkObject.Save(25, &e.effectiveNetProtos) + stateSinkObject.Save(26, &e.recentTSTime) + stateSinkObject.Save(27, &e.shutdownFlags) + stateSinkObject.Save(28, &e.tcpRecovery) + stateSinkObject.Save(29, &e.sack) + stateSinkObject.Save(30, &e.delay) + stateSinkObject.Save(31, &e.scoreboard) + stateSinkObject.Save(32, &e.segmentQueue) + stateSinkObject.Save(33, &e.userMSS) + stateSinkObject.Save(34, &e.maxSynRetries) + stateSinkObject.Save(35, &e.windowClamp) + stateSinkObject.Save(36, &e.sndQueueInfo) + stateSinkObject.Save(37, &e.cc) + stateSinkObject.Save(38, &e.keepalive) + stateSinkObject.Save(39, &e.userTimeout) + stateSinkObject.Save(40, &e.deferAccept) + stateSinkObject.Save(41, &e.acceptQueue) + stateSinkObject.Save(42, &e.rcv) + stateSinkObject.Save(43, &e.snd) + stateSinkObject.Save(44, &e.connectingAddress) + stateSinkObject.Save(45, &e.amss) + stateSinkObject.Save(46, &e.sendTOS) + stateSinkObject.Save(47, &e.gso) + stateSinkObject.Save(48, &e.stats) + stateSinkObject.Save(49, &e.tcpLingerTimeout) + stateSinkObject.Save(50, &e.closed) + stateSinkObject.Save(51, &e.txHash) + stateSinkObject.Save(52, &e.owner) + stateSinkObject.Save(53, &e.ops) + stateSinkObject.Save(54, &e.lastOutOfWindowAckTime) + stateSinkObject.Save(55, &e.pmtud) + stateSinkObject.Save(56, &e.alsoBindToV4) +} + +// +checklocksignore +func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.TCPEndpointStateInner) + stateSourceObject.Load(1, &e.TransportEndpointInfo) + stateSourceObject.Load(2, &e.DefaultSocketOptionsHandler) + stateSourceObject.Load(3, &e.stack) + stateSourceObject.Load(4, &e.protocol) + stateSourceObject.LoadWait(5, &e.waiterQueue) + stateSourceObject.Load(6, &e.hardError) + stateSourceObject.Load(7, &e.lastError) + stateSourceObject.Load(8, &e.TCPRcvBufState) + stateSourceObject.Load(9, &e.rcvMemUsed) + stateSourceObject.Load(10, &e.ownedByUser) + stateSourceObject.LoadWait(11, &e.rcvQueue) + stateSourceObject.Load(13, &e.connectionDirectionState) + stateSourceObject.Load(14, &e.isPortReserved) + stateSourceObject.Load(15, &e.isRegistered) + stateSourceObject.Load(16, &e.boundNICID) + stateSourceObject.Load(17, &e.ipv4TTL) + stateSourceObject.Load(18, &e.ipv6HopLimit) + stateSourceObject.Load(19, &e.isConnectNotified) + stateSourceObject.Load(20, &e.h) + stateSourceObject.Load(21, &e.portFlags) + stateSourceObject.Load(22, &e.boundBindToDevice) + stateSourceObject.Load(23, &e.boundPortFlags) + stateSourceObject.Load(24, &e.boundDest) + stateSourceObject.Load(25, &e.effectiveNetProtos) + stateSourceObject.Load(26, &e.recentTSTime) + stateSourceObject.Load(27, &e.shutdownFlags) + stateSourceObject.Load(28, &e.tcpRecovery) + stateSourceObject.Load(29, &e.sack) + stateSourceObject.Load(30, &e.delay) + stateSourceObject.Load(31, &e.scoreboard) + stateSourceObject.LoadWait(32, &e.segmentQueue) + stateSourceObject.Load(33, &e.userMSS) + stateSourceObject.Load(34, &e.maxSynRetries) + stateSourceObject.Load(35, &e.windowClamp) + stateSourceObject.Load(36, &e.sndQueueInfo) + stateSourceObject.Load(37, &e.cc) + stateSourceObject.Load(38, &e.keepalive) + stateSourceObject.Load(39, &e.userTimeout) + stateSourceObject.Load(40, &e.deferAccept) + stateSourceObject.Load(41, &e.acceptQueue) + stateSourceObject.LoadWait(42, &e.rcv) + stateSourceObject.LoadWait(43, &e.snd) + stateSourceObject.Load(44, &e.connectingAddress) + stateSourceObject.Load(45, &e.amss) + stateSourceObject.Load(46, &e.sendTOS) + stateSourceObject.Load(47, &e.gso) + stateSourceObject.Load(48, &e.stats) + stateSourceObject.Load(49, &e.tcpLingerTimeout) + stateSourceObject.Load(50, &e.closed) + stateSourceObject.Load(51, &e.txHash) + stateSourceObject.Load(52, &e.owner) + stateSourceObject.Load(53, &e.ops) + stateSourceObject.Load(54, &e.lastOutOfWindowAckTime) + stateSourceObject.Load(55, &e.pmtud) + stateSourceObject.Load(56, &e.alsoBindToV4) + stateSourceObject.LoadValue(12, new(EndpointState), func(y any) { e.loadState(ctx, y.(EndpointState)) }) + stateSourceObject.AfterLoad(func() { e.afterLoad(ctx) }) +} + +func (k *keepalive) StateTypeName() string { + return "pkg/tcpip/transport/tcp.keepalive" +} + +func (k *keepalive) StateFields() []string { + return []string{ + "idle", + "interval", + "count", + "unacked", + } +} + +func (k *keepalive) beforeSave() {} + +// +checklocksignore +func (k *keepalive) StateSave(stateSinkObject state.Sink) { + k.beforeSave() + stateSinkObject.Save(0, &k.idle) + stateSinkObject.Save(1, &k.interval) + stateSinkObject.Save(2, &k.count) + stateSinkObject.Save(3, &k.unacked) +} + +func (k *keepalive) afterLoad(context.Context) {} + +// +checklocksignore +func (k *keepalive) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &k.idle) + stateSourceObject.Load(1, &k.interval) + stateSourceObject.Load(2, &k.count) + stateSourceObject.Load(3, &k.unacked) +} + +func (p *protocol) StateTypeName() string { + return "pkg/tcpip/transport/tcp.protocol" +} + +func (p *protocol) StateFields() []string { + return []string{ + "stack", + "sackEnabled", + "recovery", + "delayEnabled", + "alwaysUseSynCookies", + "sendBufferSize", + "recvBufferSize", + "congestionControl", + "availableCongestionControl", + "moderateReceiveBuffer", + "lingerTimeout", + "timeWaitTimeout", + "timeWaitReuse", + "minRTO", + "maxRTO", + "maxRetries", + "synRetries", + "dispatcher", + "seqnumSecret", + "tsOffsetSecret", + } +} + +func (p *protocol) beforeSave() {} + +// +checklocksignore +func (p *protocol) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.stack) + stateSinkObject.Save(1, &p.sackEnabled) + stateSinkObject.Save(2, &p.recovery) + stateSinkObject.Save(3, &p.delayEnabled) + stateSinkObject.Save(4, &p.alwaysUseSynCookies) + stateSinkObject.Save(5, &p.sendBufferSize) + stateSinkObject.Save(6, &p.recvBufferSize) + stateSinkObject.Save(7, &p.congestionControl) + stateSinkObject.Save(8, &p.availableCongestionControl) + stateSinkObject.Save(9, &p.moderateReceiveBuffer) + stateSinkObject.Save(10, &p.lingerTimeout) + stateSinkObject.Save(11, &p.timeWaitTimeout) + stateSinkObject.Save(12, &p.timeWaitReuse) + stateSinkObject.Save(13, &p.minRTO) + stateSinkObject.Save(14, &p.maxRTO) + stateSinkObject.Save(15, &p.maxRetries) + stateSinkObject.Save(16, &p.synRetries) + stateSinkObject.Save(17, &p.dispatcher) + stateSinkObject.Save(18, &p.seqnumSecret) + stateSinkObject.Save(19, &p.tsOffsetSecret) +} + +func (p *protocol) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.stack) + stateSourceObject.Load(1, &p.sackEnabled) + stateSourceObject.Load(2, &p.recovery) + stateSourceObject.Load(3, &p.delayEnabled) + stateSourceObject.Load(4, &p.alwaysUseSynCookies) + stateSourceObject.Load(5, &p.sendBufferSize) + stateSourceObject.Load(6, &p.recvBufferSize) + stateSourceObject.Load(7, &p.congestionControl) + stateSourceObject.Load(8, &p.availableCongestionControl) + stateSourceObject.Load(9, &p.moderateReceiveBuffer) + stateSourceObject.Load(10, &p.lingerTimeout) + stateSourceObject.Load(11, &p.timeWaitTimeout) + stateSourceObject.Load(12, &p.timeWaitReuse) + stateSourceObject.Load(13, &p.minRTO) + stateSourceObject.Load(14, &p.maxRTO) + stateSourceObject.Load(15, &p.maxRetries) + stateSourceObject.Load(16, &p.synRetries) + stateSourceObject.Load(17, &p.dispatcher) + stateSourceObject.Load(18, &p.seqnumSecret) + stateSourceObject.Load(19, &p.tsOffsetSecret) +} + +func (rc *rackControl) StateTypeName() string { + return "pkg/tcpip/transport/tcp.rackControl" +} + +func (rc *rackControl) StateFields() []string { + return []string{ + "TCPRACKState", + "exitedRecovery", + "minRTT", + "tlpRxtOut", + "tlpHighRxt", + "snd", + } +} + +func (rc *rackControl) beforeSave() {} + +// +checklocksignore +func (rc *rackControl) StateSave(stateSinkObject state.Sink) { + rc.beforeSave() + stateSinkObject.Save(0, &rc.TCPRACKState) + stateSinkObject.Save(1, &rc.exitedRecovery) + stateSinkObject.Save(2, &rc.minRTT) + stateSinkObject.Save(3, &rc.tlpRxtOut) + stateSinkObject.Save(4, &rc.tlpHighRxt) + stateSinkObject.Save(5, &rc.snd) +} + +func (rc *rackControl) afterLoad(context.Context) {} + +// +checklocksignore +func (rc *rackControl) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &rc.TCPRACKState) + stateSourceObject.Load(1, &rc.exitedRecovery) + stateSourceObject.Load(2, &rc.minRTT) + stateSourceObject.Load(3, &rc.tlpRxtOut) + stateSourceObject.Load(4, &rc.tlpHighRxt) + stateSourceObject.Load(5, &rc.snd) +} + +func (r *receiver) StateTypeName() string { + return "pkg/tcpip/transport/tcp.receiver" +} + +func (r *receiver) StateFields() []string { + return []string{ + "TCPReceiverState", + "ep", + "rcvWnd", + "rcvWUP", + "prevBufUsed", + "closed", + "pendingRcvdSegments", + "lastRcvdAckTime", + } +} + +func (r *receiver) beforeSave() {} + +// +checklocksignore +func (r *receiver) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.TCPReceiverState) + stateSinkObject.Save(1, &r.ep) + stateSinkObject.Save(2, &r.rcvWnd) + stateSinkObject.Save(3, &r.rcvWUP) + stateSinkObject.Save(4, &r.prevBufUsed) + stateSinkObject.Save(5, &r.closed) + stateSinkObject.Save(6, &r.pendingRcvdSegments) + stateSinkObject.Save(7, &r.lastRcvdAckTime) +} + +func (r *receiver) afterLoad(context.Context) {} + +// +checklocksignore +func (r *receiver) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.TCPReceiverState) + stateSourceObject.Load(1, &r.ep) + stateSourceObject.Load(2, &r.rcvWnd) + stateSourceObject.Load(3, &r.rcvWUP) + stateSourceObject.Load(4, &r.prevBufUsed) + stateSourceObject.Load(5, &r.closed) + stateSourceObject.Load(6, &r.pendingRcvdSegments) + stateSourceObject.Load(7, &r.lastRcvdAckTime) +} + +func (r *renoState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.renoState" +} + +func (r *renoState) StateFields() []string { + return []string{ + "s", + } +} + +func (r *renoState) beforeSave() {} + +// +checklocksignore +func (r *renoState) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.s) +} + +func (r *renoState) afterLoad(context.Context) {} + +// +checklocksignore +func (r *renoState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.s) +} + +func (rr *renoRecovery) StateTypeName() string { + return "pkg/tcpip/transport/tcp.renoRecovery" +} + +func (rr *renoRecovery) StateFields() []string { + return []string{ + "s", + } +} + +func (rr *renoRecovery) beforeSave() {} + +// +checklocksignore +func (rr *renoRecovery) StateSave(stateSinkObject state.Sink) { + rr.beforeSave() + stateSinkObject.Save(0, &rr.s) +} + +func (rr *renoRecovery) afterLoad(context.Context) {} + +// +checklocksignore +func (rr *renoRecovery) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &rr.s) +} + +func (sr *sackRecovery) StateTypeName() string { + return "pkg/tcpip/transport/tcp.sackRecovery" +} + +func (sr *sackRecovery) StateFields() []string { + return []string{ + "s", + } +} + +func (sr *sackRecovery) beforeSave() {} + +// +checklocksignore +func (sr *sackRecovery) StateSave(stateSinkObject state.Sink) { + sr.beforeSave() + stateSinkObject.Save(0, &sr.s) +} + +func (sr *sackRecovery) afterLoad(context.Context) {} + +// +checklocksignore +func (sr *sackRecovery) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &sr.s) +} + +func (s *SACKScoreboard) StateTypeName() string { + return "pkg/tcpip/transport/tcp.SACKScoreboard" +} + +func (s *SACKScoreboard) StateFields() []string { + return []string{ + "smss", + "maxSACKED", + } +} + +func (s *SACKScoreboard) beforeSave() {} + +// +checklocksignore +func (s *SACKScoreboard) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.smss) + stateSinkObject.Save(1, &s.maxSACKED) +} + +func (s *SACKScoreboard) afterLoad(context.Context) {} + +// +checklocksignore +func (s *SACKScoreboard) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.smss) + stateSourceObject.Load(1, &s.maxSACKED) +} + +func (s *segment) StateTypeName() string { + return "pkg/tcpip/transport/tcp.segment" +} + +func (s *segment) StateFields() []string { + return []string{ + "segmentEntry", + "segmentRefs", + "ep", + "qFlags", + "pkt", + "sequenceNumber", + "ackNumber", + "flags", + "window", + "csum", + "csumValid", + "parsedOptions", + "options", + "hasNewSACKInfo", + "rcvdTime", + "xmitTime", + "xmitCount", + "acked", + "dataMemSize", + "lost", + } +} + +func (s *segment) beforeSave() {} + +// +checklocksignore +func (s *segment) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + var optionsValue []byte + optionsValue = s.saveOptions() + stateSinkObject.SaveValue(12, optionsValue) + stateSinkObject.Save(0, &s.segmentEntry) + stateSinkObject.Save(1, &s.segmentRefs) + stateSinkObject.Save(2, &s.ep) + stateSinkObject.Save(3, &s.qFlags) + stateSinkObject.Save(4, &s.pkt) + stateSinkObject.Save(5, &s.sequenceNumber) + stateSinkObject.Save(6, &s.ackNumber) + stateSinkObject.Save(7, &s.flags) + stateSinkObject.Save(8, &s.window) + stateSinkObject.Save(9, &s.csum) + stateSinkObject.Save(10, &s.csumValid) + stateSinkObject.Save(11, &s.parsedOptions) + stateSinkObject.Save(13, &s.hasNewSACKInfo) + stateSinkObject.Save(14, &s.rcvdTime) + stateSinkObject.Save(15, &s.xmitTime) + stateSinkObject.Save(16, &s.xmitCount) + stateSinkObject.Save(17, &s.acked) + stateSinkObject.Save(18, &s.dataMemSize) + stateSinkObject.Save(19, &s.lost) +} + +func (s *segment) afterLoad(context.Context) {} + +// +checklocksignore +func (s *segment) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.segmentEntry) + stateSourceObject.Load(1, &s.segmentRefs) + stateSourceObject.Load(2, &s.ep) + stateSourceObject.Load(3, &s.qFlags) + stateSourceObject.Load(4, &s.pkt) + stateSourceObject.Load(5, &s.sequenceNumber) + stateSourceObject.Load(6, &s.ackNumber) + stateSourceObject.Load(7, &s.flags) + stateSourceObject.Load(8, &s.window) + stateSourceObject.Load(9, &s.csum) + stateSourceObject.Load(10, &s.csumValid) + stateSourceObject.Load(11, &s.parsedOptions) + stateSourceObject.Load(13, &s.hasNewSACKInfo) + stateSourceObject.Load(14, &s.rcvdTime) + stateSourceObject.Load(15, &s.xmitTime) + stateSourceObject.Load(16, &s.xmitCount) + stateSourceObject.Load(17, &s.acked) + stateSourceObject.Load(18, &s.dataMemSize) + stateSourceObject.Load(19, &s.lost) + stateSourceObject.LoadValue(12, new([]byte), func(y any) { s.loadOptions(ctx, y.([]byte)) }) +} + +func (q *segmentQueue) StateTypeName() string { + return "pkg/tcpip/transport/tcp.segmentQueue" +} + +func (q *segmentQueue) StateFields() []string { + return []string{ + "list", + "ep", + "frozen", + } +} + +func (q *segmentQueue) beforeSave() {} + +// +checklocksignore +func (q *segmentQueue) StateSave(stateSinkObject state.Sink) { + q.beforeSave() + stateSinkObject.Save(0, &q.list) + stateSinkObject.Save(1, &q.ep) + stateSinkObject.Save(2, &q.frozen) +} + +func (q *segmentQueue) afterLoad(context.Context) {} + +// +checklocksignore +func (q *segmentQueue) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.LoadWait(0, &q.list) + stateSourceObject.Load(1, &q.ep) + stateSourceObject.Load(2, &q.frozen) +} + +func (s *sender) StateTypeName() string { + return "pkg/tcpip/transport/tcp.sender" +} + +func (s *sender) StateFields() []string { + return []string{ + "TCPSenderState", + "ep", + "lr", + "firstRetransmittedSegXmitTime", + "writeNext", + "writeList", + "rtt", + "minRTO", + "maxRTO", + "maxRetries", + "gso", + "state", + "cc", + "rc", + "spuriousRecovery", + "retransmitTS", + "startCork", + } +} + +func (s *sender) beforeSave() {} + +// +checklocksignore +func (s *sender) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.TCPSenderState) + stateSinkObject.Save(1, &s.ep) + stateSinkObject.Save(2, &s.lr) + stateSinkObject.Save(3, &s.firstRetransmittedSegXmitTime) + stateSinkObject.Save(4, &s.writeNext) + stateSinkObject.Save(5, &s.writeList) + stateSinkObject.Save(6, &s.rtt) + stateSinkObject.Save(7, &s.minRTO) + stateSinkObject.Save(8, &s.maxRTO) + stateSinkObject.Save(9, &s.maxRetries) + stateSinkObject.Save(10, &s.gso) + stateSinkObject.Save(11, &s.state) + stateSinkObject.Save(12, &s.cc) + stateSinkObject.Save(13, &s.rc) + stateSinkObject.Save(14, &s.spuriousRecovery) + stateSinkObject.Save(15, &s.retransmitTS) + stateSinkObject.Save(16, &s.startCork) +} + +func (s *sender) afterLoad(context.Context) {} + +// +checklocksignore +func (s *sender) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.TCPSenderState) + stateSourceObject.Load(1, &s.ep) + stateSourceObject.Load(2, &s.lr) + stateSourceObject.Load(3, &s.firstRetransmittedSegXmitTime) + stateSourceObject.Load(4, &s.writeNext) + stateSourceObject.Load(5, &s.writeList) + stateSourceObject.Load(6, &s.rtt) + stateSourceObject.Load(7, &s.minRTO) + stateSourceObject.Load(8, &s.maxRTO) + stateSourceObject.Load(9, &s.maxRetries) + stateSourceObject.Load(10, &s.gso) + stateSourceObject.Load(11, &s.state) + stateSourceObject.Load(12, &s.cc) + stateSourceObject.Load(13, &s.rc) + stateSourceObject.Load(14, &s.spuriousRecovery) + stateSourceObject.Load(15, &s.retransmitTS) + stateSourceObject.Load(16, &s.startCork) +} + +func (wl *protectedWriteList) StateTypeName() string { + return "pkg/tcpip/transport/tcp.protectedWriteList" +} + +func (wl *protectedWriteList) StateFields() []string { + return []string{ + "writeList", + "set", + } +} + +func (wl *protectedWriteList) beforeSave() {} + +// +checklocksignore +func (wl *protectedWriteList) StateSave(stateSinkObject state.Sink) { + wl.beforeSave() + stateSinkObject.Save(0, &wl.writeList) + stateSinkObject.Save(1, &wl.set) +} + +func (wl *protectedWriteList) afterLoad(context.Context) {} + +// +checklocksignore +func (wl *protectedWriteList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &wl.writeList) + stateSourceObject.Load(1, &wl.set) +} + +func (r *rtt) StateTypeName() string { + return "pkg/tcpip/transport/tcp.rtt" +} + +func (r *rtt) StateFields() []string { + return []string{ + "TCPRTTState", + } +} + +func (r *rtt) beforeSave() {} + +// +checklocksignore +func (r *rtt) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.TCPRTTState) +} + +func (r *rtt) afterLoad(context.Context) {} + +// +checklocksignore +func (r *rtt) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.TCPRTTState) +} + +func (t *TCPCubicState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPCubicState" +} + +func (t *TCPCubicState) StateFields() []string { + return []string{ + "WLastMax", + "WMax", + "T", + "TimeSinceLastCongestion", + "C", + "K", + "Beta", + "WC", + "WEst", + "EndSeq", + "CurrRTT", + "LastRTT", + "SampleCount", + "LastAck", + "RoundStart", + } +} + +func (t *TCPCubicState) beforeSave() {} + +// +checklocksignore +func (t *TCPCubicState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.WLastMax) + stateSinkObject.Save(1, &t.WMax) + stateSinkObject.Save(2, &t.T) + stateSinkObject.Save(3, &t.TimeSinceLastCongestion) + stateSinkObject.Save(4, &t.C) + stateSinkObject.Save(5, &t.K) + stateSinkObject.Save(6, &t.Beta) + stateSinkObject.Save(7, &t.WC) + stateSinkObject.Save(8, &t.WEst) + stateSinkObject.Save(9, &t.EndSeq) + stateSinkObject.Save(10, &t.CurrRTT) + stateSinkObject.Save(11, &t.LastRTT) + stateSinkObject.Save(12, &t.SampleCount) + stateSinkObject.Save(13, &t.LastAck) + stateSinkObject.Save(14, &t.RoundStart) +} + +func (t *TCPCubicState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPCubicState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.WLastMax) + stateSourceObject.Load(1, &t.WMax) + stateSourceObject.Load(2, &t.T) + stateSourceObject.Load(3, &t.TimeSinceLastCongestion) + stateSourceObject.Load(4, &t.C) + stateSourceObject.Load(5, &t.K) + stateSourceObject.Load(6, &t.Beta) + stateSourceObject.Load(7, &t.WC) + stateSourceObject.Load(8, &t.WEst) + stateSourceObject.Load(9, &t.EndSeq) + stateSourceObject.Load(10, &t.CurrRTT) + stateSourceObject.Load(11, &t.LastRTT) + stateSourceObject.Load(12, &t.SampleCount) + stateSourceObject.Load(13, &t.LastAck) + stateSourceObject.Load(14, &t.RoundStart) +} + +func (t *TCPRACKState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPRACKState" +} + +func (t *TCPRACKState) StateFields() []string { + return []string{ + "XmitTime", + "EndSequence", + "FACK", + "RTT", + "Reord", + "DSACKSeen", + "ReoWnd", + "ReoWndIncr", + "ReoWndPersist", + "RTTSeq", + } +} + +func (t *TCPRACKState) beforeSave() {} + +// +checklocksignore +func (t *TCPRACKState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.XmitTime) + stateSinkObject.Save(1, &t.EndSequence) + stateSinkObject.Save(2, &t.FACK) + stateSinkObject.Save(3, &t.RTT) + stateSinkObject.Save(4, &t.Reord) + stateSinkObject.Save(5, &t.DSACKSeen) + stateSinkObject.Save(6, &t.ReoWnd) + stateSinkObject.Save(7, &t.ReoWndIncr) + stateSinkObject.Save(8, &t.ReoWndPersist) + stateSinkObject.Save(9, &t.RTTSeq) +} + +func (t *TCPRACKState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPRACKState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.XmitTime) + stateSourceObject.Load(1, &t.EndSequence) + stateSourceObject.Load(2, &t.FACK) + stateSourceObject.Load(3, &t.RTT) + stateSourceObject.Load(4, &t.Reord) + stateSourceObject.Load(5, &t.DSACKSeen) + stateSourceObject.Load(6, &t.ReoWnd) + stateSourceObject.Load(7, &t.ReoWndIncr) + stateSourceObject.Load(8, &t.ReoWndPersist) + stateSourceObject.Load(9, &t.RTTSeq) +} + +func (t *TCPEndpointID) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPEndpointID" +} + +func (t *TCPEndpointID) StateFields() []string { + return []string{ + "LocalPort", + "LocalAddress", + "RemotePort", + "RemoteAddress", + } +} + +func (t *TCPEndpointID) beforeSave() {} + +// +checklocksignore +func (t *TCPEndpointID) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.LocalPort) + stateSinkObject.Save(1, &t.LocalAddress) + stateSinkObject.Save(2, &t.RemotePort) + stateSinkObject.Save(3, &t.RemoteAddress) +} + +func (t *TCPEndpointID) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPEndpointID) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.LocalPort) + stateSourceObject.Load(1, &t.LocalAddress) + stateSourceObject.Load(2, &t.RemotePort) + stateSourceObject.Load(3, &t.RemoteAddress) +} + +func (t *TCPFastRecoveryState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPFastRecoveryState" +} + +func (t *TCPFastRecoveryState) StateFields() []string { + return []string{ + "Active", + "First", + "Last", + "MaxCwnd", + "HighRxt", + "RescueRxt", + } +} + +func (t *TCPFastRecoveryState) beforeSave() {} + +// +checklocksignore +func (t *TCPFastRecoveryState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.Active) + stateSinkObject.Save(1, &t.First) + stateSinkObject.Save(2, &t.Last) + stateSinkObject.Save(3, &t.MaxCwnd) + stateSinkObject.Save(4, &t.HighRxt) + stateSinkObject.Save(5, &t.RescueRxt) +} + +func (t *TCPFastRecoveryState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPFastRecoveryState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.Active) + stateSourceObject.Load(1, &t.First) + stateSourceObject.Load(2, &t.Last) + stateSourceObject.Load(3, &t.MaxCwnd) + stateSourceObject.Load(4, &t.HighRxt) + stateSourceObject.Load(5, &t.RescueRxt) +} + +func (t *TCPReceiverState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPReceiverState" +} + +func (t *TCPReceiverState) StateFields() []string { + return []string{ + "RcvNxt", + "RcvAcc", + "RcvWndScale", + "PendingBufUsed", + } +} + +func (t *TCPReceiverState) beforeSave() {} + +// +checklocksignore +func (t *TCPReceiverState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.RcvNxt) + stateSinkObject.Save(1, &t.RcvAcc) + stateSinkObject.Save(2, &t.RcvWndScale) + stateSinkObject.Save(3, &t.PendingBufUsed) +} + +func (t *TCPReceiverState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPReceiverState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.RcvNxt) + stateSourceObject.Load(1, &t.RcvAcc) + stateSourceObject.Load(2, &t.RcvWndScale) + stateSourceObject.Load(3, &t.PendingBufUsed) +} + +func (t *TCPRTTState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPRTTState" +} + +func (t *TCPRTTState) StateFields() []string { + return []string{ + "SRTT", + "RTTVar", + "SRTTInited", + } +} + +func (t *TCPRTTState) beforeSave() {} + +// +checklocksignore +func (t *TCPRTTState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.SRTT) + stateSinkObject.Save(1, &t.RTTVar) + stateSinkObject.Save(2, &t.SRTTInited) +} + +func (t *TCPRTTState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPRTTState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.SRTT) + stateSourceObject.Load(1, &t.RTTVar) + stateSourceObject.Load(2, &t.SRTTInited) +} + +func (t *TCPSenderState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPSenderState" +} + +func (t *TCPSenderState) StateFields() []string { + return []string{ + "LastSendTime", + "DupAckCount", + "SndCwnd", + "Ssthresh", + "SndCAAckCount", + "Outstanding", + "SackedOut", + "SndWnd", + "SndUna", + "SndNxt", + "RTTMeasureSeqNum", + "RTTMeasureTime", + "Closed", + "RTO", + "RTTState", + "MaxPayloadSize", + "SndWndScale", + "MaxSentAck", + "FastRecovery", + "Cubic", + "RACKState", + "RetransmitTS", + "SpuriousRecovery", + } +} + +func (t *TCPSenderState) beforeSave() {} + +// +checklocksignore +func (t *TCPSenderState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.LastSendTime) + stateSinkObject.Save(1, &t.DupAckCount) + stateSinkObject.Save(2, &t.SndCwnd) + stateSinkObject.Save(3, &t.Ssthresh) + stateSinkObject.Save(4, &t.SndCAAckCount) + stateSinkObject.Save(5, &t.Outstanding) + stateSinkObject.Save(6, &t.SackedOut) + stateSinkObject.Save(7, &t.SndWnd) + stateSinkObject.Save(8, &t.SndUna) + stateSinkObject.Save(9, &t.SndNxt) + stateSinkObject.Save(10, &t.RTTMeasureSeqNum) + stateSinkObject.Save(11, &t.RTTMeasureTime) + stateSinkObject.Save(12, &t.Closed) + stateSinkObject.Save(13, &t.RTO) + stateSinkObject.Save(14, &t.RTTState) + stateSinkObject.Save(15, &t.MaxPayloadSize) + stateSinkObject.Save(16, &t.SndWndScale) + stateSinkObject.Save(17, &t.MaxSentAck) + stateSinkObject.Save(18, &t.FastRecovery) + stateSinkObject.Save(19, &t.Cubic) + stateSinkObject.Save(20, &t.RACKState) + stateSinkObject.Save(21, &t.RetransmitTS) + stateSinkObject.Save(22, &t.SpuriousRecovery) +} + +func (t *TCPSenderState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPSenderState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.LastSendTime) + stateSourceObject.Load(1, &t.DupAckCount) + stateSourceObject.Load(2, &t.SndCwnd) + stateSourceObject.Load(3, &t.Ssthresh) + stateSourceObject.Load(4, &t.SndCAAckCount) + stateSourceObject.Load(5, &t.Outstanding) + stateSourceObject.Load(6, &t.SackedOut) + stateSourceObject.Load(7, &t.SndWnd) + stateSourceObject.Load(8, &t.SndUna) + stateSourceObject.Load(9, &t.SndNxt) + stateSourceObject.Load(10, &t.RTTMeasureSeqNum) + stateSourceObject.Load(11, &t.RTTMeasureTime) + stateSourceObject.Load(12, &t.Closed) + stateSourceObject.Load(13, &t.RTO) + stateSourceObject.Load(14, &t.RTTState) + stateSourceObject.Load(15, &t.MaxPayloadSize) + stateSourceObject.Load(16, &t.SndWndScale) + stateSourceObject.Load(17, &t.MaxSentAck) + stateSourceObject.Load(18, &t.FastRecovery) + stateSourceObject.Load(19, &t.Cubic) + stateSourceObject.Load(20, &t.RACKState) + stateSourceObject.Load(21, &t.RetransmitTS) + stateSourceObject.Load(22, &t.SpuriousRecovery) +} + +func (t *TCPSACKInfo) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPSACKInfo" +} + +func (t *TCPSACKInfo) StateFields() []string { + return []string{ + "Blocks", + "ReceivedBlocks", + "MaxSACKED", + } +} + +func (t *TCPSACKInfo) beforeSave() {} + +// +checklocksignore +func (t *TCPSACKInfo) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.Blocks) + stateSinkObject.Save(1, &t.ReceivedBlocks) + stateSinkObject.Save(2, &t.MaxSACKED) +} + +func (t *TCPSACKInfo) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPSACKInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.Blocks) + stateSourceObject.Load(1, &t.ReceivedBlocks) + stateSourceObject.Load(2, &t.MaxSACKED) +} + +func (r *RcvBufAutoTuneParams) StateTypeName() string { + return "pkg/tcpip/transport/tcp.RcvBufAutoTuneParams" +} + +func (r *RcvBufAutoTuneParams) StateFields() []string { + return []string{ + "MeasureTime", + "CopiedBytes", + "PrevCopiedBytes", + "RcvBufSize", + "RTT", + "RTTVar", + "RTTMeasureSeqNumber", + "RTTMeasureTime", + "Disabled", + } +} + +func (r *RcvBufAutoTuneParams) beforeSave() {} + +// +checklocksignore +func (r *RcvBufAutoTuneParams) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.MeasureTime) + stateSinkObject.Save(1, &r.CopiedBytes) + stateSinkObject.Save(2, &r.PrevCopiedBytes) + stateSinkObject.Save(3, &r.RcvBufSize) + stateSinkObject.Save(4, &r.RTT) + stateSinkObject.Save(5, &r.RTTVar) + stateSinkObject.Save(6, &r.RTTMeasureSeqNumber) + stateSinkObject.Save(7, &r.RTTMeasureTime) + stateSinkObject.Save(8, &r.Disabled) +} + +func (r *RcvBufAutoTuneParams) afterLoad(context.Context) {} + +// +checklocksignore +func (r *RcvBufAutoTuneParams) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.MeasureTime) + stateSourceObject.Load(1, &r.CopiedBytes) + stateSourceObject.Load(2, &r.PrevCopiedBytes) + stateSourceObject.Load(3, &r.RcvBufSize) + stateSourceObject.Load(4, &r.RTT) + stateSourceObject.Load(5, &r.RTTVar) + stateSourceObject.Load(6, &r.RTTMeasureSeqNumber) + stateSourceObject.Load(7, &r.RTTMeasureTime) + stateSourceObject.Load(8, &r.Disabled) +} + +func (t *TCPRcvBufState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPRcvBufState" +} + +func (t *TCPRcvBufState) StateFields() []string { + return []string{ + "RcvBufUsed", + "RcvAutoParams", + "RcvClosed", + } +} + +func (t *TCPRcvBufState) beforeSave() {} + +// +checklocksignore +func (t *TCPRcvBufState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.RcvBufUsed) + stateSinkObject.Save(1, &t.RcvAutoParams) + stateSinkObject.Save(2, &t.RcvClosed) +} + +func (t *TCPRcvBufState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPRcvBufState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.RcvBufUsed) + stateSourceObject.Load(1, &t.RcvAutoParams) + stateSourceObject.Load(2, &t.RcvClosed) +} + +func (t *TCPSndBufState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPSndBufState" +} + +func (t *TCPSndBufState) StateFields() []string { + return []string{ + "SndBufSize", + "SndBufUsed", + "SndClosed", + "PacketTooBigCount", + "SndMTU", + "AutoTuneSndBufDisabled", + } +} + +func (t *TCPSndBufState) beforeSave() {} + +// +checklocksignore +func (t *TCPSndBufState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.SndBufSize) + stateSinkObject.Save(1, &t.SndBufUsed) + stateSinkObject.Save(2, &t.SndClosed) + stateSinkObject.Save(3, &t.PacketTooBigCount) + stateSinkObject.Save(4, &t.SndMTU) + stateSinkObject.Save(5, &t.AutoTuneSndBufDisabled) +} + +func (t *TCPSndBufState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPSndBufState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.SndBufSize) + stateSourceObject.Load(1, &t.SndBufUsed) + stateSourceObject.Load(2, &t.SndClosed) + stateSourceObject.Load(3, &t.PacketTooBigCount) + stateSourceObject.Load(4, &t.SndMTU) + stateSourceObject.Load(5, &t.AutoTuneSndBufDisabled) +} + +func (t *TCPEndpointStateInner) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPEndpointStateInner" +} + +func (t *TCPEndpointStateInner) StateFields() []string { + return []string{ + "TSOffset", + "SACKPermitted", + "SendTSOk", + "RecentTS", + } +} + +func (t *TCPEndpointStateInner) beforeSave() {} + +// +checklocksignore +func (t *TCPEndpointStateInner) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.TSOffset) + stateSinkObject.Save(1, &t.SACKPermitted) + stateSinkObject.Save(2, &t.SendTSOk) + stateSinkObject.Save(3, &t.RecentTS) +} + +func (t *TCPEndpointStateInner) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPEndpointStateInner) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.TSOffset) + stateSourceObject.Load(1, &t.SACKPermitted) + stateSourceObject.Load(2, &t.SendTSOk) + stateSourceObject.Load(3, &t.RecentTS) +} + +func (t *TCPEndpointState) StateTypeName() string { + return "pkg/tcpip/transport/tcp.TCPEndpointState" +} + +func (t *TCPEndpointState) StateFields() []string { + return []string{ + "TCPEndpointStateInner", + "ID", + "SegTime", + "RcvBufState", + "SndBufState", + "SACK", + "Receiver", + "Sender", + } +} + +func (t *TCPEndpointState) beforeSave() {} + +// +checklocksignore +func (t *TCPEndpointState) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.TCPEndpointStateInner) + stateSinkObject.Save(1, &t.ID) + stateSinkObject.Save(2, &t.SegTime) + stateSinkObject.Save(3, &t.RcvBufState) + stateSinkObject.Save(4, &t.SndBufState) + stateSinkObject.Save(5, &t.SACK) + stateSinkObject.Save(6, &t.Receiver) + stateSinkObject.Save(7, &t.Sender) +} + +func (t *TCPEndpointState) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCPEndpointState) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.TCPEndpointStateInner) + stateSourceObject.Load(1, &t.ID) + stateSourceObject.Load(2, &t.SegTime) + stateSourceObject.Load(3, &t.RcvBufState) + stateSourceObject.Load(4, &t.SndBufState) + stateSourceObject.Load(5, &t.SACK) + stateSourceObject.Load(6, &t.Receiver) + stateSourceObject.Load(7, &t.Sender) +} + +func (l *endpointList) StateTypeName() string { + return "pkg/tcpip/transport/tcp.endpointList" +} + +func (l *endpointList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *endpointList) beforeSave() {} + +// +checklocksignore +func (l *endpointList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *endpointList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *endpointList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *endpointEntry) StateTypeName() string { + return "pkg/tcpip/transport/tcp.endpointEntry" +} + +func (e *endpointEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *endpointEntry) beforeSave() {} + +// +checklocksignore +func (e *endpointEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *endpointEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *endpointEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func (l *segmentList) StateTypeName() string { + return "pkg/tcpip/transport/tcp.segmentList" +} + +func (l *segmentList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *segmentList) beforeSave() {} + +// +checklocksignore +func (l *segmentList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *segmentList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *segmentList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *segmentEntry) StateTypeName() string { + return "pkg/tcpip/transport/tcp.segmentEntry" +} + +func (e *segmentEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *segmentEntry) beforeSave() {} + +// +checklocksignore +func (e *segmentEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *segmentEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *segmentEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func (r *segmentRefs) StateTypeName() string { + return "pkg/tcpip/transport/tcp.segmentRefs" +} + +func (r *segmentRefs) StateFields() []string { + return []string{ + "refCount", + } +} + +func (r *segmentRefs) beforeSave() {} + +// +checklocksignore +func (r *segmentRefs) StateSave(stateSinkObject state.Sink) { + r.beforeSave() + stateSinkObject.Save(0, &r.refCount) +} + +// +checklocksignore +func (r *segmentRefs) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &r.refCount) + stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) }) +} + +func init() { + state.Register((*acceptQueue)(nil)) + state.Register((*handshake)(nil)) + state.Register((*cubicState)(nil)) + state.Register((*epQueue)(nil)) + state.Register((*processor)(nil)) + state.Register((*dispatcher)(nil)) + state.Register((*jenkinsHasher)(nil)) + state.Register((*SACKInfo)(nil)) + state.Register((*ReceiveErrors)(nil)) + state.Register((*SendErrors)(nil)) + state.Register((*Stats)(nil)) + state.Register((*sndQueueInfo)(nil)) + state.Register((*Endpoint)(nil)) + state.Register((*keepalive)(nil)) + state.Register((*protocol)(nil)) + state.Register((*rackControl)(nil)) + state.Register((*receiver)(nil)) + state.Register((*renoState)(nil)) + state.Register((*renoRecovery)(nil)) + state.Register((*sackRecovery)(nil)) + state.Register((*SACKScoreboard)(nil)) + state.Register((*segment)(nil)) + state.Register((*segmentQueue)(nil)) + state.Register((*sender)(nil)) + state.Register((*protectedWriteList)(nil)) + state.Register((*rtt)(nil)) + state.Register((*TCPCubicState)(nil)) + state.Register((*TCPRACKState)(nil)) + state.Register((*TCPEndpointID)(nil)) + state.Register((*TCPFastRecoveryState)(nil)) + state.Register((*TCPReceiverState)(nil)) + state.Register((*TCPRTTState)(nil)) + state.Register((*TCPSenderState)(nil)) + state.Register((*TCPSACKInfo)(nil)) + state.Register((*RcvBufAutoTuneParams)(nil)) + state.Register((*TCPRcvBufState)(nil)) + state.Register((*TCPSndBufState)(nil)) + state.Register((*TCPEndpointStateInner)(nil)) + state.Register((*TCPEndpointState)(nil)) + state.Register((*endpointList)(nil)) + state.Register((*endpointEntry)(nil)) + state.Register((*segmentList)(nil)) + state.Register((*segmentEntry)(nil)) + state.Register((*segmentRefs)(nil)) +} diff --git a/pkg/tcpip/transport/tcp/tcp_unsafe_state_autogen.go b/pkg/tcpip/transport/tcp/tcp_unsafe_state_autogen.go new file mode 100644 index 0000000..4cb82fc --- /dev/null +++ b/pkg/tcpip/transport/tcp/tcp_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package tcp diff --git a/pkg/tcpip/transport/tcp/timer.go b/pkg/tcpip/transport/tcp/timer.go new file mode 100644 index 0000000..022ceb5 --- /dev/null +++ b/pkg/tcpip/transport/tcp/timer.go @@ -0,0 +1,160 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcp + +import ( + "time" + + "github.com/sagernet/gvisor/pkg/tcpip" +) + +type timerState int + +const ( + // The timer has not been initialized yet or has been cleaned up. + timerUninitialized timerState = iota + // The timer is disabled. + timerStateDisabled + // The timer is enabled, but the clock timer may be set to an earlier + // expiration time due to a previous orphaned state. + timerStateEnabled + // The timer is disabled, but the clock timer is enabled, which means that + // it will cause a spurious wakeup unless the timer is enabled before the + // clock timer fires. + timerStateOrphaned +) + +// timer is a timer implementation that reduces the interactions with the +// clock timer infrastructure by letting timers run (and potentially +// eventually expire) even if they are stopped. It makes it cheaper to +// disable/reenable timers at the expense of spurious wakes. This is useful for +// cases when the same timer is disabled/reenabled repeatedly with relatively +// long timeouts farther into the future. +// +// TCP retransmit timers benefit from this because they the timeouts are long +// (currently at least 200ms), and get disabled when acks are received, and +// reenabled when new pending segments are sent. +// +// It is advantageous to avoid interacting with the clock because it acquires +// a global mutex and performs O(log n) operations, where n is the global number +// of timers, whenever a timer is enabled or disabled, and may make a syscall. +// +// This struct is thread-compatible. +type timer struct { + state timerState + + clock tcpip.Clock + + // target is the expiration time of the current timer. It is only + // meaningful in the enabled state. + target tcpip.MonotonicTime + + // clockTarget is the expiration time of the clock timer. It is + // meaningful in the enabled and orphaned states. + clockTarget tcpip.MonotonicTime + + // timer is the clock timer used to wait on. + timer tcpip.Timer + + // callback is the function that's called when the timer expires. + callback func() +} + +// init initializes the timer. Once it expires the function callback +// passed will be called. +func (t *timer) init(clock tcpip.Clock, f func()) { + t.state = timerStateDisabled + t.clock = clock + t.callback = f +} + +// cleanup frees all resources associated with the timer. +func (t *timer) cleanup() { + if t.timer == nil { + // No cleanup needed. + return + } + t.timer.Stop() + *t = timer{} +} + +// isUninitialized returns true if the timer is in the uninitialized state. This +// is only true if init() has never been called or if cleanup has been called. +func (t *timer) isUninitialized() bool { + return t.state == timerUninitialized +} + +// checkExpiration checks if the given timer has actually expired, it should be +// called whenever the callback function is called, and is used to check if it's +// a spurious timer expiration (due to a previously orphaned timer) or a +// legitimate one. +func (t *timer) checkExpiration() bool { + // Transition to fully disabled state if we're just consuming an + // orphaned timer. + if t.state == timerStateOrphaned { + t.state = timerStateDisabled + return false + } + + // The timer is enabled, but it may have expired early. Check if that's + // the case, and if so, reset the runtime timer to the correct time. + now := t.clock.NowMonotonic() + if now.Before(t.target) { + t.clockTarget = t.target + t.timer.Reset(t.target.Sub(now)) + return false + } + + // The timer has actually expired, disable it for now and inform the + // caller. + t.state = timerStateDisabled + return true +} + +// disable disables the timer, leaving it in an orphaned state if it wasn't +// already disabled. +func (t *timer) disable() { + if t.state != timerStateDisabled { + t.state = timerStateOrphaned + } +} + +// enabled returns true if the timer is currently enabled, false otherwise. +func (t *timer) enabled() bool { + return t.state == timerStateEnabled +} + +// enable enables the timer, programming the runtime timer if necessary. +func (t *timer) enable(d time.Duration) { + t.target = t.clock.NowMonotonic().Add(d) + + // Check if we need to set the runtime timer. + if t.state == timerStateDisabled || t.target.Before(t.clockTarget) { + t.clockTarget = t.target + t.resetOrStart(d) + } + + t.state = timerStateEnabled +} + +// resetOrStart creates the timer if it doesn't already exist or resets it with +// the given duration if it does. +func (t *timer) resetOrStart(d time.Duration) { + if t.timer == nil { + t.timer = t.clock.AfterFunc(d, t.callback) + } else { + t.timer.Reset(d) + } +} diff --git a/pkg/tcpip/transport/tcpconntrack/tcp_conntrack.go b/pkg/tcpip/transport/tcpconntrack/tcp_conntrack.go new file mode 100644 index 0000000..1936768 --- /dev/null +++ b/pkg/tcpip/transport/tcpconntrack/tcp_conntrack.go @@ -0,0 +1,417 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tcpconntrack implements a TCP connection tracking object. It allows +// users with access to a segment stream to figure out when a connection is +// established, reset, and closed (and in the last case, who closed first). +package tcpconntrack + +import ( + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/seqnum" +) + +// Result is returned when the state of a TCB is updated in response to a +// segment. +type Result int + +const ( + // ResultDrop indicates that the segment should be dropped. + ResultDrop Result = iota + + // ResultConnecting indicates that the connection remains in a + // connecting state. + ResultConnecting + + // ResultAlive indicates that the connection remains alive (connected). + ResultAlive + + // ResultReset indicates that the connection was reset. + ResultReset + + // ResultClosedByResponder indicates that the connection was gracefully + // closed, and the reply stream was closed first. + ResultClosedByResponder + + // ResultClosedByOriginator indicates that the connection was gracefully + // closed, and the original stream was closed first. + ResultClosedByOriginator +) + +// maxWindowShift is the maximum shift value of the per the windows scale +// option defined by RFC 1323. +const maxWindowShift = 14 + +// TCB is a TCP Control Block. It holds state necessary to keep track of a TCP +// connection and inform the caller when the connection has been closed. +// +// +stateify savable +type TCB struct { + reply stream + original stream + + // State handlers. hdr is not guaranteed to contain bytes beyond the TCP + // header itself, i.e. it may not contain the payload. + // TODO(b/341946753): Restore them when netstack is savable. + handlerReply func(tcb *TCB, hdr header.TCP, dataLen int) Result `state:"nosave"` + handlerOriginal func(tcb *TCB, hdr header.TCP, dataLen int) Result `state:"nosave"` + + // firstFin holds a pointer to the first stream to send a FIN. + firstFin *stream + + // state is the current state of the stream. + state Result +} + +// Init initializes the state of the TCB according to the initial SYN. +func (t *TCB) Init(initialSyn header.TCP, dataLen int) Result { + t.handlerReply = synSentStateReply + t.handlerOriginal = synSentStateOriginal + + iss := seqnum.Value(initialSyn.SequenceNumber()) + t.original.una = iss + t.original.nxt = iss.Add(logicalLenSyn(initialSyn, dataLen)) + t.original.end = t.original.nxt + // TODO(gvisor.dev/issue/6734): Cache TCP options instead of re-parsing them. + // Because original and reply are streams, scale applies to the reply; it is + // the receive window in the reply direction. + t.reply.shiftCnt = header.ParseSynOptions(initialSyn.Options(), false /* isAck */).WS + + // Even though "end" is a sequence number, we don't know the initial + // receive sequence number yet, so we store the window size until we get + // a SYN from the server. + t.reply.una = 0 + t.reply.nxt = 0 + t.reply.end = seqnum.Value(initialSyn.WindowSize()) + t.state = ResultConnecting + return t.state +} + +// UpdateStateReply updates the state of the TCB based on the supplied reply +// segment. +func (t *TCB) UpdateStateReply(tcp header.TCP, dataLen int) Result { + st := t.handlerReply(t, tcp, dataLen) + if st != ResultDrop { + t.state = st + } + return st +} + +// UpdateStateOriginal updates the state of the TCB based on the supplied +// original segment. +func (t *TCB) UpdateStateOriginal(tcp header.TCP, dataLen int) Result { + st := t.handlerOriginal(t, tcp, dataLen) + if st != ResultDrop { + t.state = st + } + return st +} + +// State returns the current state of the TCB. +func (t *TCB) State() Result { + return t.state +} + +// IsAlive returns true as long as the connection is established(Alive) +// or connecting state. +func (t *TCB) IsAlive() bool { + return !t.reply.rstSeen && !t.original.rstSeen && (!t.reply.closed() || !t.original.closed()) +} + +// OriginalSendSequenceNumber returns the snd.NXT for the original stream. +func (t *TCB) OriginalSendSequenceNumber() seqnum.Value { + return t.original.nxt +} + +// ReplySendSequenceNumber returns the snd.NXT for the reply stream. +func (t *TCB) ReplySendSequenceNumber() seqnum.Value { + return t.reply.nxt +} + +// adapResult modifies the supplied "Result" according to the state of the TCB; +// if r is anything other than "Alive", or if one of the streams isn't closed +// yet, it is returned unmodified. Otherwise it's converted to either +// ClosedByOriginator or ClosedByResponder depending on which stream was closed +// first. +func (t *TCB) adaptResult(r Result) Result { + // Check the unmodified case. + if r != ResultAlive || !t.reply.closed() || !t.original.closed() { + return r + } + + // Find out which was closed first. + if t.firstFin == &t.original { + return ResultClosedByOriginator + } + + return ResultClosedByResponder +} + +// synSentStateReply is the state handler for reply segments when the +// connection is in SYN-SENT state. +func synSentStateReply(t *TCB, tcp header.TCP, dataLen int) Result { + flags := tcp.Flags() + ackPresent := flags&header.TCPFlagAck != 0 + ack := seqnum.Value(tcp.AckNumber()) + + // Ignore segment if ack is present but not acceptable. + if ackPresent && !(ack-1).InRange(t.original.una, t.original.nxt) { + return ResultConnecting + } + + // If reset is specified, we will let the packet through no matter what + // but we will also destroy the connection if the ACK is present (and + // implicitly acceptable). + if flags&header.TCPFlagRst != 0 { + if ackPresent { + t.reply.rstSeen = true + return ResultReset + } + return ResultConnecting + } + + // Ignore segment if SYN is not set. + if flags&header.TCPFlagSyn == 0 { + return ResultConnecting + } + + // TODO(gvisor.dev/issue/6734): Cache TCP options instead of re-parsing them. + // Because original and reply are streams, scale applies to the reply; it is + // the receive window in the original direction. + t.original.shiftCnt = header.ParseSynOptions(tcp.Options(), ackPresent).WS + + // Window scaling works only when both ends use the scale option. + if t.original.shiftCnt != -1 && t.reply.shiftCnt != -1 { + // Per RFC 1323 section 2.3: + // + // "If a Window Scale option is received with a shift.cnt value exceeding + // 14, the TCP should log the error but use 14 instead of the specified + // value." + if t.original.shiftCnt > maxWindowShift { + t.original.shiftCnt = maxWindowShift + } + if t.reply.shiftCnt > maxWindowShift { + t.original.shiftCnt = maxWindowShift + } + } else { + t.original.shiftCnt = 0 + t.reply.shiftCnt = 0 + } + // Update state informed by this SYN. + irs := seqnum.Value(tcp.SequenceNumber()) + t.reply.una = irs + t.reply.nxt = irs.Add(logicalLen(tcp, dataLen, seqnum.Size(t.reply.end) /* end currently holds the receive window size */)) + t.reply.end <<= t.reply.shiftCnt + t.reply.end.UpdateForward(seqnum.Size(irs)) + + windowSize := t.original.windowSize(tcp) + t.original.end = t.original.una.Add(windowSize) + + // If the ACK was set (it is acceptable), update our unacknowledgement + // tracking. + if ackPresent { + // Advance the "una" and "end" indices of the original stream. + if t.original.una.LessThan(ack) { + t.original.una = ack + } + + if end := ack.Add(seqnum.Size(windowSize)); t.original.end.LessThan(end) { + t.original.end = end + } + } + + // Update handlers so that new calls will be handled by new state. + t.handlerReply = allOtherReply + t.handlerOriginal = allOtherOriginal + + return ResultAlive +} + +// synSentStateOriginal is the state handler for original segments when the +// connection is in SYN-SENT state. +func synSentStateOriginal(t *TCB, tcp header.TCP, _ int) Result { + // Drop original segments that aren't retransmits of the original one. + if tcp.Flags() != header.TCPFlagSyn || tcp.SequenceNumber() != uint32(t.original.una) { + return ResultDrop + } + + // Update the receive window. We only remember the largest value seen. + if wnd := seqnum.Value(tcp.WindowSize()); wnd > t.reply.end { + t.reply.end = wnd + } + + return ResultConnecting +} + +// update updates the state of reply and original streams, given the supplied +// reply segment. For original segments, this same function can be called with +// swapped reply/original streams. +func update(tcp header.TCP, reply, original *stream, firstFin **stream, dataLen int) Result { + // Ignore segments out of the window. + s := seqnum.Value(tcp.SequenceNumber()) + if !reply.acceptable(s, seqnum.Size(dataLen)) { + return ResultAlive + } + + flags := tcp.Flags() + if flags&header.TCPFlagRst != 0 { + reply.rstSeen = true + return ResultReset + } + + // Ignore segments that don't have the ACK flag, and those with the SYN + // flag. + if flags&header.TCPFlagAck == 0 || flags&header.TCPFlagSyn != 0 { + return ResultAlive + } + + // Ignore segments that acknowledge not yet sent data. + ack := seqnum.Value(tcp.AckNumber()) + if original.nxt.LessThan(ack) { + return ResultAlive + } + + // Advance the "una" and "end" indices of the original stream. + if original.una.LessThan(ack) { + original.una = ack + } + + if end := ack.Add(original.windowSize(tcp)); original.end.LessThan(end) { + original.end = end + } + + // Advance the "nxt" index of the reply stream. + end := s.Add(logicalLen(tcp, dataLen, reply.rwndSize())) + if reply.nxt.LessThan(end) { + reply.nxt = end + } + + // Note the index of the FIN segment. And stash away a pointer to the + // first stream to see a FIN. + if flags&header.TCPFlagFin != 0 && !reply.finSeen { + reply.finSeen = true + reply.fin = end - 1 + + if *firstFin == nil { + *firstFin = reply + } + } + + return ResultAlive +} + +// allOtherReply is the state handler for reply segments in all states +// except SYN-SENT. +func allOtherReply(t *TCB, tcp header.TCP, dataLen int) Result { + return t.adaptResult(update(tcp, &t.reply, &t.original, &t.firstFin, dataLen)) +} + +// allOtherOriginal is the state handler for original segments in all states +// except SYN-SENT. +func allOtherOriginal(t *TCB, tcp header.TCP, dataLen int) Result { + return t.adaptResult(update(tcp, &t.original, &t.reply, &t.firstFin, dataLen)) +} + +// streams holds the state of a TCP unidirectional stream. +// +// +stateify savable +type stream struct { + // The interval [una, end) is the allowed interval as defined by the + // receiver, i.e., anything less than una has already been acknowledged + // and anything greater than or equal to end is beyond the receiver + // window. The interval [una, nxt) is the acknowledgable range, whose + // right edge indicates the sequence number of the next byte to be sent + // by the sender, i.e., anything greater than or equal to nxt hasn't + // been sent yet. + una seqnum.Value + nxt seqnum.Value + end seqnum.Value + + // finSeen indicates if a FIN has already been sent on this stream. + finSeen bool + + // fin is the sequence number of the FIN. It is only valid after finSeen + // is set to true. + fin seqnum.Value + + // rstSeen indicates if a RST has already been sent on this stream. + rstSeen bool + + // shiftCnt is the shift of the window scale of the receiver of the stream, + // i.e. in a stream from A to B it is B's receive window scale. It cannot be + // greater than maxWindowScale. + shiftCnt int +} + +// acceptable determines if the segment with the given sequence number and data +// length is acceptable, i.e., if it's within the [una, end) window or, in case +// the window is zero, if it's a packet with no payload and sequence number +// equal to una. +func (s *stream) acceptable(segSeq seqnum.Value, segLen seqnum.Size) bool { + return header.Acceptable(segSeq, segLen, s.una, s.end) +} + +// closed determines if the stream has already been closed. This happens when +// a FIN has been set by the sender and acknowledged by the receiver. +func (s *stream) closed() bool { + return s.finSeen && s.fin.LessThan(s.una) +} + +// rwndSize returns the stream's receive window size. +func (s *stream) rwndSize() seqnum.Size { + return s.una.Size(s.end) +} + +// windowSize returns the stream's window size accounting for scale. +func (s *stream) windowSize(tcp header.TCP) seqnum.Size { + return seqnum.Size(tcp.WindowSize()) << s.shiftCnt +} + +// logicalLenSyn calculates the logical length of a SYN (without ACK) segment. +// It is similar to logicalLen, but does not impose a window size requirement +// because of the SYN. +func logicalLenSyn(tcp header.TCP, dataLen int) seqnum.Size { + length := seqnum.Size(dataLen) + flags := tcp.Flags() + if flags&header.TCPFlagSyn != 0 { + length++ + } + if flags&header.TCPFlagFin != 0 { + length++ + } + return length +} + +// logicalLen calculates the logical length of the TCP segment. +func logicalLen(tcp header.TCP, dataLen int, windowSize seqnum.Size) seqnum.Size { + // If the segment is too large, TCP trims the payload per RFC 793 page 70. + length := logicalLenSyn(tcp, dataLen) + if length > windowSize { + length = windowSize + } + return length +} + +// IsEmpty returns true if tcb is not initialized. +func (t *TCB) IsEmpty() bool { + if t.reply != (stream{}) || t.original != (stream{}) { + return false + } + + if t.firstFin != nil || t.state != ResultDrop { + return false + } + + return true +} diff --git a/pkg/tcpip/transport/tcpconntrack/tcpconntrack_state_autogen.go b/pkg/tcpip/transport/tcpconntrack/tcpconntrack_state_autogen.go new file mode 100644 index 0000000..45f25c8 --- /dev/null +++ b/pkg/tcpip/transport/tcpconntrack/tcpconntrack_state_autogen.go @@ -0,0 +1,91 @@ +// automatically generated by stateify. + +package tcpconntrack + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (t *TCB) StateTypeName() string { + return "pkg/tcpip/transport/tcpconntrack.TCB" +} + +func (t *TCB) StateFields() []string { + return []string{ + "reply", + "original", + "firstFin", + "state", + } +} + +func (t *TCB) beforeSave() {} + +// +checklocksignore +func (t *TCB) StateSave(stateSinkObject state.Sink) { + t.beforeSave() + stateSinkObject.Save(0, &t.reply) + stateSinkObject.Save(1, &t.original) + stateSinkObject.Save(2, &t.firstFin) + stateSinkObject.Save(3, &t.state) +} + +func (t *TCB) afterLoad(context.Context) {} + +// +checklocksignore +func (t *TCB) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &t.reply) + stateSourceObject.Load(1, &t.original) + stateSourceObject.Load(2, &t.firstFin) + stateSourceObject.Load(3, &t.state) +} + +func (s *stream) StateTypeName() string { + return "pkg/tcpip/transport/tcpconntrack.stream" +} + +func (s *stream) StateFields() []string { + return []string{ + "una", + "nxt", + "end", + "finSeen", + "fin", + "rstSeen", + "shiftCnt", + } +} + +func (s *stream) beforeSave() {} + +// +checklocksignore +func (s *stream) StateSave(stateSinkObject state.Sink) { + s.beforeSave() + stateSinkObject.Save(0, &s.una) + stateSinkObject.Save(1, &s.nxt) + stateSinkObject.Save(2, &s.end) + stateSinkObject.Save(3, &s.finSeen) + stateSinkObject.Save(4, &s.fin) + stateSinkObject.Save(5, &s.rstSeen) + stateSinkObject.Save(6, &s.shiftCnt) +} + +func (s *stream) afterLoad(context.Context) {} + +// +checklocksignore +func (s *stream) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &s.una) + stateSourceObject.Load(1, &s.nxt) + stateSourceObject.Load(2, &s.end) + stateSourceObject.Load(3, &s.finSeen) + stateSourceObject.Load(4, &s.fin) + stateSourceObject.Load(5, &s.rstSeen) + stateSourceObject.Load(6, &s.shiftCnt) +} + +func init() { + state.Register((*TCB)(nil)) + state.Register((*stream)(nil)) +} diff --git a/pkg/tcpip/transport/transport.go b/pkg/tcpip/transport/transport.go new file mode 100644 index 0000000..4c2ae87 --- /dev/null +++ b/pkg/tcpip/transport/transport.go @@ -0,0 +1,16 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package transport supports transport protocols. +package transport diff --git a/pkg/tcpip/transport/transport_state_autogen.go b/pkg/tcpip/transport/transport_state_autogen.go new file mode 100644 index 0000000..c023165 --- /dev/null +++ b/pkg/tcpip/transport/transport_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package transport diff --git a/pkg/tcpip/transport/udp/endpoint.go b/pkg/tcpip/transport/udp/endpoint.go new file mode 100644 index 0000000..6f6a64c --- /dev/null +++ b/pkg/tcpip/transport/udp/endpoint.go @@ -0,0 +1,1098 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package udp + +import ( + "bytes" + "fmt" + "io" + "math" + "time" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/ports" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" + "github.com/sagernet/gvisor/pkg/tcpip/transport/internal/network" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// +stateify savable +type udpPacket struct { + udpPacketEntry + netProto tcpip.NetworkProtocolNumber + senderAddress tcpip.FullAddress + destinationAddress tcpip.FullAddress + packetInfo tcpip.IPPacketInfo + pkt *stack.PacketBuffer + receivedAt time.Time `state:".(int64)"` + // tosOrTClass stores either the Type of Service for IPv4 or the Traffic Class + // for IPv6. + tosOrTClass uint8 + // ttlOrHopLimit stores either the TTL for IPv4 or the HopLimit for IPv6 + ttlOrHopLimit uint8 +} + +// endpoint represents a UDP endpoint. This struct serves as the interface +// between users of the endpoint and the protocol implementation; it is legal to +// have concurrent goroutines make calls into the endpoint, they are properly +// synchronized. +// +// It implements tcpip.Endpoint. +// +// +stateify savable +type endpoint struct { + tcpip.DefaultSocketOptionsHandler + + // The following fields are initialized at creation time and do not + // change throughout the lifetime of the endpoint. + stack *stack.Stack + waiterQueue *waiter.Queue + net network.Endpoint + stats tcpip.TransportEndpointStats + ops tcpip.SocketOptions + + // The following fields are used to manage the receive queue, and are + // protected by rcvMu. + rcvMu sync.Mutex `state:"nosave"` + rcvReady bool + rcvList udpPacketList + rcvBufSize int + rcvClosed bool + + lastErrorMu sync.Mutex `state:"nosave"` + lastError tcpip.Error + + // The following fields are protected by the mu mutex. + mu sync.RWMutex `state:"nosave"` + portFlags ports.Flags + + // Values used to reserve a port or register a transport endpoint. + // (which ever happens first). + boundBindToDevice tcpip.NICID + boundPortFlags ports.Flags + + readShutdown bool + + // effectiveNetProtos contains the network protocols actually in use. In + // most cases it will only contain "netProto", but in cases like IPv6 + // endpoints with v6only set to false, this could include multiple + // protocols (e.g., IPv6 and IPv4) or a single different protocol (e.g., + // IPv4 when IPv6 endpoint is bound or connected to an IPv4 mapped + // address). + effectiveNetProtos []tcpip.NetworkProtocolNumber + + // frozen indicates if the packets should be delivered to the endpoint + // during restore. + frozen bool + + localPort uint16 + remotePort uint16 +} + +func newEndpoint(s *stack.Stack, netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) *endpoint { + e := &endpoint{ + stack: s, + waiterQueue: waiterQueue, + } + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + e.ops.SetMulticastLoop(true) + e.ops.SetSendBufferSize(32*1024, false /* notify */) + e.ops.SetReceiveBufferSize(32*1024, false /* notify */) + e.net.Init(s, netProto, header.UDPProtocolNumber, &e.ops, waiterQueue) + + // Override with stack defaults. + var ss tcpip.SendBufferSizeOption + if err := s.Option(&ss); err == nil { + e.ops.SetSendBufferSize(int64(ss.Default), false /* notify */) + } + + var rs tcpip.ReceiveBufferSizeOption + if err := s.Option(&rs); err == nil { + e.ops.SetReceiveBufferSize(int64(rs.Default), false /* notify */) + } + + return e +} + +// WakeupWriters implements tcpip.SocketOptionsHandler. +func (e *endpoint) WakeupWriters() { + e.net.MaybeSignalWritable() +} + +func (e *endpoint) LastError() tcpip.Error { + e.lastErrorMu.Lock() + defer e.lastErrorMu.Unlock() + + err := e.lastError + e.lastError = nil + return err +} + +// UpdateLastError implements tcpip.SocketOptionsHandler. +func (e *endpoint) UpdateLastError(err tcpip.Error) { + e.lastErrorMu.Lock() + e.lastError = err + e.lastErrorMu.Unlock() +} + +// Abort implements stack.TransportEndpoint. +func (e *endpoint) Abort() { + e.Close() +} + +// Close puts the endpoint in a closed state and frees all resources +// associated with it. +func (e *endpoint) Close() { + e.mu.Lock() + defer e.mu.Unlock() + e.closeLocked() +} + +// Preconditions: e.mu is locked. +// +checklocks:e.mu +func (e *endpoint) closeLocked() { + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial: + case transport.DatagramEndpointStateClosed: + return + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + id := e.net.Info().ID + id.LocalPort = e.localPort + id.RemotePort = e.remotePort + e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, ProtocolNumber, id, e, e.boundPortFlags, e.boundBindToDevice) + portRes := ports.Reservation{ + Networks: e.effectiveNetProtos, + Transport: ProtocolNumber, + Addr: id.LocalAddress, + Port: id.LocalPort, + Flags: e.boundPortFlags, + BindToDevice: e.boundBindToDevice, + Dest: tcpip.FullAddress{}, + } + e.stack.ReleasePort(portRes) + e.boundBindToDevice = 0 + e.boundPortFlags = ports.Flags{} + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } + + // Close the receive list and drain it. + e.rcvMu.Lock() + e.rcvClosed = true + e.rcvBufSize = 0 + for !e.rcvList.Empty() { + p := e.rcvList.Front() + e.rcvList.Remove(p) + p.pkt.DecRef() + } + e.rcvMu.Unlock() + + e.net.Shutdown() + e.net.Close() + e.readShutdown = true + + e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents) +} + +// ModerateRecvBuf implements tcpip.Endpoint. +func (*endpoint) ModerateRecvBuf(int) {} + +// Read implements tcpip.Endpoint. +func (e *endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult, tcpip.Error) { + if err := e.LastError(); err != nil { + return tcpip.ReadResult{}, err + } + + e.rcvMu.Lock() + + if e.rcvList.Empty() { + var err tcpip.Error = &tcpip.ErrWouldBlock{} + if e.rcvClosed { + e.stats.ReadErrors.ReadClosed.Increment() + err = &tcpip.ErrClosedForReceive{} + } + e.rcvMu.Unlock() + return tcpip.ReadResult{}, err + } + + p := e.rcvList.Front() + if !opts.Peek { + e.rcvList.Remove(p) + defer p.pkt.DecRef() + e.rcvBufSize -= p.pkt.Data().Size() + } + e.rcvMu.Unlock() + + // Control Messages + // TODO(https://gvisor.dev/issue/7012): Share control message code with other + // network endpoints. + cm := tcpip.ReceivableControlMessages{ + HasTimestamp: true, + Timestamp: p.receivedAt, + } + switch p.netProto { + case header.IPv4ProtocolNumber: + if e.ops.GetReceiveTOS() { + cm.HasTOS = true + cm.TOS = p.tosOrTClass + } + if e.ops.GetReceiveTTL() { + cm.HasTTL = true + cm.TTL = p.ttlOrHopLimit + } + if e.ops.GetReceivePacketInfo() { + cm.HasIPPacketInfo = true + cm.PacketInfo = p.packetInfo + } + case header.IPv6ProtocolNumber: + if e.ops.GetReceiveTClass() { + cm.HasTClass = true + // Although TClass is an 8-bit value it's read in the CMsg as a uint32. + cm.TClass = uint32(p.tosOrTClass) + } + if e.ops.GetReceiveHopLimit() { + cm.HasHopLimit = true + cm.HopLimit = p.ttlOrHopLimit + } + if e.ops.GetIPv6ReceivePacketInfo() { + cm.HasIPv6PacketInfo = true + cm.IPv6PacketInfo = tcpip.IPv6PacketInfo{ + NIC: p.packetInfo.NIC, + Addr: p.packetInfo.DestinationAddr, + } + } + default: + panic(fmt.Sprintf("unrecognized network protocol = %d", p.netProto)) + } + + if e.ops.GetReceiveOriginalDstAddress() { + cm.HasOriginalDstAddress = true + cm.OriginalDstAddress = p.destinationAddress + } + + // Read Result + res := tcpip.ReadResult{ + Total: p.pkt.Data().Size(), + ControlMessages: cm, + } + if opts.NeedRemoteAddr { + res.RemoteAddr = p.senderAddress + } + + n, err := p.pkt.Data().ReadTo(dst, opts.Peek) + if n == 0 && err != nil { + return res, &tcpip.ErrBadBuffer{} + } + res.Count = n + return res, nil +} + +// prepareForWriteInner prepares the endpoint for sending data. In particular, +// it binds it if it's still in the initial state. To do so, it must first +// reacquire the mutex in exclusive mode. +// +// Returns true for retry if preparation should be retried. +// +checklocksread:e.mu +func (e *endpoint) prepareForWriteInner(to *tcpip.FullAddress) (retry bool, err tcpip.Error) { + switch e.net.State() { + case transport.DatagramEndpointStateInitial: + case transport.DatagramEndpointStateConnected: + return false, nil + + case transport.DatagramEndpointStateBound: + if to == nil { + return false, &tcpip.ErrDestinationRequired{} + } + return false, nil + default: + return false, &tcpip.ErrInvalidEndpointState{} + } + + e.mu.RUnlock() + e.mu.Lock() + defer e.mu.DowngradeLock() + + // The state changed when we released the shared locked and re-acquired + // it in exclusive mode. Try again. + if e.net.State() != transport.DatagramEndpointStateInitial { + return true, nil + } + + // The state is still 'initial', so try to bind the endpoint. + if err := e.bindLocked(tcpip.FullAddress{}); err != nil { + return false, err + } + + return true, nil +} + +var _ tcpip.EndpointWithPreflight = (*endpoint)(nil) + +// Validates the passed WriteOptions and prepares the endpoint for writes +// using those options. If the endpoint is unbound and the `To` address +// is specified, binds the endpoint to that address. +func (e *endpoint) Preflight(opts tcpip.WriteOptions) tcpip.Error { + var r bytes.Reader + udpInfo, err := e.prepareForWrite(&r, opts) + if err == nil { + udpInfo.ctx.Release() + } + return err +} + +// Write writes data to the endpoint's peer. This method does not block +// if the data cannot be written. +func (e *endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + n, err := e.write(p, opts) + switch err.(type) { + case nil: + e.stats.PacketsSent.Increment() + case *tcpip.ErrMessageTooLong, *tcpip.ErrInvalidOptionValue: + e.stats.WriteErrors.InvalidArgs.Increment() + case *tcpip.ErrClosedForSend: + e.stats.WriteErrors.WriteClosed.Increment() + case *tcpip.ErrInvalidEndpointState: + e.stats.WriteErrors.InvalidEndpointState.Increment() + case *tcpip.ErrHostUnreachable, *tcpip.ErrBroadcastDisabled, *tcpip.ErrNetworkUnreachable: + // Errors indicating any problem with IP routing of the packet. + e.stats.SendErrors.NoRoute.Increment() + default: + // For all other errors when writing to the network layer. + e.stats.SendErrors.SendToNetworkFailed.Increment() + } + return n, err +} + +func (e *endpoint) prepareForWrite(p tcpip.Payloader, opts tcpip.WriteOptions) (udpPacketInfo, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + // Prepare for write. + for { + retry, err := e.prepareForWriteInner(opts.To) + if err != nil { + return udpPacketInfo{}, err + } + + if !retry { + break + } + } + + dst, connected := e.net.GetRemoteAddress() + dst.Port = e.remotePort + if opts.To != nil { + if opts.To.Port == 0 { + // Port 0 is an invalid port to send to. + return udpPacketInfo{}, &tcpip.ErrInvalidEndpointState{} + } + + dst = *opts.To + } else if !connected { + return udpPacketInfo{}, &tcpip.ErrDestinationRequired{} + } + + ctx, err := e.net.AcquireContextForWrite(opts) + if err != nil { + return udpPacketInfo{}, err + } + + if p.Len() > header.UDPMaximumPacketSize { + // Native linux behaviour differs for IPv4 and IPv6 packets; IPv4 packet + // errors aren't report to the error queue at all. + if ctx.PacketInfo().NetProto == header.IPv6ProtocolNumber { + so := e.SocketOptions() + if so.GetIPv6RecvError() { + so.QueueLocalErr( + &tcpip.ErrMessageTooLong{}, + e.net.NetProto(), + uint32(p.Len()), + dst, + nil, + ) + } + } + ctx.Release() + return udpPacketInfo{}, &tcpip.ErrMessageTooLong{} + } + + return udpPacketInfo{ + ctx: ctx, + localPort: e.localPort, + remotePort: dst.Port, + }, nil +} + +func (e *endpoint) write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcpip.Error) { + // Do not hold lock when sending as loopback is synchronous and if the UDP + // datagram ends up generating an ICMP response then it can result in a + // deadlock where the ICMP response handling ends up acquiring this endpoint's + // mutex using e.mu.RLock() in endpoint.HandleControlPacket which can cause a + // deadlock if another caller is trying to acquire e.mu in exclusive mode w/ + // e.mu.Lock(). Since e.mu.Lock() prevents any new read locks to ensure the + // lock can be eventually acquired. + // + // See: https://golang.org/pkg/sync/#RWMutex for details on why recursive read + // locking is prohibited. + + if err := e.LastError(); err != nil { + return 0, err + } + + udpInfo, err := e.prepareForWrite(p, opts) + if err != nil { + return 0, err + } + defer udpInfo.ctx.Release() + + dataSz := p.Len() + pktInfo := udpInfo.ctx.PacketInfo() + pkt := udpInfo.ctx.TryNewPacketBufferFromPayloader(header.UDPMinimumSize+int(pktInfo.MaxHeaderLength), p) + if pkt == nil { + return 0, &tcpip.ErrWouldBlock{} + } + defer pkt.DecRef() + + // Initialize the UDP header. + udp := header.UDP(pkt.TransportHeader().Push(header.UDPMinimumSize)) + pkt.TransportProtocolNumber = ProtocolNumber + + length := uint16(pkt.Size()) + udp.Encode(&header.UDPFields{ + SrcPort: udpInfo.localPort, + DstPort: udpInfo.remotePort, + Length: length, + }) + + // Set the checksum field unless TX checksum offload is enabled. + // On IPv4, UDP checksum is optional, and a zero value indicates the + // transmitter skipped the checksum generation (RFC768). + // On IPv6, UDP checksum is not optional (RFC2460 Section 8.1). + if pktInfo.RequiresTXTransportChecksum && + (!e.ops.GetNoChecksum() || pktInfo.NetProto == header.IPv6ProtocolNumber) { + xsum := udp.CalculateChecksum(checksum.Combine( + header.PseudoHeaderChecksum(ProtocolNumber, pktInfo.LocalAddress, pktInfo.RemoteAddress, length), + pkt.Data().Checksum(), + )) + // As per RFC 768 page 2, + // + // Checksum is the 16-bit one's complement of the one's complement sum of + // a pseudo header of information from the IP header, the UDP header, and + // the data, padded with zero octets at the end (if necessary) to make a + // multiple of two octets. + // + // The pseudo header conceptually prefixed to the UDP header contains the + // source address, the destination address, the protocol, and the UDP + // length. This information gives protection against misrouted datagrams. + // This checksum procedure is the same as is used in TCP. + // + // If the computed checksum is zero, it is transmitted as all ones (the + // equivalent in one's complement arithmetic). An all zero transmitted + // checksum value means that the transmitter generated no checksum (for + // debugging or for higher level protocols that don't care). + // + // To avoid the zero value, we only calculate the one's complement of the + // one's complement sum if the sum is not all ones. + if xsum != math.MaxUint16 { + xsum = ^xsum + } + udp.SetChecksum(xsum) + } + if err := udpInfo.ctx.WritePacket(pkt, false /* headerIncluded */); err != nil { + e.stack.Stats().UDP.PacketSendErrors.Increment() + return 0, err + } + + // Track count of packets sent. + e.stack.Stats().UDP.PacketsSent.Increment() + return int64(dataSz), nil +} + +// OnReuseAddressSet implements tcpip.SocketOptionsHandler. +func (e *endpoint) OnReuseAddressSet(v bool) { + e.mu.Lock() + e.portFlags.MostRecent = v + e.mu.Unlock() +} + +// OnReusePortSet implements tcpip.SocketOptionsHandler. +func (e *endpoint) OnReusePortSet(v bool) { + e.mu.Lock() + e.portFlags.LoadBalanced = v + e.mu.Unlock() +} + +// SetSockOptInt implements tcpip.Endpoint. +func (e *endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { + return e.net.SetSockOptInt(opt, v) +} + +var _ tcpip.SocketOptionsHandler = (*endpoint)(nil) + +// HasNIC implements tcpip.SocketOptionsHandler. +func (e *endpoint) HasNIC(id int32) bool { + return e.stack.HasNIC(tcpip.NICID(id)) +} + +// SetSockOpt implements tcpip.Endpoint. +func (e *endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { + return e.net.SetSockOpt(opt) +} + +// GetSockOptInt implements tcpip.Endpoint. +func (e *endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { + switch opt { + case tcpip.ReceiveQueueSizeOption: + v := 0 + e.rcvMu.Lock() + if !e.rcvList.Empty() { + p := e.rcvList.Front() + v = p.pkt.Data().Size() + } + e.rcvMu.Unlock() + return v, nil + + default: + return e.net.GetSockOptInt(opt) + } +} + +// GetSockOpt implements tcpip.Endpoint. +func (e *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { + return e.net.GetSockOpt(opt) +} + +// udpPacketInfo holds information needed to send a UDP packet. +type udpPacketInfo struct { + ctx network.WriteContext + localPort uint16 + remotePort uint16 +} + +// Disconnect implements tcpip.Endpoint. +func (e *endpoint) Disconnect() tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + if e.net.State() != transport.DatagramEndpointStateConnected { + return nil + } + var ( + id stack.TransportEndpointID + btd tcpip.NICID + ) + + // We change this value below and we need the old value to unregister + // the endpoint. + boundPortFlags := e.boundPortFlags + + // Exclude ephemerally bound endpoints. + info := e.net.Info() + info.ID.LocalPort = e.localPort + info.ID.RemotePort = e.remotePort + if e.net.WasBound() { + var err tcpip.Error + id = stack.TransportEndpointID{ + LocalPort: info.ID.LocalPort, + LocalAddress: info.ID.LocalAddress, + } + id, btd, err = e.registerWithStack(e.effectiveNetProtos, id) + if err != nil { + return err + } + boundPortFlags = e.boundPortFlags + } else { + if info.ID.LocalPort != 0 { + // Release the ephemeral port. + portRes := ports.Reservation{ + Networks: e.effectiveNetProtos, + Transport: ProtocolNumber, + Addr: info.ID.LocalAddress, + Port: info.ID.LocalPort, + Flags: boundPortFlags, + BindToDevice: e.boundBindToDevice, + Dest: tcpip.FullAddress{}, + } + e.stack.ReleasePort(portRes) + e.boundPortFlags = ports.Flags{} + } + } + + e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, ProtocolNumber, info.ID, e, boundPortFlags, e.boundBindToDevice) + e.boundBindToDevice = btd + e.localPort = id.LocalPort + e.remotePort = id.RemotePort + + e.net.Disconnect() + + return nil +} + +// Connect connects the endpoint to its peer. Specifying a NIC is optional. +func (e *endpoint) Connect(addr tcpip.FullAddress) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + err := e.net.ConnectAndThen(addr, func(netProto tcpip.NetworkProtocolNumber, previousID, nextID stack.TransportEndpointID) tcpip.Error { + nextID.LocalPort = e.localPort + nextID.RemotePort = addr.Port + + // Even if we're connected, this endpoint can still be used to send + // packets on a different network protocol, so we register both even if + // v6only is set to false and this is an ipv6 endpoint. + netProtos := []tcpip.NetworkProtocolNumber{netProto} + if netProto == header.IPv6ProtocolNumber && !e.ops.GetV6Only() && e.stack.CheckNetworkProtocol(header.IPv4ProtocolNumber) { + netProtos = []tcpip.NetworkProtocolNumber{ + header.IPv4ProtocolNumber, + header.IPv6ProtocolNumber, + } + } + + oldPortFlags := e.boundPortFlags + + // Remove the old registration. + if e.localPort != 0 { + previousID.LocalPort = e.localPort + previousID.RemotePort = e.remotePort + e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, ProtocolNumber, previousID, e, oldPortFlags, e.boundBindToDevice) + } + + nextID, btd, err := e.registerWithStack(netProtos, nextID) + if err != nil { + return err + } + + e.localPort = nextID.LocalPort + e.remotePort = nextID.RemotePort + e.boundBindToDevice = btd + e.effectiveNetProtos = netProtos + return nil + }) + if err != nil { + return err + } + + e.rcvMu.Lock() + e.rcvReady = true + e.rcvMu.Unlock() + return nil +} + +// ConnectEndpoint is not supported. +func (*endpoint) ConnectEndpoint(tcpip.Endpoint) tcpip.Error { + return &tcpip.ErrInvalidEndpointState{} +} + +// Shutdown closes the read and/or write end of the endpoint connection +// to its peer. +func (e *endpoint) Shutdown(flags tcpip.ShutdownFlags) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: + return &tcpip.ErrNotConnected{} + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + default: + panic(fmt.Sprintf("unhandled state = %s", state)) + } + + if flags&tcpip.ShutdownWrite != 0 { + if err := e.net.Shutdown(); err != nil { + return err + } + } + + if flags&tcpip.ShutdownRead != 0 { + e.readShutdown = true + + e.rcvMu.Lock() + wasClosed := e.rcvClosed + e.rcvClosed = true + e.rcvMu.Unlock() + + if !wasClosed { + e.waiterQueue.Notify(waiter.ReadableEvents) + } + } + + if e.net.State() == transport.DatagramEndpointStateBound { + return &tcpip.ErrNotConnected{} + } + return nil +} + +// Listen is not supported by UDP, it just fails. +func (*endpoint) Listen(int) tcpip.Error { + return &tcpip.ErrNotSupported{} +} + +// Accept is not supported by UDP, it just fails. +func (*endpoint) Accept(*tcpip.FullAddress) (tcpip.Endpoint, *waiter.Queue, tcpip.Error) { + return nil, nil, &tcpip.ErrNotSupported{} +} + +func (e *endpoint) registerWithStack(netProtos []tcpip.NetworkProtocolNumber, id stack.TransportEndpointID) (stack.TransportEndpointID, tcpip.NICID, tcpip.Error) { + bindToDevice := tcpip.NICID(e.ops.GetBindToDevice()) + if e.localPort == 0 { + portRes := ports.Reservation{ + Networks: netProtos, + Transport: ProtocolNumber, + Addr: id.LocalAddress, + Port: id.LocalPort, + Flags: e.portFlags, + BindToDevice: bindToDevice, + Dest: tcpip.FullAddress{}, + } + port, err := e.stack.ReservePort(e.stack.SecureRNG(), portRes, nil /* testPort */) + if err != nil { + return id, bindToDevice, err + } + id.LocalPort = port + } + e.boundPortFlags = e.portFlags + + err := e.stack.RegisterTransportEndpoint(netProtos, ProtocolNumber, id, e, e.boundPortFlags, bindToDevice) + if err != nil { + portRes := ports.Reservation{ + Networks: netProtos, + Transport: ProtocolNumber, + Addr: id.LocalAddress, + Port: id.LocalPort, + Flags: e.boundPortFlags, + BindToDevice: bindToDevice, + Dest: tcpip.FullAddress{}, + } + e.stack.ReleasePort(portRes) + e.boundPortFlags = ports.Flags{} + } + return id, bindToDevice, err +} + +func (e *endpoint) bindLocked(addr tcpip.FullAddress) tcpip.Error { + // Don't allow binding once endpoint is not in the initial state + // anymore. + if e.net.State() != transport.DatagramEndpointStateInitial { + return &tcpip.ErrInvalidEndpointState{} + } + + err := e.net.BindAndThen(addr, func(boundNetProto tcpip.NetworkProtocolNumber, boundAddr tcpip.Address) tcpip.Error { + // Expand netProtos to include v4 and v6 if the caller is binding to a + // wildcard (empty) address, and this is an IPv6 endpoint with v6only + // set to false. + netProtos := []tcpip.NetworkProtocolNumber{boundNetProto} + if boundNetProto == header.IPv6ProtocolNumber && !e.ops.GetV6Only() && boundAddr == (tcpip.Address{}) && e.stack.CheckNetworkProtocol(header.IPv4ProtocolNumber) { + netProtos = []tcpip.NetworkProtocolNumber{ + header.IPv6ProtocolNumber, + header.IPv4ProtocolNumber, + } + } + + id := stack.TransportEndpointID{ + LocalPort: addr.Port, + LocalAddress: boundAddr, + } + id, btd, err := e.registerWithStack(netProtos, id) + if err != nil { + return err + } + + e.localPort = id.LocalPort + e.boundBindToDevice = btd + e.effectiveNetProtos = netProtos + return nil + }) + if err != nil { + return err + } + + e.rcvMu.Lock() + e.rcvReady = true + e.rcvMu.Unlock() + return nil +} + +// Bind binds the endpoint to a specific local address and port. +// Specifying a NIC is optional. +func (e *endpoint) Bind(addr tcpip.FullAddress) tcpip.Error { + e.mu.Lock() + defer e.mu.Unlock() + + err := e.bindLocked(addr) + if err != nil { + return err + } + + return nil +} + +// GetLocalAddress returns the address to which the endpoint is bound. +func (e *endpoint) GetLocalAddress() (tcpip.FullAddress, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + addr := e.net.GetLocalAddress() + addr.Port = e.localPort + return addr, nil +} + +// GetRemoteAddress returns the address to which the endpoint is connected. +func (e *endpoint) GetRemoteAddress() (tcpip.FullAddress, tcpip.Error) { + e.mu.RLock() + defer e.mu.RUnlock() + + addr, connected := e.net.GetRemoteAddress() + if !connected || e.remotePort == 0 { + return tcpip.FullAddress{}, &tcpip.ErrNotConnected{} + } + + addr.Port = e.remotePort + return addr, nil +} + +// Readiness returns the current readiness of the endpoint. For example, if +// waiter.EventIn is set, the endpoint is immediately readable. +func (e *endpoint) Readiness(mask waiter.EventMask) waiter.EventMask { + var result waiter.EventMask + + if e.net.HasSendSpace() { + result |= waiter.WritableEvents & mask + } + + // Determine if the endpoint is readable if requested. + if mask&waiter.ReadableEvents != 0 { + e.rcvMu.Lock() + if !e.rcvList.Empty() || e.rcvClosed { + result |= waiter.ReadableEvents + } + e.rcvMu.Unlock() + } + + e.lastErrorMu.Lock() + hasError := e.lastError != nil + e.lastErrorMu.Unlock() + if hasError { + result |= waiter.EventErr + } + return result +} + +// HandlePacket is called by the stack when new packets arrive to this transport +// endpoint. +func (e *endpoint) HandlePacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) { + // Get the header then trim it from the view. + hdr := header.UDP(pkt.TransportHeader().Slice()) + netHdr := pkt.Network() + lengthValid, csumValid := header.UDPValid( + hdr, + func() uint16 { return pkt.Data().Checksum() }, + uint16(pkt.Data().Size()), + pkt.NetworkProtocolNumber, + netHdr.SourceAddress(), + netHdr.DestinationAddress(), + pkt.RXChecksumValidated) + if !lengthValid { + // Malformed packet. + e.stack.Stats().UDP.MalformedPacketsReceived.Increment() + e.stats.ReceiveErrors.MalformedPacketsReceived.Increment() + return + } + + if !csumValid { + e.stack.Stats().UDP.ChecksumErrors.Increment() + e.stats.ReceiveErrors.ChecksumErrors.Increment() + return + } + + e.stack.Stats().UDP.PacketsReceived.Increment() + e.stats.PacketsReceived.Increment() + + e.rcvMu.Lock() + // Drop the packet if our buffer is not ready to receive packets. + if !e.rcvReady || e.rcvClosed { + e.rcvMu.Unlock() + e.stack.Stats().UDP.ReceiveBufferErrors.Increment() + e.stats.ReceiveErrors.ClosedReceiver.Increment() + return + } + + rcvBufSize := e.ops.GetReceiveBufferSize() + // Drop the packet if our buffer is currently full. + if e.frozen || e.rcvBufSize >= int(rcvBufSize) { + e.rcvMu.Unlock() + e.stack.Stats().UDP.ReceiveBufferErrors.Increment() + e.stats.ReceiveErrors.ReceiveBufferOverflow.Increment() + return + } + + wasEmpty := e.rcvBufSize == 0 + + // Push new packet into receive list and increment the buffer size. + packet := &udpPacket{ + netProto: pkt.NetworkProtocolNumber, + senderAddress: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: id.RemoteAddress, + Port: hdr.SourcePort(), + }, + destinationAddress: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: id.LocalAddress, + Port: hdr.DestinationPort(), + }, + // We need to clone the packet because ReadTo modifies the write index of + // the underlying buffer. Clone does not copy the data, just the metadata. + pkt: pkt.Clone(), + } + e.rcvList.PushBack(packet) + e.rcvBufSize += pkt.Data().Size() + + // Save any useful information from the network header to the packet. + packet.tosOrTClass, _ = pkt.Network().TOS() + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + packet.ttlOrHopLimit = header.IPv4(pkt.NetworkHeader().Slice()).TTL() + case header.IPv6ProtocolNumber: + packet.ttlOrHopLimit = header.IPv6(pkt.NetworkHeader().Slice()).HopLimit() + } + + // TODO(gvisor.dev/issue/3556): r.LocalAddress may be a multicast or broadcast + // address. packetInfo.LocalAddr should hold a unicast address that can be + // used to respond to the incoming packet. + localAddr := pkt.Network().DestinationAddress() + packet.packetInfo.LocalAddr = localAddr + packet.packetInfo.DestinationAddr = localAddr + packet.packetInfo.NIC = pkt.NICID + packet.receivedAt = e.stack.Clock().Now() + + e.rcvMu.Unlock() + + // Notify any waiters that there's data to be read now. + if wasEmpty { + e.waiterQueue.Notify(waiter.ReadableEvents) + } +} + +func (e *endpoint) onICMPError(err tcpip.Error, transErr stack.TransportError, pkt *stack.PacketBuffer) { + // Update last error first. + e.lastErrorMu.Lock() + e.lastError = err + e.lastErrorMu.Unlock() + + var recvErr bool + switch pkt.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + recvErr = e.SocketOptions().GetIPv4RecvError() + case header.IPv6ProtocolNumber: + recvErr = e.SocketOptions().GetIPv6RecvError() + default: + panic(fmt.Sprintf("unhandled network protocol number = %d", pkt.NetworkProtocolNumber)) + } + + if recvErr { + // Linux passes the payload without the UDP header. + payload := pkt.Data().AsRange().ToView() + udp := header.UDP(payload.AsSlice()) + if len(udp) >= header.UDPMinimumSize { + payload.TrimFront(header.UDPMinimumSize) + } + + id := e.net.Info().ID + e.mu.RLock() + e.SocketOptions().QueueErr(&tcpip.SockError{ + Err: err, + Cause: transErr, + Payload: payload, + Dst: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: id.RemoteAddress, + Port: e.remotePort, + }, + Offender: tcpip.FullAddress{ + NIC: pkt.NICID, + Addr: id.LocalAddress, + Port: e.localPort, + }, + NetProto: pkt.NetworkProtocolNumber, + }) + e.mu.RUnlock() + } + + // Notify of the error. + e.waiterQueue.Notify(waiter.EventErr) +} + +// HandleError implements stack.TransportEndpoint. +func (e *endpoint) HandleError(transErr stack.TransportError, pkt *stack.PacketBuffer) { + // TODO(gvisor.dev/issues/5270): Handle all transport errors. + switch transErr.Kind() { + case stack.DestinationPortUnreachableTransportError: + if e.net.State() == transport.DatagramEndpointStateConnected { + e.onICMPError(&tcpip.ErrConnectionRefused{}, transErr, pkt) + } + } +} + +// State implements tcpip.Endpoint. +func (e *endpoint) State() uint32 { + return uint32(e.net.State()) +} + +// Info returns a copy of the endpoint info. +func (e *endpoint) Info() tcpip.EndpointInfo { + e.mu.RLock() + defer e.mu.RUnlock() + info := e.net.Info() + info.ID.LocalPort = e.localPort + info.ID.RemotePort = e.remotePort + return &info +} + +// Stats returns a pointer to the endpoint stats. +func (e *endpoint) Stats() tcpip.EndpointStats { + return &e.stats +} + +// Wait implements tcpip.Endpoint. +func (*endpoint) Wait() {} + +// SetOwner implements tcpip.Endpoint. +func (e *endpoint) SetOwner(owner tcpip.PacketOwner) { + e.net.SetOwner(owner) +} + +// SocketOptions implements tcpip.Endpoint. +func (e *endpoint) SocketOptions() *tcpip.SocketOptions { + return &e.ops +} + +// freeze prevents any more packets from being delivered to the endpoint. +func (e *endpoint) freeze() { + e.mu.Lock() + e.frozen = true + e.mu.Unlock() +} + +// thaw unfreezes a previously frozen endpoint using endpoint.freeze() allows +// new packets to be delivered again. +func (e *endpoint) thaw() { + e.mu.Lock() + e.frozen = false + e.mu.Unlock() +} diff --git a/pkg/tcpip/transport/udp/endpoint_state.go b/pkg/tcpip/transport/udp/endpoint_state.go new file mode 100644 index 0000000..66ed425 --- /dev/null +++ b/pkg/tcpip/transport/udp/endpoint_state.go @@ -0,0 +1,96 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package udp + +import ( + "context" + "time" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport" +) + +// saveReceivedAt is invoked by stateify. +func (p *udpPacket) saveReceivedAt() int64 { + return p.receivedAt.UnixNano() +} + +// loadReceivedAt is invoked by stateify. +func (p *udpPacket) loadReceivedAt(_ context.Context, nsec int64) { + p.receivedAt = time.Unix(0, nsec) +} + +// afterLoad is invoked by stateify. +func (e *endpoint) afterLoad(ctx context.Context) { + if e.stack.IsSaveRestoreEnabled() { + e.stack.RegisterRestoredEndpoint(e) + } else { + stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) + } +} + +// beforeSave is invoked by stateify. +func (e *endpoint) beforeSave() { + e.freeze() + e.stack.RegisterResumableEndpoint(e) +} + +// Restore implements tcpip.RestoredEndpoint.Restore. +func (e *endpoint) Restore(s *stack.Stack) { + e.mu.Lock() + defer e.mu.Unlock() + + if err := e.net.Resume(s); err != nil { + log.Warningf("Closing the UDP endpoint as it cannot be restored, err: %v", err) + e.closeLocked() + return + } + + // Unfreeze the endpoint to handle packets. + e.frozen = false + if e.stack.IsSaveRestoreEnabled() { + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + return + } + e.stack = s + e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) + + switch state := e.net.State(); state { + case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: + case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: + // Our saved state had a port, but we don't actually have a + // reservation. We need to remove the port from our state, but still + // pass it to the reservation machinery. + var err tcpip.Error + id := e.net.Info().ID + id.LocalPort = e.localPort + id.RemotePort = e.remotePort + id, e.boundBindToDevice, err = e.registerWithStack(e.effectiveNetProtos, id) + if err != nil { + panic("registering udp endpoint with the stack failed during restore") + } + e.localPort = id.LocalPort + e.remotePort = id.RemotePort + default: + panic("unhandled state") + } +} + +// Resume implements tcpip.ResumableEndpoint.Resume. +func (e *endpoint) Resume() { + e.thaw() +} diff --git a/pkg/tcpip/transport/udp/forwarder.go b/pkg/tcpip/transport/udp/forwarder.go new file mode 100644 index 0000000..3205b1a --- /dev/null +++ b/pkg/tcpip/transport/udp/forwarder.go @@ -0,0 +1,112 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package udp + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/waiter" +) + +// ForwarderHandler handles incoming requests. Returning true marks the +// request as handled, returning false marks the request as unhandled. +// Stack may send an ICMP port unreachable message for unhandled requests. +type ForwarderHandler func(*ForwarderRequest) (handled bool) + +// Forwarder is a session request forwarder, which allows clients to decide +// what to do with a session request, for example: ignore it, or process it. +// +// The canonical way of using it is to pass the Forwarder.HandlePacket function +// to stack.SetTransportProtocolHandler. +type Forwarder struct { + handler ForwarderHandler + + stack *stack.Stack +} + +// NewForwarder allocates and initializes a new forwarder. +func NewForwarder(s *stack.Stack, handler ForwarderHandler) *Forwarder { + return &Forwarder{ + stack: s, + handler: handler, + } +} + +// HandlePacket handles all packets. +// +// This function is expected to be passed as an argument to the +// stack.SetTransportProtocolHandler function. +func (f *Forwarder) HandlePacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) bool { + return f.handler(&ForwarderRequest{ + stack: f.stack, + id: id, + pkt: pkt.Clone(), + }) +} + +// ForwarderRequest represents a session request received by the forwarder and +// passed to the client. Clients may optionally create an endpoint to represent +// it via CreateEndpoint. +type ForwarderRequest struct { + stack *stack.Stack + id stack.TransportEndpointID + pkt *stack.PacketBuffer +} + +// ID returns the 4-tuple (src address, src port, dst address, dst port) that +// represents the session request. +func (r *ForwarderRequest) ID() stack.TransportEndpointID { + return r.id +} + +// CreateEndpoint creates a connected UDP endpoint for the session request. +func (r *ForwarderRequest) CreateEndpoint(queue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + ep := newEndpoint(r.stack, r.pkt.NetworkProtocolNumber, queue) + ep.mu.Lock() + defer ep.mu.Unlock() + + netHdr := r.pkt.Network() + if err := ep.net.Bind(tcpip.FullAddress{NIC: r.pkt.NICID, Addr: netHdr.DestinationAddress(), Port: r.id.LocalPort}); err != nil { + ep.closeLocked() + return nil, err + } + + if err := ep.net.Connect(tcpip.FullAddress{NIC: r.pkt.NICID, Addr: netHdr.SourceAddress(), Port: r.id.RemotePort}); err != nil { + ep.closeLocked() + return nil, err + } + + if err := r.stack.RegisterTransportEndpoint([]tcpip.NetworkProtocolNumber{r.pkt.NetworkProtocolNumber}, ProtocolNumber, r.id, ep, ep.portFlags, tcpip.NICID(ep.ops.GetBindToDevice())); err != nil { + ep.closeLocked() + return nil, err + } + + ep.localPort = r.id.LocalPort + ep.remotePort = r.id.RemotePort + ep.effectiveNetProtos = []tcpip.NetworkProtocolNumber{r.pkt.NetworkProtocolNumber} + ep.boundPortFlags = ep.portFlags + + ep.rcvMu.Lock() + ep.rcvReady = true + ep.rcvMu.Unlock() + + ep.HandlePacket(r.id, r.pkt) + + return ep, nil +} + +func (r *ForwarderRequest) Packet() *stack.PacketBuffer { + return r.pkt +} diff --git a/pkg/tcpip/transport/udp/protocol.go b/pkg/tcpip/transport/udp/protocol.go new file mode 100644 index 0000000..3fa6566 --- /dev/null +++ b/pkg/tcpip/transport/udp/protocol.go @@ -0,0 +1,138 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package udp contains the implementation of the UDP transport protocol. +package udp + +import ( + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/gvisor/pkg/tcpip/transport/raw" + "github.com/sagernet/gvisor/pkg/waiter" +) + +const ( + // ProtocolNumber is the udp protocol number. + ProtocolNumber = header.UDPProtocolNumber + + // MinBufferSize is the smallest size of a receive or send buffer. + MinBufferSize = 4 << 10 // 4KiB bytes. + + // DefaultSendBufferSize is the default size of the send buffer for + // an endpoint. + DefaultSendBufferSize = 32 << 10 // 32KiB + + // DefaultReceiveBufferSize is the default size of the receive buffer + // for an endpoint. + DefaultReceiveBufferSize = 32 << 10 // 32KiB + + // MaxBufferSize is the largest size a receive/send buffer can grow to. + MaxBufferSize = 4 << 20 // 4MiB +) + +// +stateify savable +type protocol struct { + stack *stack.Stack +} + +// Number returns the udp protocol number. +func (*protocol) Number() tcpip.TransportProtocolNumber { + return ProtocolNumber +} + +// NewEndpoint creates a new udp endpoint. +func (p *protocol) NewEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return newEndpoint(p.stack, netProto, waiterQueue), nil +} + +// NewRawEndpoint creates a new raw UDP endpoint. It implements +// stack.TransportProtocol.NewRawEndpoint. +func (p *protocol) NewRawEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) { + return raw.NewEndpoint(p.stack, netProto, header.UDPProtocolNumber, waiterQueue) +} + +// MinimumPacketSize returns the minimum valid udp packet size. +func (*protocol) MinimumPacketSize() int { + return header.UDPMinimumSize +} + +// ParsePorts returns the source and destination ports stored in the given udp +// packet. +func (*protocol) ParsePorts(v []byte) (src, dst uint16, err tcpip.Error) { + h := header.UDP(v) + return h.SourcePort(), h.DestinationPort(), nil +} + +// HandleUnknownDestinationPacket handles packets that are targeted at this +// protocol but don't match any existing endpoint. +func (p *protocol) HandleUnknownDestinationPacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) stack.UnknownDestinationPacketDisposition { + hdr := header.UDP(pkt.TransportHeader().Slice()) + netHdr := pkt.Network() + lengthValid, csumValid := header.UDPValid( + hdr, + func() uint16 { return pkt.Data().Checksum() }, + uint16(pkt.Data().Size()), + pkt.NetworkProtocolNumber, + netHdr.SourceAddress(), + netHdr.DestinationAddress(), + pkt.RXChecksumValidated) + if !lengthValid { + p.stack.Stats().UDP.MalformedPacketsReceived.Increment() + return stack.UnknownDestinationPacketMalformed + } + + if !csumValid { + p.stack.Stats().UDP.ChecksumErrors.Increment() + return stack.UnknownDestinationPacketMalformed + } + + return stack.UnknownDestinationPacketUnhandled +} + +// SetOption implements stack.TransportProtocol.SetOption. +func (*protocol) SetOption(tcpip.SettableTransportProtocolOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// Option implements stack.TransportProtocol.Option. +func (*protocol) Option(tcpip.GettableTransportProtocolOption) tcpip.Error { + return &tcpip.ErrUnknownProtocolOption{} +} + +// Close implements stack.TransportProtocol.Close. +func (*protocol) Close() {} + +// Wait implements stack.TransportProtocol.Wait. +func (*protocol) Wait() {} + +// Pause implements stack.TransportProtocol.Pause. +func (*protocol) Pause() {} + +// Resume implements stack.TransportProtocol.Resume. +func (*protocol) Resume() {} + +// Restore implements stack.TransportProtocol.Restore. +func (*protocol) Restore() {} + +// Parse implements stack.TransportProtocol.Parse. +func (*protocol) Parse(pkt *stack.PacketBuffer) bool { + return parse.UDP(pkt) +} + +// NewProtocol returns a UDP transport protocol. +func NewProtocol(s *stack.Stack) stack.TransportProtocol { + return &protocol{stack: s} +} diff --git a/pkg/tcpip/transport/udp/udp_packet_list.go b/pkg/tcpip/transport/udp/udp_packet_list.go new file mode 100644 index 0000000..ff855ef --- /dev/null +++ b/pkg/tcpip/transport/udp/udp_packet_list.go @@ -0,0 +1,239 @@ +package udp + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type udpPacketElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (udpPacketElementMapper) linkerFor(elem *udpPacket) *udpPacket { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type udpPacketList struct { + head *udpPacket + tail *udpPacket +} + +// Reset resets list l to the empty state. +func (l *udpPacketList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *udpPacketList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *udpPacketList) Front() *udpPacket { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *udpPacketList) Back() *udpPacket { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *udpPacketList) Len() (count int) { + for e := l.Front(); e != nil; e = (udpPacketElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *udpPacketList) PushFront(e *udpPacket) { + linker := udpPacketElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + udpPacketElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *udpPacketList) PushFrontList(m *udpPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + udpPacketElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + udpPacketElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *udpPacketList) PushBack(e *udpPacket) { + linker := udpPacketElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + udpPacketElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *udpPacketList) PushBackList(m *udpPacketList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + udpPacketElementMapper{}.linkerFor(l.tail).SetNext(m.head) + udpPacketElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *udpPacketList) InsertAfter(b, e *udpPacket) { + bLinker := udpPacketElementMapper{}.linkerFor(b) + eLinker := udpPacketElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + udpPacketElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *udpPacketList) InsertBefore(a, e *udpPacket) { + aLinker := udpPacketElementMapper{}.linkerFor(a) + eLinker := udpPacketElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + udpPacketElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *udpPacketList) Remove(e *udpPacket) { + linker := udpPacketElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + udpPacketElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + udpPacketElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type udpPacketEntry struct { + next *udpPacket + prev *udpPacket +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *udpPacketEntry) Next() *udpPacket { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *udpPacketEntry) Prev() *udpPacket { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *udpPacketEntry) SetNext(elem *udpPacket) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *udpPacketEntry) SetPrev(elem *udpPacket) { + e.prev = elem +} diff --git a/pkg/tcpip/transport/udp/udp_state_autogen.go b/pkg/tcpip/transport/udp/udp_state_autogen.go new file mode 100644 index 0000000..4b868be --- /dev/null +++ b/pkg/tcpip/transport/udp/udp_state_autogen.go @@ -0,0 +1,225 @@ +// automatically generated by stateify. + +package udp + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (p *udpPacket) StateTypeName() string { + return "pkg/tcpip/transport/udp.udpPacket" +} + +func (p *udpPacket) StateFields() []string { + return []string{ + "udpPacketEntry", + "netProto", + "senderAddress", + "destinationAddress", + "packetInfo", + "pkt", + "receivedAt", + "tosOrTClass", + "ttlOrHopLimit", + } +} + +func (p *udpPacket) beforeSave() {} + +// +checklocksignore +func (p *udpPacket) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + var receivedAtValue int64 + receivedAtValue = p.saveReceivedAt() + stateSinkObject.SaveValue(6, receivedAtValue) + stateSinkObject.Save(0, &p.udpPacketEntry) + stateSinkObject.Save(1, &p.netProto) + stateSinkObject.Save(2, &p.senderAddress) + stateSinkObject.Save(3, &p.destinationAddress) + stateSinkObject.Save(4, &p.packetInfo) + stateSinkObject.Save(5, &p.pkt) + stateSinkObject.Save(7, &p.tosOrTClass) + stateSinkObject.Save(8, &p.ttlOrHopLimit) +} + +func (p *udpPacket) afterLoad(context.Context) {} + +// +checklocksignore +func (p *udpPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.udpPacketEntry) + stateSourceObject.Load(1, &p.netProto) + stateSourceObject.Load(2, &p.senderAddress) + stateSourceObject.Load(3, &p.destinationAddress) + stateSourceObject.Load(4, &p.packetInfo) + stateSourceObject.Load(5, &p.pkt) + stateSourceObject.Load(7, &p.tosOrTClass) + stateSourceObject.Load(8, &p.ttlOrHopLimit) + stateSourceObject.LoadValue(6, new(int64), func(y any) { p.loadReceivedAt(ctx, y.(int64)) }) +} + +func (e *endpoint) StateTypeName() string { + return "pkg/tcpip/transport/udp.endpoint" +} + +func (e *endpoint) StateFields() []string { + return []string{ + "DefaultSocketOptionsHandler", + "stack", + "waiterQueue", + "net", + "stats", + "ops", + "rcvReady", + "rcvList", + "rcvBufSize", + "rcvClosed", + "lastError", + "portFlags", + "boundBindToDevice", + "boundPortFlags", + "readShutdown", + "effectiveNetProtos", + "frozen", + "localPort", + "remotePort", + } +} + +// +checklocksignore +func (e *endpoint) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.DefaultSocketOptionsHandler) + stateSinkObject.Save(1, &e.stack) + stateSinkObject.Save(2, &e.waiterQueue) + stateSinkObject.Save(3, &e.net) + stateSinkObject.Save(4, &e.stats) + stateSinkObject.Save(5, &e.ops) + stateSinkObject.Save(6, &e.rcvReady) + stateSinkObject.Save(7, &e.rcvList) + stateSinkObject.Save(8, &e.rcvBufSize) + stateSinkObject.Save(9, &e.rcvClosed) + stateSinkObject.Save(10, &e.lastError) + stateSinkObject.Save(11, &e.portFlags) + stateSinkObject.Save(12, &e.boundBindToDevice) + stateSinkObject.Save(13, &e.boundPortFlags) + stateSinkObject.Save(14, &e.readShutdown) + stateSinkObject.Save(15, &e.effectiveNetProtos) + stateSinkObject.Save(16, &e.frozen) + stateSinkObject.Save(17, &e.localPort) + stateSinkObject.Save(18, &e.remotePort) +} + +// +checklocksignore +func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.DefaultSocketOptionsHandler) + stateSourceObject.Load(1, &e.stack) + stateSourceObject.Load(2, &e.waiterQueue) + stateSourceObject.Load(3, &e.net) + stateSourceObject.Load(4, &e.stats) + stateSourceObject.Load(5, &e.ops) + stateSourceObject.Load(6, &e.rcvReady) + stateSourceObject.Load(7, &e.rcvList) + stateSourceObject.Load(8, &e.rcvBufSize) + stateSourceObject.Load(9, &e.rcvClosed) + stateSourceObject.Load(10, &e.lastError) + stateSourceObject.Load(11, &e.portFlags) + stateSourceObject.Load(12, &e.boundBindToDevice) + stateSourceObject.Load(13, &e.boundPortFlags) + stateSourceObject.Load(14, &e.readShutdown) + stateSourceObject.Load(15, &e.effectiveNetProtos) + stateSourceObject.Load(16, &e.frozen) + stateSourceObject.Load(17, &e.localPort) + stateSourceObject.Load(18, &e.remotePort) + stateSourceObject.AfterLoad(func() { e.afterLoad(ctx) }) +} + +func (p *protocol) StateTypeName() string { + return "pkg/tcpip/transport/udp.protocol" +} + +func (p *protocol) StateFields() []string { + return []string{ + "stack", + } +} + +func (p *protocol) beforeSave() {} + +// +checklocksignore +func (p *protocol) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.stack) +} + +func (p *protocol) afterLoad(context.Context) {} + +// +checklocksignore +func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.stack) +} + +func (l *udpPacketList) StateTypeName() string { + return "pkg/tcpip/transport/udp.udpPacketList" +} + +func (l *udpPacketList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *udpPacketList) beforeSave() {} + +// +checklocksignore +func (l *udpPacketList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *udpPacketList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *udpPacketList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *udpPacketEntry) StateTypeName() string { + return "pkg/tcpip/transport/udp.udpPacketEntry" +} + +func (e *udpPacketEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *udpPacketEntry) beforeSave() {} + +// +checklocksignore +func (e *udpPacketEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *udpPacketEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *udpPacketEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*udpPacket)(nil)) + state.Register((*endpoint)(nil)) + state.Register((*protocol)(nil)) + state.Register((*udpPacketList)(nil)) + state.Register((*udpPacketEntry)(nil)) +} diff --git a/pkg/timing/timing.go b/pkg/timing/timing.go new file mode 100644 index 0000000..8ca759c --- /dev/null +++ b/pkg/timing/timing.go @@ -0,0 +1,901 @@ +// Copyright 2025 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package timing provides a way to record the timing of a series of +// operations across one or more goroutines. +package timing + +import ( + "fmt" + "sort" + "strings" + "time" + "unicode/utf8" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/log" +) + +const ( + // DEBUG TIP: Set this to true if you find yourself debugging the case where + // you have stranded timers + // (Log messages like "Still waiting for XYZ child timelines to finish".) + // This will enable **thread-unsafe** code that logs which timers are still + // running. + debugStrandedTimers = false + + // fullTimestampFormat is the format string for a timestamp with nanosecond + // precision but no date component. + fullTimestampFormat = "15:04:05.000000000" + + // microsTimestampFormat is the format string for a timestamp with + // microsecond precision but no date component. + microsTimestampFormat = "15:04:05.000000" +) + +// Timeline is a series of points in time. +// +// A Timeline always has a defined start time, and will eventually have an end +// time. For this reason, `End` must always be called on a Timeline. +// +// A Timeline may have zero or more mid-points contained between the start and +// end times. +// +// A Timeline may fork to represent other timelines running concurrently. Such +// children Timelines may or may not end later than the parent does. +// +// A single Timeline struct should be owned by a single goroutine at a given +// time until it ends (i.e. its endpoint becomes defined), at which point +// ownership transfers to the goroutine that owns the Timer that created it. +// +// A Timeline may be nil, in which case all methods are no-ops. This means all +// code that takes in a Timeline parameter does not need to check for nilness. +type Timeline struct { + // name is the name of the Timeline. + name string + + // fullName is the fully-qualified name of this Timeline, including the + // names of its ancestors. + fullName string + + // timer is the Timer that owns this Timeline. + // This is nil on orphaned Timelines. + timer *Timer + + // start is when the Timeline started. + start time.Time + + // midpoints is a list of MidPoints that have been reached on this Timeline. + midpoints []MidPoint + + // end is when the Timeline ended. + // The zero value means the Timeline has not yet ended. + end time.Time + + // children is a list of forked timelines that are children of this one. + // Note that children do not necessarily need to end before the parent does. + children []*Timeline + + // invalidReason, if non-empty, is the reason why this Timeline is invalid. + // A Timeline is also invalid if any of its children are invalid. + invalidReason string +} + +// MidPoint is a named point in time on a Timeline. +// The starting and ending points of a Timeline are not MidPoints. +type MidPoint struct { + // when is when the midpoint was reached. + when time.Time + + // name is the name of the midpoint. + name string +} + +// Reached records a new midpoint on the Timeline. +func (s *Timeline) Reached(name string) { + if s == nil { + return + } + s.ReachedAt(name, time.Now()) +} + +// ReachedAt records a new midpoint on the root Timeline of the Timer with +// the given timestamp. +func (s *Timeline) ReachedAt(name string, when time.Time) { + if s == nil { + return + } + s.midpoints = append(s.midpoints, MidPoint{ + when: when, + name: name, + }) + if log.IsLogging(log.Debug) { + if s.timer != nil { + log.Debugf("Timer for %s: Timeline %s reached midpoint %s at %s (unix nanos: %d)", s.timer.root.name, s.fullName, name, when.Format(fullTimestampFormat), when.UnixNano()) + } else { + log.Debugf("Orphaned timeline %s reached midpoint %s at %s (unix nanos: %d)", s.name, name, when.Format(fullTimestampFormat), when.UnixNano()) + } + } +} + +// Fork creates a new Timeline that is a child of this one. +// A midpoint is implicitly added to the current Timeline. +// +// The returned Timeline is initially owned by the caller, but may be passed +// to another goroutine if desired. +// +// A child timeline may but does not need to end before the parent does. +// +// Forked timelines are useful to represent parallel operations like separate +// goroutines, and are actually required in such cases so that the goroutine +// can own its own Timeline, but non-concurrent code may also use Fork to +// represent its own linear operations as a tree if it so desires. +func (s *Timeline) Fork(name string) *Timeline { + if s == nil { + return nil + } + now := time.Now() + if s.timer == nil { + panic("timing.Timeline.Fork called on Timeline that has no parent; must call Timer.Adopt first") + } + s.timer.runningTimelines.Add(1) + sub := &Timeline{ + name: name, + fullName: fmt.Sprintf("%s/%s", s.fullName, name), + timer: s.timer, + start: now, + } + s.children = append(s.children, sub) + s.midpoints = append(s.midpoints, MidPoint{ + when: now, + name: "forked", + }) + // Check for log level here to avoid allocating a string to format the + // timestamp if it is not going to be logged. + if log.IsLogging(log.Debug) { + log.Debugf("Timer for %s: Timeline %s forked into child timeline %s at %s (unix nanos: %d)", s.timer.root.name, s.fullName, sub.fullName, now.Format(fullTimestampFormat), now.UnixNano()) + } + return sub +} + +// MultiFork creates new Timelines that are children of this one. +// It returns as many Timelines as there are names in `names`. +// All of them share the same start time. +// A midpoint is implicitly added to the current Timeline. +// The same semantics as `Timeline.Fork` apply. +func (s *Timeline) MultiFork(names []string) []*Timeline { + if len(names) == 0 { + return nil + } + if s == nil { + return make([]*Timeline, len(names)) + } + now := time.Now() + if s.timer == nil { + panic("timing.Timeline.MultiFork called on Timeline that has no parent; must call Timer.Adopt first") + } + s.timer.runningTimelines.Add(int64(len(names))) + children := make([]*Timeline, len(names)) + for i, name := range names { + children[i] = &Timeline{ + name: name, + fullName: fmt.Sprintf("%s/%s", s.fullName, name), + timer: s.timer, + start: now, + } + } + s.children = append(s.children, children...) + s.midpoints = append(s.midpoints, MidPoint{ + when: now, + name: "forked", + }) + // Check for log level here to avoid allocating a string to format the + // timestamp if it is not going to be logged. + if log.IsLogging(log.Debug) { + log.Debugf("Timer for %s: Timeline %s forked %d-way into child timelines %v at %s (unix nanos: %d)", s.timer.root.name, s.fullName, len(names), names, now.Format(fullTimestampFormat), now.UnixNano()) + } + return children +} + +// traverse visits all Timelines in the tree rooted at s. +// fn is called exactly once per Timeline as the `child` argument. +// The root Timeline has a `nil` parent. +func (s *Timeline) traverse(parent *Timeline, fn func(parent, child *Timeline)) { + if s == nil { + return + } + fn(parent, s) + for _, child := range s.children { + child.traverse(s, fn) + } +} + +// End marks the Timeline as having ended. It must be eventually called on all +// Timelines. +// After End is called, the ownership of the Timeline struct moves to the +// goroutine that owns the Timer that created it. +func (s *Timeline) End() { + if s == nil { + return + } + end := time.Now() + if s.timer == nil { + log.Debugf("Orphaned timeline %s ended without having been adopted. This is possibly unintended.", s.name) + return + } + if !s.end.IsZero() { + log.Debugf("Timer for %s: Timeline %s ended twice. This is possibly unintended.", s.timer.root.name, s.fullName) + return + } + s.end = end + s.timer.runningTimelines.Add(-1) + // Check for log level here to avoid allocating a string to format the + // timestamp if it is not going to be logged. + if log.IsLogging(log.Debug) { + log.Debugf("Timer for %s: Timeline %s ended at %s (unix nanos: %d)", s.timer.root.name, s.fullName, s.end.Format(fullTimestampFormat), s.end.UnixNano()) + } +} + +// Invalidate marks the Timeline as invalid. +func (s *Timeline) Invalidate(reason string) { + if s == nil { + return + } + if s.invalidReason != "" { + log.Warningf("Timer for %s: Timeline %s was already invalid (%v), but tried to invalidate again (%v)", s.timer.root.name, s.fullName, s.invalidReason, reason) + return + } + s.invalidReason = reason +} + +// A Lease is a reference to a Timeline that is valid until the Lease is +// canceled. After calling Lease on a Timeline, the caller should no longer +// use the Timeline directly, and should instead use the Lease exclusively. +// +// Leases should typically not cross function boundaries. +// +// Leases are useful in complex functions where ownership of a Timeline +// needs to be *conditionally transferred* to a different goroutine at some +// late point in the function. Consider this example: +// +// ``` +// +// func SomeLongFunction(timeline *timing.Timeline) { +// defer timeline.End() // Convenient to defer `End` to hit all the `return` branches. +// timeline.Reached("some_point") +// +// if err := something(timeline); err != nil { +// return +// } +// timeline.Reached("some_other_point") +// if err := somethingElse(); err != nil { +// timeline.Reached("some_error") +// return +// } +// timeline.Reached("another_point") +// go doSomethingElse(timeline) +// +// // Don't want to call `End` anymore here! +// } +// +// ``` +// +// With a Lease: +// +// ``` +// +// func SomeLongFunction(timeline *timing.Timeline) { +// lease := timeline.Lease() +// defer lease.End() +// lease.Reached("some_point") +// +// if err := something(); err != nil { +// lease.Reached("some_error") +// return +// } +// if err := somethingElse(); err != nil { +// lease.Reached("some_other_error") +// return +// } +// lease.Reached("another_point") +// go doSomethingElse(lease.Transfer()) +// +// // `End` is not called anymore here. +// } +// +// ``` +type Lease struct { + timeline *Timeline + valid bool +} + +// Reached records a new midpoint on the Timeline if the Lease is valid. +// See `Timeline.Reached` for more details. +func (l *Lease) Reached(name string) { + if l == nil || !l.valid { + return + } + l.timeline.Reached(name) +} + +// Fork forks the Timeline if the Lease is valid. +// See `Timeline.Fork` for more details. +func (l *Lease) Fork(name string) *Timeline { + if l == nil || !l.valid { + return nil + } + return l.timeline.Fork(name) +} + +// MultiFork forks the Timeline if the Lease is valid. +// See `Timeline.MultiFork` for more details. +func (l *Lease) MultiFork(names []string) []*Timeline { + if l == nil || !l.valid { + return nil + } + return l.timeline.MultiFork(names) +} + +// End ends the Timeline if the Lease is valid. +// The lease is invalidated after this call. +// See `Timeline.End` for more details. +func (l *Lease) End() { + if l == nil || !l.valid { + return + } + l.valid = false + l.timeline.End() +} + +// Invalidate invalidates the Timeline if the Lease is valid. +// See `Timeline.Invalidate` for more details. +func (l *Lease) Invalidate(reason string) { + if l == nil || !l.valid { + return + } + l.timeline.Invalidate(reason) +} + +// Transfer invalidates the current Lease and returns the underlying Timeline. +// Typically useful when transferring ownership of a Timeline to a different +// goroutine while giving up ownership in the current one. +// See `Lease` documentation for example usage. +func (l *Lease) Transfer() *Timeline { + if l == nil { + return nil + } + if !l.valid { + panic("timing.Lease.Transfer called on invalid Lease") + } + l.valid = false + return l.timeline +} + +// Lease returns a Lease for the Timeline. +// The Lease is valid until it is canceled by calling `End` or `Transfer`. +// See `Lease` for example usage. +func (s *Timeline) Lease() *Lease { + if s == nil { + return nil + } + return &Lease{ + timeline: s, + valid: true, + } +} + +// OrphanTimeline creates a new Timeline that is not owned by any Timer. +// This is useful for operations that are part of a broader sequence of +// operations represented by a Timer, but where timing measurements are +// desired before this broader parent Timer is known. +// The returned Timeline must be parented with `timer.Adopt` in order to +// be useful. +func OrphanTimeline(name string, startTime time.Time) *Timeline { + // Check for log level here to avoid allocating a string to format the + // timestamp if it is not going to be logged. + if log.IsLogging(log.Debug) { + log.Debugf("Orphaned timeline %s started at %s (unix nanos: %d)", name, startTime.Format(fullTimestampFormat), startTime.UnixNano()) + } + return &Timeline{ + name: name, + fullName: "_ORPHANED_", + start: startTime, + } +} + +// Timer is a root Timeline. It keeps track of one or more running Timelines, +// and can be pretty-printed to show timing information once all Timelines have +// ended. +// +// A Timer struct may move between goroutines, but only one goroutine may own +// it at a time. +// +// A Timer struct may be nil, in which case all methods are no-ops. This means +// all code that takes in a Timer parameter does not need to check for nilness. +type Timer struct { + // root is the root Timeline of the Timer. + root *Timeline + + // runningTimelines is the number of Timelines that have not yet ended. + // When dumping timing data, this is used to wait for all Timelines to end. + runningTimelines atomicbitops.Int64 +} + +// New creates a new Timer. +// The given name is used to identify the Timer in pretty-printed output. +// The given startTime is used as the start time of the Timer's root Timeline. +func New(name string, startTime time.Time) *Timer { + log.Infof("Timer for %s: Starting.", name) + timer := &Timer{} + timer.runningTimelines.Store(1) + root := &Timeline{ + name: name, + fullName: name, + timer: timer, + start: startTime, + } + timer.root = root + return timer +} + +// StartTime returns the start time of the Timer. +func (t *Timer) StartTime() time.Time { + if t == nil { + return time.Time{} + } + return t.root.start +} + +// Reached records a new midpoint on the root Timeline of the Timer. +func (t *Timer) Reached(name string) { + if t == nil { + return + } + t.root.Reached(name) +} + +// ReachedAt records a new midpoint on the root Timeline of the Timer with +// the given timestamp. +func (t *Timer) ReachedAt(name string, when time.Time) { + if t == nil { + return + } + t.root.ReachedAt(name, when) +} + +// Fork creates a new Timeline that is a child of the root Timeline of this +// Timer. +// The returned Timeline is initially owned by the caller, but may be passed +// to another goroutine if desired. +// This child Timeline may but does not need to end before the root timeline +// does. +// Forked timelines are useful to represent parallel operations like separate +// goroutines, and are actually required in such cases so that the goroutine +// can own its own Timeline, but non-concurrent code may also use Fork to +// represent its own linear operations as a tree if it so desires. +func (t *Timer) Fork(name string) *Timeline { + if t == nil { + return nil + } + return t.root.Fork(name) +} + +// MultiFork creates new Timelines that are children of the root Timeline of +// this Timer. +// See Timeline.MultiFork for more details. +func (t *Timer) MultiFork(names []string) []*Timeline { + if len(names) == 0 { + return nil + } + if t == nil { + return make([]*Timeline, len(names)) + } + return t.root.MultiFork(names) +} + +// Adopt adopts a Timeline into this Timer. +// May only be called with Timelines created by `OrphanTimeline`, +// and may only be called once per such Timeline. +// The calling goroutine must own both the Timeline and the Timer. +func (t *Timer) Adopt(child *Timeline) { + if t == nil || child == nil { + return + } + if child.timer != nil { + panic("timing.Timeline.Adopt called on Timeline that already has a parent") + } + if child.end.IsZero() { + t.runningTimelines.Add(1) + } + t.root.children = append(t.root.children, child) + t.root.midpoints = append(t.root.midpoints, MidPoint{ + name: "new orphan", + when: child.start, + }) + child.timer = t + child.fullName = fmt.Sprintf("%s/%s", t.root.fullName, child.name) + log.Debugf("Timer for %s: Timeline %s adopted.", t.root.name, child.fullName) +} + +// End waits for all Timelines owned by this Timer to end, then pretty-prints +// timing information. +// If not all Timelines have ended by the time End is called, End will spin in +// place until they do, and eventually print a warning log if it spins for too +// long (but will not give up). +// End is called implicitly by Log, so it is not necessary to call End +// explicitly unless there is a need to end the root timeline at a different +// time than when logging its data is desired. +func (t *Timer) End() { + if t == nil { + return + } + var peakRunningChildren int64 + if t.root.end.IsZero() { + t.root.End() + if peakRunningChildren = t.runningTimelines.Load(); peakRunningChildren > 0 { + log.Infof("Timer for %s: Root timeline ended, but %d child timelines are still running...", t.root.name, peakRunningChildren) + } else { + log.Infof("Timer for %s: Ended.", t.root.name) + } + } + const ( + stillWaitingLogThreshold = 10 * time.Second + stillWaitingLogInterval = 1 * time.Second + ) + startedWaiting := time.Now() + var rlLogger log.Logger + for runningTimelines := t.runningTimelines.Load(); runningTimelines != 0; runningTimelines = t.runningTimelines.Load() { + if runningTimelines < 0 { + panic("timing.Timeline.End called too many times in aggregate") + } + peakRunningChildren = max(peakRunningChildren, runningTimelines) + time.Sleep(1 * time.Millisecond) + if rlLogger == nil && time.Since(startedWaiting) > stillWaitingLogThreshold { + rlLogger = log.BasicRateLimitedLogger(stillWaitingLogInterval) + } + if rlLogger != nil { + if debugStrandedTimers { + // **Thread-unsafe** code to traverse the tree and print out all + // timelines that are still running. Only executed when + // the `debugStrandedTimers` const is true. + var timelineNames []string + t.root.traverse(nil, func(_, child *Timeline) { + if child.end.IsZero() { + timelineNames = append(timelineNames, child.fullName) + } + }) + rlLogger.Debugf("Timer for %s: Still waiting for %d child timelines to finish: %v", t.root.name, runningTimelines, timelineNames) + // Sleep longer to make sure we don't do the above traversal every single millisecond. + time.Sleep(100 * time.Millisecond) + } else { + rlLogger.Infof("Timer for %s: Still waiting for %d child timelines to finish... (If you are not expecting this, flip `debugStrandedTimers` to debug.)", t.root.name, runningTimelines) + } + } + } + if peakRunningChildren > 0 { + log.Infof("Timer for %s: All child timelines have ended.", t.root.name) + } +} + +// Log pretty-prints timing information for the root Timeline of the Timer. +// If `t.End` has not yet been called, it will be called implicitly. +// This also means that this function will wait for all child Timelines to end +// before pretty-printing, and will spin in place until this is the case. +// If debug logging is enabled, this function will also log a flat list of +// events that can be easily machine-parsed to the debug log. +func (t *Timer) Log() { + if t == nil { + return + } + t.End() + type pointType int + const ( + pointTypeStart pointType = iota + pointTypeMid + pointTypeEnd + ) + type point struct { + timeline *Timeline + pointType pointType + midpointName string + } + type event struct { + when time.Time + point point + } + totalDuration := t.root.end.Sub(t.root.start) + formatDuration := func(d time.Duration) string { + switch { + case totalDuration < time.Second: + us := d.Microseconds() + if us >= 1000 { + return fmt.Sprintf("%d %03dµs", us/1000, us%1000) + } + return fmt.Sprintf("%dµs", us) + case totalDuration < 3*time.Minute: + return fmt.Sprintf("%.3fs", float64(d.Milliseconds())/1000) + default: + return d.Truncate(time.Second).String() + } + } + var flatTimelines []*Timeline + t.root.traverse(nil, func(_, child *Timeline) { + flatTimelines = append(flatTimelines, child) + }) + var invalidReasons []string + for _, timeline := range flatTimelines { + if timeline.invalidReason != "" { + invalidReasons = append(invalidReasons, timeline.invalidReason) + } + } + if len(invalidReasons) > 0 { + log.Warningf("Timer for %s: Timeline was invalidated, so not displaying timing data: %v", t.root.name, invalidReasons) + return + } + var events []event + for _, timeline := range flatTimelines { + events = append(events, event{when: timeline.start, point: point{timeline: timeline, pointType: pointTypeStart}}) + for _, mid := range timeline.midpoints { + events = append(events, event{when: mid.when, point: point{timeline: timeline, pointType: pointTypeMid, midpointName: mid.name}}) + } + events = append(events, event{when: timeline.end, point: point{timeline: timeline, pointType: pointTypeEnd}}) + } + sort.Slice(events, func(i, j int) bool { + return events[i].when.Before(events[j].when) + }) + type dedupEvent struct { + when time.Time + points []point + } + var dedupEvents []dedupEvent + for _, e := range events { + if len(dedupEvents) == 0 || !dedupEvents[len(dedupEvents)-1].when.Equal(e.when) { + dedupEvents = append(dedupEvents, dedupEvent{when: e.when, points: []point{e.point}}) + } else { + dedupEvents[len(dedupEvents)-1].points = append(dedupEvents[len(dedupEvents)-1].points, e.point) + } + } + if len(dedupEvents) == 0 { + return + } + largestInterval := time.Duration(0) + for i := 1; i < len(dedupEvents); i++ { + if interval := dedupEvents[i].when.Sub(dedupEvents[i-1].when); interval > largestInterval { + largestInterval = interval + } + } + + rows := make([][]string, len(dedupEvents)) + colWidths := make([]int, len(flatTimelines)+3) + lastTimestampPerTimeline := make(map[*Timeline]time.Time) + for i, e := range dedupEvents { + colData := make([]string, 0, len(flatTimelines)+3) + colData = append(colData, e.when.Format(microsTimestampFormat)) + if i == 0 { + colData = append(colData, "") + colData = append(colData, "") + } else { + sincePrevious := e.when.Sub(dedupEvents[i-1].when) + colData = append(colData, fmt.Sprintf("+%s", formatDuration(sincePrevious))) + colData = append(colData, barChart(float64(sincePrevious)/float64(largestInterval), 12)) + } + for _, timeline := range flatTimelines { + lastTimestamp, ok := lastTimestampPerTimeline[timeline] + if !ok { + lastTimestamp = timeline.start + } + timelineChanged := false + for _, p := range e.points { + if p.timeline == timeline { + switch p.pointType { + case pointTypeStart: + colData = append(colData, fmt.Sprintf("╭─ %s", timeline.name)) + case pointTypeMid: + colData = append(colData, fmt.Sprintf("├─ %s: %s", p.midpointName, formatDuration(e.when.Sub(lastTimestamp)))) + case pointTypeEnd: + colData = append(colData, fmt.Sprintf("╰─ END %s, total %s", formatDuration(timeline.end.Sub(lastTimestamp)), formatDuration(timeline.end.Sub(timeline.start)))) + } + timelineChanged = true + break + } + } + if timelineChanged { + lastTimestampPerTimeline[timeline] = e.when + continue + } + if e.when.Before(timeline.start) || e.when.After(timeline.end) { + colData = append(colData, "") + } else { + colData = append(colData, fmt.Sprintf("│ ... %s ...", formatDuration(e.when.Sub(lastTimestamp)))) + } + } + rows[i] = colData + } + + // Best-effort minimization of the number of columns. + for collapsed := true; collapsed; { + collapsed = false + for col := 4; col < len(rows[0]); col++ { + // Find continuous ranges of non-blank cells. + type colRange struct{ start, end int } + var ranges []colRange + rangeStart := -1 + for i := 0; i < len(rows); i++ { + if isBlank := rows[i][col] == ""; rangeStart == -1 && !isBlank { + rangeStart = i + } else if rangeStart != -1 && isBlank { + ranges = append(ranges, colRange{start: rangeStart, end: i}) + rangeStart = -1 + } + } + if rangeStart != -1 { + ranges = append(ranges, colRange{start: rangeStart, end: len(rows) - 1}) + } + // For each range, check if it can be collapsed. + rangIndex := -1 + for k, rang := range ranges { + allClear := true + for i := rang.start; i <= rang.end; i++ { + if rows[i][col-1] != "" { + allClear = false + break + } + } + if allClear { + rangIndex = k + break + } + } + if rangIndex != -1 { + for i := ranges[rangIndex].start; i <= ranges[rangIndex].end; i++ { + rows[i][col-1], rows[i][col] = rows[i][col], rows[i][col-1] + } + collapsed = true + } + } + } + + // Trim empty columns from the right. + largestNonEmptyCol := len(rows[0]) - 1 + for col := len(rows[0]) - 1; col >= 4; col-- { + allEmpty := true + for i := range rows { + if rows[i][col] != "" { + allEmpty = false + break + } + } + if !allEmpty { + largestNonEmptyCol = col + break + } + } + for i := range rows { + rows[i] = rows[i][:largestNonEmptyCol+1] + } + + // Measure column widths. + for _, row := range rows { + for i, cell := range row { + if i < len(row)-1 { + restEmpty := true + if i < len(row)-1 { + for j := i + 1; j < len(row); j++ { + if row[j] != "" { + restEmpty = false + break + } + } + } + // Only count cells where the rest of the row is not empty. + if !restEmpty { + colWidths[i] = max(colWidths[i], utf8.RuneCountInString(cell)) + } + } + } + } + + // Build table. + var sb strings.Builder + sb.WriteString("---- ") + sb.WriteString(t.root.name) + sb.WriteString(" timing information ----\n") + for i, row := range rows { + for j, cell := range row { + // Check if the rest of the row is empty, and if so, break. + emptyRest := true + for k := j; k < len(row); k++ { + if row[k] != "" { + emptyRest = false + break + } + } + if emptyRest { + break + } + // Process cell. + switch j { + case 0: // Timestamp column. + sb.WriteRune('[') + for s := utf8.RuneCountInString(cell); s < colWidths[j]; s++ { + sb.WriteRune(' ') + } + sb.WriteString(cell) + sb.WriteRune(']') + case 1: // Delta column. + sb.WriteRune('\t') + for s := utf8.RuneCountInString(cell); s < colWidths[j]; s++ { + sb.WriteRune(' ') + } + sb.WriteString(cell) + case 2: // Delta bar chart column. + sb.WriteRune(' ') // Only one space of width from first column since it is reflecting the same quantity. + sb.WriteString(cell) + for s := utf8.RuneCountInString(cell); s < colWidths[j]; s++ { + sb.WriteRune(' ') + } + default: + // Other columns. + sb.WriteRune('\t') + sb.WriteString(cell) + if j < len(rows[i])-1 { + for s := utf8.RuneCountInString(cell); s < colWidths[j]; s++ { + sb.WriteRune(' ') + } + } + } + } + sb.WriteRune('\n') + } + sb.WriteString("---- End of ") + sb.WriteString(t.root.name) + sb.WriteString(" timing information ----\n") + + // Log it. + log.Infof("%s", sb.String()) + + // In debug mode, also log a flat list of events that can be easily machine-parsed. + if log.IsLogging(log.Debug) { + for _, e := range events { + switch e.point.pointType { + case pointTypeStart: + log.Debugf("Timer for %s: time %d %s start %s", t.root.name, e.when.UnixNano(), e.when.Format(fullTimestampFormat), e.point.timeline.fullName) + case pointTypeMid: + log.Debugf("Timer for %s: time %d %s mid %s = %s", t.root.name, e.when.UnixNano(), e.when.Format(fullTimestampFormat), e.point.timeline.fullName, e.point.midpointName) + case pointTypeEnd: + log.Debugf("Timer for %s: time %d %s end %s", t.root.name, e.when.UnixNano(), e.when.Format(fullTimestampFormat), e.point.timeline.fullName) + } + } + } +} + +// barChart returns a string of width characters that represents the given +// fraction of the given width. +func barChart(fraction float64, width int) string { + const chars = " ▏▎▍▌▋▊█▉" + numChars := utf8.RuneCountInString(chars) + pivotIndex := int(fraction * float64(width)) + pivotRuneIndex := max(0, min(numChars-1, int((fraction-(float64(pivotIndex)/float64(width)))*float64(width*numChars)))) + runes := make([]rune, width) + for i := 0; i < width; i++ { + if i < pivotIndex { + runes[i] = []rune(chars)[numChars-1] + } else if i == pivotIndex { + runes[i] = []rune(chars)[pivotRuneIndex] + } else { + runes[i] = []rune(chars)[0] + } + } + return string(runes) +} diff --git a/pkg/timing/timing_state_autogen.go b/pkg/timing/timing_state_autogen.go new file mode 100644 index 0000000..76a0475 --- /dev/null +++ b/pkg/timing/timing_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package timing diff --git a/pkg/unet/unet.go b/pkg/unet/unet.go new file mode 100644 index 0000000..700a290 --- /dev/null +++ b/pkg/unet/unet.go @@ -0,0 +1,547 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package unet provides a minimal net package based on Unix Domain Sockets. +// +// This does no pooling, and should only be used for a limited number of +// connections in a Go process. Don't use this package for arbitrary servers. +package unet + +import ( + "errors" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/eventfd" + "github.com/sagernet/gvisor/pkg/sync" + "golang.org/x/sys/unix" +) + +// backlog is used for the listen request. +const backlog = 16 + +// errClosing is returned by wait if the Socket is in the process of closing. +var errClosing = errors.New("Socket is closing") + +// errMessageTruncated indicates that data was lost because the provided buffer +// was too small. +var errMessageTruncated = errors.New("message truncated") + +// socketType returns the appropriate type. +func socketType(packet bool) int { + if packet { + return unix.SOCK_SEQPACKET + } + return unix.SOCK_STREAM +} + +// socket creates a new host socket. +func socket(packet bool) (int, error) { + // Make a new socket. + fd, err := unix.Socket(unix.AF_UNIX, socketType(packet), 0) + if err != nil { + return 0, err + } + + return fd, nil +} + +// Socket is a connected unix domain socket. +type Socket struct { + // gate protects use of fd. + gate sync.Gate + + // fd is the bound socket. + // + // fd only remains valid if read while within gate. + fd atomicbitops.Int32 + + // efd is an event FD that is signaled when the socket is closing. + // + // efd is immutable and remains valid until Close/Release. + efd eventfd.Eventfd + + // race is an atomic variable used to avoid triggering the race + // detector. See comment in SocketPair below. + race *atomicbitops.Int32 +} + +// NewSocket returns a socket from an existing FD. +// +// NewSocket takes ownership of fd. +func NewSocket(fd int) (*Socket, error) { + // fd must be non-blocking for non-blocking unix.Accept in + // ServerSocket.Accept. + if err := unix.SetNonblock(fd, true); err != nil { + return nil, err + } + + efd, err := eventfd.Create() + if err != nil { + return nil, err + } + + return &Socket{ + fd: atomicbitops.FromInt32(int32(fd)), + efd: efd, + }, nil +} + +// finish completes use of s.fd by evicting any waiters, closing the gate, and +// closing the event FD. +func (s *Socket) finish() error { + // Signal any blocked or future polls. + if err := s.efd.Notify(); err != nil { + return err + } + + // Close the gate, blocking until all FD users leave. + s.gate.Close() + + return s.efd.Close() +} + +// Close closes the socket. +func (s *Socket) Close() error { + // Set the FD in the socket to -1, to ensure that all future calls to + // FD/Release get nothing and Close calls return immediately. + fd := int(s.fd.Swap(-1)) + if fd < 0 { + // Already closed or closing. + return unix.EBADF + } + + // Shutdown the socket to cancel any pending accepts. + s.shutdown(fd) + + if err := s.finish(); err != nil { + return err + } + + return unix.Close(fd) +} + +// Release releases ownership of the socket FD. +// +// The returned FD is non-blocking. +// +// Any concurrent or future callers of Socket methods will receive EBADF. +func (s *Socket) Release() (int, error) { + // Set the FD in the socket to -1, to ensure that all future calls to + // FD/Release get nothing and Close calls return immediately. + fd := int(s.fd.Swap(-1)) + if fd < 0 { + // Already closed or closing. + return -1, unix.EBADF + } + + if err := s.finish(); err != nil { + return -1, err + } + + return fd, nil +} + +// FD returns the FD for this Socket. +// +// The FD is non-blocking and must not be made blocking. +// +// N.B. os.File.Fd makes the FD blocking. Use of Release instead of FD is +// strongly preferred. +// +// The returned FD cannot be used safely if there may be concurrent callers to +// Close or Release. +// +// Use Release to take ownership of the FD. +func (s *Socket) FD() int { + return int(s.fd.Load()) +} + +// enterFD enters the FD gate and returns the FD value. +// +// If enterFD returns ok, s.gate.Leave must be called when done with the FD. +// Callers may only block while within the gate using s.wait. +// +// The returned FD is guaranteed to remain valid until s.gate.Leave. +func (s *Socket) enterFD() (int, bool) { + if !s.gate.Enter() { + return -1, false + } + + fd := int(s.fd.Load()) + if fd < 0 { + s.gate.Leave() + return -1, false + } + + return fd, true +} + +// SocketPair creates a pair of connected sockets. +func SocketPair(packet bool) (*Socket, *Socket, error) { + // Make a new pair. + fds, err := unix.Socketpair(unix.AF_UNIX, socketType(packet)|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + + // race is an atomic variable used to avoid triggering the race + // detector. We have to fool TSAN into thinking there is a race + // variable between our two sockets. We only use SocketPair in tests + // anyway. + // + // NOTE(b/27107811): This is purely due to the fact that the raw + // syscall does not serve as a boundary for the sanitizer. + a, err := NewSocket(fds[0]) + if err != nil { + unix.Close(fds[0]) + unix.Close(fds[1]) + return nil, nil, err + } + var race atomicbitops.Int32 + a.race = &race + b, err := NewSocket(fds[1]) + if err != nil { + a.Close() + unix.Close(fds[1]) + return nil, nil, err + } + b.race = &race + return a, b, nil +} + +// Connect connects to a server. +func Connect(addr string, packet bool) (*Socket, error) { + fd, err := socket(packet) + if err != nil { + return nil, err + } + + // Connect the socket. + usa := &unix.SockaddrUnix{Name: addr} + if err := unix.Connect(fd, usa); err != nil { + unix.Close(fd) + return nil, err + } + + return NewSocket(fd) +} + +// ControlMessage wraps around a byte array and provides functions for parsing +// as a Unix Domain Socket control message. +type ControlMessage []byte + +// EnableFDs enables receiving FDs via control message. +// +// This guarantees only a MINIMUM number of FDs received. You may receive MORE +// than this due to the way FDs are packed. To be specific, the number of +// receivable buffers will be rounded up to the nearest even number. +// +// This must be called prior to ReadVec if you want to receive FDs. +func (c *ControlMessage) EnableFDs(count int) { + *c = make([]byte, unix.CmsgSpace(count*4)) +} + +// ExtractFDs returns the list of FDs in the control message. +// +// Either this or CloseFDs should be used after EnableFDs. +func (c *ControlMessage) ExtractFDs() ([]int, error) { + msgs, err := unix.ParseSocketControlMessage(*c) + if err != nil { + return nil, err + } + var fds []int + for _, msg := range msgs { + thisFds, err := unix.ParseUnixRights(&msg) + if err != nil { + // Different control message. + return nil, err + } + for _, fd := range thisFds { + if fd >= 0 { + fds = append(fds, fd) + } + } + } + return fds, nil +} + +// CloseFDs closes the list of FDs in the control message. +// +// Either this or ExtractFDs should be used after EnableFDs. +func (c *ControlMessage) CloseFDs() { + fds, _ := c.ExtractFDs() + for _, fd := range fds { + if fd >= 0 { + unix.Close(fd) + } + } +} + +// PackFDs packs the given list of FDs in the control message. +// +// This must be used prior to WriteVec. +func (c *ControlMessage) PackFDs(fds ...int) { + *c = ControlMessage(unix.UnixRights(fds...)) +} + +// UnpackFDs clears the control message. +func (c *ControlMessage) UnpackFDs() { + *c = nil +} + +// SocketWriter wraps an individual send operation. +// +// The normal entrypoint is WriteVec. +type SocketWriter struct { + socket *Socket + to []byte + blocking bool + race *atomicbitops.Int32 + + ControlMessage +} + +// Writer returns a writer for this socket. +func (s *Socket) Writer(blocking bool) SocketWriter { + return SocketWriter{socket: s, blocking: blocking, race: s.race} +} + +// Write implements io.Writer.Write. +func (s *Socket) Write(p []byte) (int, error) { + r := s.Writer(true) + return r.WriteVec([][]byte{p}) +} + +// GetSockOpt gets the given socket option. +func (s *Socket) GetSockOpt(level int, name int, b []byte) (uint32, error) { + fd, ok := s.enterFD() + if !ok { + return 0, unix.EBADF + } + defer s.gate.Leave() + + return getsockopt(fd, level, name, b) +} + +// SetSockOpt sets the given socket option. +func (s *Socket) SetSockOpt(level, name int, b []byte) error { + fd, ok := s.enterFD() + if !ok { + return unix.EBADF + } + defer s.gate.Leave() + + return setsockopt(fd, level, name, b) +} + +// GetSockName returns the socket name. +func (s *Socket) GetSockName() ([]byte, error) { + fd, ok := s.enterFD() + if !ok { + return nil, unix.EBADF + } + defer s.gate.Leave() + + var buf []byte + l := unix.SizeofSockaddrAny + + for { + // If the buffer is not large enough, allocate a new one with the hint. + buf = make([]byte, l) + l, err := getsockname(fd, buf) + if err != nil { + return nil, err + } + + if l <= uint32(len(buf)) { + return buf[:l], nil + } + } +} + +// GetPeerName returns the peer name. +func (s *Socket) GetPeerName() ([]byte, error) { + fd, ok := s.enterFD() + if !ok { + return nil, unix.EBADF + } + defer s.gate.Leave() + + var buf []byte + l := unix.SizeofSockaddrAny + + for { + // See above. + buf = make([]byte, l) + l, err := getpeername(fd, buf) + if err != nil { + return nil, err + } + + if l <= uint32(len(buf)) { + return buf[:l], nil + } + } +} + +// SocketReader wraps an individual receive operation. +// +// This may be used for doing vectorized reads and/or sending additional +// control messages (e.g. FDs). The normal entrypoint is ReadVec. +// +// One of ExtractFDs or DisposeFDs must be called if EnableFDs is used. +type SocketReader struct { + socket *Socket + source []byte + blocking bool + race *atomicbitops.Int32 + + ControlMessage +} + +// Reader returns a reader for this socket. +func (s *Socket) Reader(blocking bool) SocketReader { + return SocketReader{socket: s, blocking: blocking, race: s.race} +} + +// Read implements io.Reader.Read. +func (s *Socket) Read(p []byte) (int, error) { + r := s.Reader(true) + return r.ReadVec([][]byte{p}) +} + +func (s *Socket) shutdown(fd int) error { + // Shutdown the socket to cancel any pending accepts. + return unix.Shutdown(fd, unix.SHUT_RDWR) +} + +// Shutdown closes the socket for read and write. +func (s *Socket) Shutdown() error { + fd, ok := s.enterFD() + if !ok { + return unix.EBADF + } + defer s.gate.Leave() + + return s.shutdown(fd) +} + +// ServerSocket is a bound unix domain socket. +type ServerSocket struct { + socket *Socket +} + +// NewServerSocket returns a socket from an existing FD. +func NewServerSocket(fd int) (*ServerSocket, error) { + s, err := NewSocket(fd) + if err != nil { + return nil, err + } + return &ServerSocket{socket: s}, nil +} + +// Bind creates and binds a new socket. +func Bind(addr string, packet bool) (*ServerSocket, error) { + fd, err := socket(packet) + if err != nil { + return nil, err + } + + // Do the bind. + usa := &unix.SockaddrUnix{Name: addr} + if err := unix.Bind(fd, usa); err != nil { + unix.Close(fd) + return nil, err + } + + return NewServerSocket(fd) +} + +// BindAndListen creates, binds and listens on a new socket. +func BindAndListen(addr string, packet bool) (*ServerSocket, error) { + s, err := Bind(addr, packet) + if err != nil { + return nil, err + } + + // Start listening. + if err := s.Listen(); err != nil { + s.Close() + return nil, err + } + + return s, nil +} + +// Listen starts listening on the socket. +func (s *ServerSocket) Listen() error { + fd, ok := s.socket.enterFD() + if !ok { + return unix.EBADF + } + defer s.socket.gate.Leave() + + return unix.Listen(fd, backlog) +} + +// Accept accepts a new connection. +// +// This is always blocking. +// +// Preconditions: +// - ServerSocket is listening (Listen called). +func (s *ServerSocket) Accept() (*Socket, error) { + fd, ok := s.socket.enterFD() + if !ok { + return nil, unix.EBADF + } + defer s.socket.gate.Leave() + + for { + nfd, _, err := unix.Accept(fd) + switch err { + case nil: + return NewSocket(nfd) + case unix.EAGAIN: + err = s.socket.wait(false) + if err == errClosing { + err = unix.EBADF + } + } + if err != nil { + return nil, err + } + } +} + +// Close closes the server socket. +// +// This must only be called once. +func (s *ServerSocket) Close() error { + return s.socket.Close() +} + +// FD returns the socket's file descriptor. +// +// See Socket.FD. +func (s *ServerSocket) FD() int { + return s.socket.FD() +} + +// Release releases ownership of the socket's file descriptor. +// +// See Socket.Release. +func (s *ServerSocket) Release() (int, error) { + return s.socket.Release() +} diff --git a/pkg/unet/unet_state_autogen.go b/pkg/unet/unet_state_autogen.go new file mode 100644 index 0000000..9bbf31d --- /dev/null +++ b/pkg/unet/unet_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package unet diff --git a/pkg/unet/unet_unsafe.go b/pkg/unet/unet_unsafe.go new file mode 100644 index 0000000..1ac3824 --- /dev/null +++ b/pkg/unet/unet_unsafe.go @@ -0,0 +1,286 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unet + +import ( + "io" + "unsafe" + + "golang.org/x/sys/unix" +) + +// wait blocks until the socket FD is ready for reading or writing, depending +// on the value of write. +// +// Returns errClosing if the Socket is in the process of closing. +func (s *Socket) wait(write bool) error { + for { + // Checking the FD on each loop is not strictly necessary, it + // just avoids an extra poll call. + fd := s.fd.Load() + if fd < 0 { + return errClosing + } + + events := []unix.PollFd{ + { + // The actual socket FD. + Fd: fd, + Events: unix.POLLIN, + }, + { + // The eventfd, signaled when we are closing. + Fd: int32(s.efd.FD()), + Events: unix.POLLIN, + }, + } + if write { + events[0].Events = unix.POLLOUT + } + + _, _, e := unix.Syscall6(unix.SYS_PPOLL, uintptr(unsafe.Pointer(&events[0])), 2, 0, 0, 0, 0) + if e == unix.EINTR { + continue + } + if e != 0 { + return e + } + + if events[1].Revents&unix.POLLIN == unix.POLLIN { + // eventfd signaled, we're closing. + return errClosing + } + + return nil + } +} + +// buildIovec builds an iovec slice from the given []byte slice. +// +// iovecs is used as an initial slice, to avoid excessive allocations. +func buildIovec(bufs [][]byte, iovecs []unix.Iovec) ([]unix.Iovec, int) { + var length int + for i := range bufs { + if l := len(bufs[i]); l > 0 { + iovecs = append(iovecs, unix.Iovec{ + Base: &bufs[i][0], + Len: uint64(l), + }) + length += l + } + } + return iovecs, length +} + +// ReadVec reads into the pre-allocated bufs. Returns bytes read. +// +// The pre-allocatted space used by ReadVec is based upon slice lengths. +// +// This function is not guaranteed to read all available data, it +// returns as soon as a single recvmsg call succeeds. +func (r *SocketReader) ReadVec(bufs [][]byte) (int, error) { + iovecs, length := buildIovec(bufs, make([]unix.Iovec, 0, 2)) + + var msg unix.Msghdr + if len(r.source) != 0 { + msg.Name = &r.source[0] + msg.Namelen = uint32(len(r.source)) + } + + if len(r.ControlMessage) != 0 { + msg.Control = &r.ControlMessage[0] + msg.Controllen = uint64(len(r.ControlMessage)) + } + + if len(iovecs) != 0 { + msg.Iov = &iovecs[0] + msg.Iovlen = uint64(len(iovecs)) + } + + // n is the bytes received. + var n uintptr + + fd, ok := r.socket.enterFD() + if !ok { + return 0, unix.EBADF + } + // Leave on returns below. + for { + var e unix.Errno + + // Try a non-blocking recv first, so we don't give up the go runtime M. + n, _, e = unix.RawSyscall(unix.SYS_RECVMSG, uintptr(fd), uintptr(unsafe.Pointer(&msg)), unix.MSG_DONTWAIT|unix.MSG_TRUNC) + if e == 0 { + break + } + if e == unix.EINTR { + continue + } + if !r.blocking { + r.socket.gate.Leave() + return 0, e + } + if e != unix.EAGAIN && e != unix.EWOULDBLOCK { + r.socket.gate.Leave() + return 0, e + } + + // Wait for the socket to become readable. + err := r.socket.wait(false) + if err == errClosing { + err = unix.EBADF + } + if err != nil { + r.socket.gate.Leave() + return 0, err + } + } + + r.socket.gate.Leave() + + if msg.Controllen < uint64(len(r.ControlMessage)) { + r.ControlMessage = r.ControlMessage[:msg.Controllen] + } + + if msg.Namelen < uint32(len(r.source)) { + r.source = r.source[:msg.Namelen] + } + + // All unet sockets are SOCK_STREAM or SOCK_SEQPACKET, both of which + // indicate that the other end is closed by returning a 0 length read + // with no error. + if n == 0 { + return 0, io.EOF + } + + if r.race != nil { + // See comments on Socket.race. + r.race.Add(1) + } + + if int(n) > length { + return length, errMessageTruncated + } + + return int(n), nil +} + +// WriteVec writes the bufs to the socket. Returns bytes written. +// +// This function is not guaranteed to send all data, it returns +// as soon as a single sendmsg call succeeds. +func (w *SocketWriter) WriteVec(bufs [][]byte) (int, error) { + iovecs, _ := buildIovec(bufs, make([]unix.Iovec, 0, 2)) + + if w.race != nil { + // See comments on Socket.race. + w.race.Add(1) + } + + var msg unix.Msghdr + if len(w.to) != 0 { + msg.Name = &w.to[0] + msg.Namelen = uint32(len(w.to)) + } + + if len(w.ControlMessage) != 0 { + msg.Control = &w.ControlMessage[0] + msg.Controllen = uint64(len(w.ControlMessage)) + } + + if len(iovecs) > 0 { + msg.Iov = &iovecs[0] + msg.Iovlen = uint64(len(iovecs)) + } + + fd, ok := w.socket.enterFD() + if !ok { + return 0, unix.EBADF + } + // Leave on returns below. + for { + // Try a non-blocking send first, so we don't give up the go runtime M. + n, _, e := unix.RawSyscall(unix.SYS_SENDMSG, uintptr(fd), uintptr(unsafe.Pointer(&msg)), unix.MSG_DONTWAIT|unix.MSG_NOSIGNAL) + if e == 0 { + w.socket.gate.Leave() + return int(n), nil + } + if e == unix.EINTR { + continue + } + if !w.blocking { + w.socket.gate.Leave() + return 0, e + } + if e != unix.EAGAIN && e != unix.EWOULDBLOCK { + w.socket.gate.Leave() + return 0, e + } + + // Wait for the socket to become writeable. + err := w.socket.wait(true) + if err == errClosing { + err = unix.EBADF + } + if err != nil { + w.socket.gate.Leave() + return 0, err + } + } + // Unreachable, no s.gate.Leave needed. +} + +// getsockopt issues a getsockopt unix. +func getsockopt(fd int, level int, optname int, buf []byte) (uint32, error) { + l := uint32(len(buf)) + _, _, e := unix.RawSyscall6(unix.SYS_GETSOCKOPT, uintptr(fd), uintptr(level), uintptr(optname), uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&l)), 0) + if e != 0 { + return 0, e + } + + return l, nil +} + +// setsockopt issues a setsockopt unix. +func setsockopt(fd int, level int, optname int, buf []byte) error { + _, _, e := unix.RawSyscall6(unix.SYS_SETSOCKOPT, uintptr(fd), uintptr(level), uintptr(optname), uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf)), 0) + if e != 0 { + return e + } + + return nil +} + +// getsockname issues a getsockname unix. +func getsockname(fd int, buf []byte) (uint32, error) { + l := uint32(len(buf)) + _, _, e := unix.RawSyscall(unix.SYS_GETSOCKNAME, uintptr(fd), uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&l))) + if e != 0 { + return 0, e + } + + return l, nil +} + +// getpeername issues a getpeername unix. +func getpeername(fd int, buf []byte) (uint32, error) { + l := uint32(len(buf)) + _, _, e := unix.RawSyscall(unix.SYS_GETPEERNAME, uintptr(fd), uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&l))) + if e != 0 { + return 0, e + } + + return l, nil +} diff --git a/pkg/unet/unet_unsafe_state_autogen.go b/pkg/unet/unet_unsafe_state_autogen.go new file mode 100644 index 0000000..9bbf31d --- /dev/null +++ b/pkg/unet/unet_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package unet diff --git a/pkg/waiter/waiter.go b/pkg/waiter/waiter.go new file mode 100644 index 0000000..71b29d4 --- /dev/null +++ b/pkg/waiter/waiter.go @@ -0,0 +1,280 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package waiter provides the implementation of a wait queue, where waiters can +// be enqueued to be notified when an event of interest happens. +// +// Becoming readable and/or writable are examples of events. Waiters are +// expected to use a pattern similar to this to make a blocking function out of +// a non-blocking one: +// +// func (o *object) blockingRead(...) error { +// err := o.nonBlockingRead(...) +// if err != ErrAgain { +// // Completed with no need to wait! +// return err +// } +// +// e := createOrGetWaiterEntry(...) +// o.EventRegister(&e, waiter.EventIn) +// defer o.EventUnregister(&e) +// +// // We need to try to read again after registration because the +// // object may have become readable between the last attempt to +// // read and read registration. +// err = o.nonBlockingRead(...) +// for err == ErrAgain { +// wait() +// err = o.nonBlockingRead(...) +// } +// +// return err +// } +// +// Another goroutine needs to notify waiters when events happen. For example: +// +// func (o *object) Write(...) ... { +// // Do write work. +// [...] +// +// if oldDataAvailableSize == 0 && dataAvailableSize > 0 { +// // If no data was available and now some data is +// // available, the object became readable, so notify +// // potential waiters about this. +// o.Notify(waiter.EventIn) +// } +// } +package waiter + +import ( + "github.com/sagernet/gvisor/pkg/sync" +) + +// EventMask represents io events as used in the poll() syscall. +type EventMask uint64 + +// Events that waiters can wait on. The meaning is the same as those in the +// poll() syscall. +const ( + EventIn EventMask = 0x01 // POLLIN + EventPri EventMask = 0x02 // POLLPRI + EventOut EventMask = 0x04 // POLLOUT + EventErr EventMask = 0x08 // POLLERR + EventHUp EventMask = 0x10 // POLLHUP + EventRdNorm EventMask = 0x0040 // POLLRDNORM + EventWrNorm EventMask = 0x0100 // POLLWRNORM + EventInternal EventMask = 0x1000 + EventRdHUp EventMask = 0x2000 // POLLRDHUP + + AllEvents EventMask = 0x1f | EventRdNorm | EventWrNorm | EventRdHUp + ReadableEvents EventMask = EventIn | EventRdNorm + WritableEvents EventMask = EventOut | EventWrNorm +) + +// EventMaskFromLinux returns an EventMask representing the supported events +// from the Linux events e, which is in the format used by poll(2). +func EventMaskFromLinux(e uint32) EventMask { + // Our flag definitions are currently identical to Linux. + return EventMask(e) & AllEvents +} + +// ToLinux returns e in the format used by Linux poll(2). +func (e EventMask) ToLinux() uint32 { + // Our flag definitions are currently identical to Linux. + return uint32(e) +} + +// Waitable contains the methods that need to be implemented by waitable +// objects. +type Waitable interface { + // Readiness returns what the object is currently ready for. If it's + // not ready for a desired purpose, the caller may use EventRegister and + // EventUnregister to get notifications once the object becomes ready. + // + // Implementations should allow for events like EventHUp and EventErr + // to be returned regardless of whether they are in the input EventMask. + Readiness(mask EventMask) EventMask + + // EventRegister registers the given waiter entry to receive + // notifications when an event occurs that makes the object ready for + // at least one of the events in mask. + EventRegister(e *Entry) error + + // EventUnregister unregisters a waiter entry previously registered with + // EventRegister(). + EventUnregister(e *Entry) +} + +// EventListener provides a notify callback. +type EventListener interface { + // NotifyEvent is the function to be called when the waiter entry is + // notified. It is responsible for doing whatever is needed to wake up + // the waiter. + // + // The callback is supposed to perform minimal work, and cannot call + // any method on the queue itself because it will be locked while the + // callback is running. + // + // The mask indicates the events that occurred and that the entry is + // interested in. + NotifyEvent(mask EventMask) +} + +// Entry represents a waiter that can be add to the a wait queue. It can +// only be in one queue at a time, and is added "intrusively" to the queue with +// no extra memory allocations. +// +// +stateify savable +type Entry struct { + waiterEntry + + // eventListener receives the notification. + eventListener EventListener + + // mask should be immutable once queued. + mask EventMask +} + +// Init initializes the Entry. +// +// This must only be called when unregistered. +func (e *Entry) Init(eventListener EventListener, mask EventMask) { + e.eventListener = eventListener + e.mask = mask +} + +// Mask returns the entry mask. +func (e *Entry) Mask() EventMask { + return e.mask +} + +// NotifyEvent notifies the event listener. +// +// Mask should be the full set of active events. +func (e *Entry) NotifyEvent(mask EventMask) { + if m := mask & e.mask; m != 0 { + e.eventListener.NotifyEvent(m) + } +} + +// ChannelNotifier is a simple channel-based notification. +type ChannelNotifier chan struct{} + +// NotifyEvent implements waiter.EventListener.NotifyEvent. +func (c ChannelNotifier) NotifyEvent(EventMask) { + select { + case c <- struct{}{}: + default: + } +} + +// NewChannelEntry initializes a new Entry that does a non-blocking write to a +// struct{} channel when the callback is called. It returns the new Entry +// instance and the channel being used. +func NewChannelEntry(mask EventMask) (e Entry, ch chan struct{}) { + ch = make(chan struct{}, 1) + e.Init(ChannelNotifier(ch), mask) + return e, ch +} + +type functionNotifier func(EventMask) + +// NotifyEvent implements waiter.EventListener.NotifyEvent. +func (f functionNotifier) NotifyEvent(mask EventMask) { + f(mask) +} + +// NewFunctionEntry initializes a new Entry that calls the given function. +func NewFunctionEntry(mask EventMask, fn func(EventMask)) (e Entry) { + e.Init(functionNotifier(fn), mask) + return e +} + +// Queue represents the wait queue where waiters can be added and +// notifiers can notify them when events happen. +// +// The zero value for waiter.Queue is an empty queue ready for use. +// +// +stateify savable +type Queue struct { + list waiterList + mu sync.RWMutex `state:"nosave"` +} + +// EventRegister adds a waiter to the wait queue. +func (q *Queue) EventRegister(e *Entry) { + q.mu.Lock() + q.list.PushBack(e) + q.mu.Unlock() +} + +// EventUnregister removes the given waiter entry from the wait queue. +func (q *Queue) EventUnregister(e *Entry) { + q.mu.Lock() + q.list.Remove(e) + q.mu.Unlock() +} + +// Notify notifies all waiters in the queue whose masks have at least one bit +// in common with the notification mask. +func (q *Queue) Notify(mask EventMask) { + q.mu.RLock() + for e := q.list.Front(); e != nil; e = e.Next() { + m := mask & e.mask + if m == 0 { + continue + } + e.eventListener.NotifyEvent(m) // Skip intermediate call. + } + q.mu.RUnlock() +} + +// Events returns the set of events being waited on. It is the union of the +// masks of all registered entries. +func (q *Queue) Events() EventMask { + q.mu.RLock() + defer q.mu.RUnlock() + ret := EventMask(0) + for e := q.list.Front(); e != nil; e = e.Next() { + ret |= e.mask + } + return ret +} + +// IsEmpty returns if the wait queue is empty or not. +func (q *Queue) IsEmpty() bool { + q.mu.RLock() + defer q.mu.RUnlock() + return q.list.Front() == nil +} + +// NeverReady implements the Waitable interface but is never ready. Otherwise, +// this is exactly the same as AlwaysReady. +type NeverReady struct{} + +// Readiness always returns 0 because this object is never ready. +func (*NeverReady) Readiness(EventMask) EventMask { + return 0 +} + +// EventRegister doesn't do anything because this object doesn't need to issue +// notifications because its readiness never changes. +func (*NeverReady) EventRegister(*Entry) error { + return nil +} + +// EventUnregister doesn't do anything because this object doesn't need to issue +// notifications because its readiness never changes. +func (*NeverReady) EventUnregister(*Entry) { +} diff --git a/pkg/waiter/waiter_list.go b/pkg/waiter/waiter_list.go new file mode 100644 index 0000000..f2e2193 --- /dev/null +++ b/pkg/waiter/waiter_list.go @@ -0,0 +1,239 @@ +package waiter + +// ElementMapper provides an identity mapping by default. +// +// This can be replaced to provide a struct that maps elements to linker +// objects, if they are not the same. An ElementMapper is not typically +// required if: Linker is left as is, Element is left as is, or Linker and +// Element are the same type. +type waiterElementMapper struct{} + +// linkerFor maps an Element to a Linker. +// +// This default implementation should be inlined. +// +//go:nosplit +func (waiterElementMapper) linkerFor(elem *Entry) *Entry { return elem } + +// List is an intrusive list. Entries can be added to or removed from the list +// in O(1) time and with no additional memory allocations. +// +// The zero value for List is an empty list ready to use. +// +// To iterate over a list (where l is a List): +// +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e. +// } +// +// +stateify savable +type waiterList struct { + head *Entry + tail *Entry +} + +// Reset resets list l to the empty state. +func (l *waiterList) Reset() { + l.head = nil + l.tail = nil +} + +// Empty returns true iff the list is empty. +// +//go:nosplit +func (l *waiterList) Empty() bool { + return l.head == nil +} + +// Front returns the first element of list l or nil. +// +//go:nosplit +func (l *waiterList) Front() *Entry { + return l.head +} + +// Back returns the last element of list l or nil. +// +//go:nosplit +func (l *waiterList) Back() *Entry { + return l.tail +} + +// Len returns the number of elements in the list. +// +// NOTE: This is an O(n) operation. +// +//go:nosplit +func (l *waiterList) Len() (count int) { + for e := l.Front(); e != nil; e = (waiterElementMapper{}.linkerFor(e)).Next() { + count++ + } + return count +} + +// PushFront inserts the element e at the front of list l. +// +//go:nosplit +func (l *waiterList) PushFront(e *Entry) { + linker := waiterElementMapper{}.linkerFor(e) + linker.SetNext(l.head) + linker.SetPrev(nil) + if l.head != nil { + waiterElementMapper{}.linkerFor(l.head).SetPrev(e) + } else { + l.tail = e + } + + l.head = e +} + +// PushFrontList inserts list m at the start of list l, emptying m. +// +//go:nosplit +func (l *waiterList) PushFrontList(m *waiterList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + waiterElementMapper{}.linkerFor(l.head).SetPrev(m.tail) + waiterElementMapper{}.linkerFor(m.tail).SetNext(l.head) + + l.head = m.head + } + m.head = nil + m.tail = nil +} + +// PushBack inserts the element e at the back of list l. +// +//go:nosplit +func (l *waiterList) PushBack(e *Entry) { + linker := waiterElementMapper{}.linkerFor(e) + linker.SetNext(nil) + linker.SetPrev(l.tail) + if l.tail != nil { + waiterElementMapper{}.linkerFor(l.tail).SetNext(e) + } else { + l.head = e + } + + l.tail = e +} + +// PushBackList inserts list m at the end of list l, emptying m. +// +//go:nosplit +func (l *waiterList) PushBackList(m *waiterList) { + if l.head == nil { + l.head = m.head + l.tail = m.tail + } else if m.head != nil { + waiterElementMapper{}.linkerFor(l.tail).SetNext(m.head) + waiterElementMapper{}.linkerFor(m.head).SetPrev(l.tail) + + l.tail = m.tail + } + m.head = nil + m.tail = nil +} + +// InsertAfter inserts e after b. +// +//go:nosplit +func (l *waiterList) InsertAfter(b, e *Entry) { + bLinker := waiterElementMapper{}.linkerFor(b) + eLinker := waiterElementMapper{}.linkerFor(e) + + a := bLinker.Next() + + eLinker.SetNext(a) + eLinker.SetPrev(b) + bLinker.SetNext(e) + + if a != nil { + waiterElementMapper{}.linkerFor(a).SetPrev(e) + } else { + l.tail = e + } +} + +// InsertBefore inserts e before a. +// +//go:nosplit +func (l *waiterList) InsertBefore(a, e *Entry) { + aLinker := waiterElementMapper{}.linkerFor(a) + eLinker := waiterElementMapper{}.linkerFor(e) + + b := aLinker.Prev() + eLinker.SetNext(a) + eLinker.SetPrev(b) + aLinker.SetPrev(e) + + if b != nil { + waiterElementMapper{}.linkerFor(b).SetNext(e) + } else { + l.head = e + } +} + +// Remove removes e from l. +// +//go:nosplit +func (l *waiterList) Remove(e *Entry) { + linker := waiterElementMapper{}.linkerFor(e) + prev := linker.Prev() + next := linker.Next() + + if prev != nil { + waiterElementMapper{}.linkerFor(prev).SetNext(next) + } else if l.head == e { + l.head = next + } + + if next != nil { + waiterElementMapper{}.linkerFor(next).SetPrev(prev) + } else if l.tail == e { + l.tail = prev + } + + linker.SetNext(nil) + linker.SetPrev(nil) +} + +// Entry is a default implementation of Linker. Users can add anonymous fields +// of this type to their structs to make them automatically implement the +// methods needed by List. +// +// +stateify savable +type waiterEntry struct { + next *Entry + prev *Entry +} + +// Next returns the entry that follows e in the list. +// +//go:nosplit +func (e *waiterEntry) Next() *Entry { + return e.next +} + +// Prev returns the entry that precedes e in the list. +// +//go:nosplit +func (e *waiterEntry) Prev() *Entry { + return e.prev +} + +// SetNext assigns 'entry' as the entry that follows e in the list. +// +//go:nosplit +func (e *waiterEntry) SetNext(elem *Entry) { + e.next = elem +} + +// SetPrev assigns 'entry' as the entry that precedes e in the list. +// +//go:nosplit +func (e *waiterEntry) SetPrev(elem *Entry) { + e.prev = elem +} diff --git a/pkg/waiter/waiter_state_autogen.go b/pkg/waiter/waiter_state_autogen.go new file mode 100644 index 0000000..073c9e0 --- /dev/null +++ b/pkg/waiter/waiter_state_autogen.go @@ -0,0 +1,128 @@ +// automatically generated by stateify. + +package waiter + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (e *Entry) StateTypeName() string { + return "pkg/waiter.Entry" +} + +func (e *Entry) StateFields() []string { + return []string{ + "waiterEntry", + "eventListener", + "mask", + } +} + +func (e *Entry) beforeSave() {} + +// +checklocksignore +func (e *Entry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.waiterEntry) + stateSinkObject.Save(1, &e.eventListener) + stateSinkObject.Save(2, &e.mask) +} + +func (e *Entry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *Entry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.waiterEntry) + stateSourceObject.Load(1, &e.eventListener) + stateSourceObject.Load(2, &e.mask) +} + +func (q *Queue) StateTypeName() string { + return "pkg/waiter.Queue" +} + +func (q *Queue) StateFields() []string { + return []string{ + "list", + } +} + +func (q *Queue) beforeSave() {} + +// +checklocksignore +func (q *Queue) StateSave(stateSinkObject state.Sink) { + q.beforeSave() + stateSinkObject.Save(0, &q.list) +} + +func (q *Queue) afterLoad(context.Context) {} + +// +checklocksignore +func (q *Queue) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &q.list) +} + +func (l *waiterList) StateTypeName() string { + return "pkg/waiter.waiterList" +} + +func (l *waiterList) StateFields() []string { + return []string{ + "head", + "tail", + } +} + +func (l *waiterList) beforeSave() {} + +// +checklocksignore +func (l *waiterList) StateSave(stateSinkObject state.Sink) { + l.beforeSave() + stateSinkObject.Save(0, &l.head) + stateSinkObject.Save(1, &l.tail) +} + +func (l *waiterList) afterLoad(context.Context) {} + +// +checklocksignore +func (l *waiterList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &l.head) + stateSourceObject.Load(1, &l.tail) +} + +func (e *waiterEntry) StateTypeName() string { + return "pkg/waiter.waiterEntry" +} + +func (e *waiterEntry) StateFields() []string { + return []string{ + "next", + "prev", + } +} + +func (e *waiterEntry) beforeSave() {} + +// +checklocksignore +func (e *waiterEntry) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.next) + stateSinkObject.Save(1, &e.prev) +} + +func (e *waiterEntry) afterLoad(context.Context) {} + +// +checklocksignore +func (e *waiterEntry) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.next) + stateSourceObject.Load(1, &e.prev) +} + +func init() { + state.Register((*Entry)(nil)) + state.Register((*Queue)(nil)) + state.Register((*waiterList)(nil)) + state.Register((*waiterEntry)(nil)) +} diff --git a/pkg/xdp/completionqueue.go b/pkg/xdp/completionqueue.go new file mode 100644 index 0000000..f86f3e7 --- /dev/null +++ b/pkg/xdp/completionqueue.go @@ -0,0 +1,119 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" +) + +// The CompletionQueue is how the kernel tells a process which buffers have +// been transmitted and can be reused. +// +// CompletionQueue is not thread-safe and requires external synchronization +type CompletionQueue struct { + // mem is the mmap'd area shared with the kernel. Many other fields of + // this struct point into mem. + mem []byte + + // ring is the actual ring buffer. It is a list of frame addresses + // ready to be reused. + // + // len(ring) must be a power of 2. + ring []uint64 + + // mask is used whenever indexing into ring. It is always len(ring)-1. + // It prevents index out of bounds errors while allowing the producer + // and consumer pointers to repeatedly "overflow" and loop back around + // the ring. + mask uint32 + + // producer points to the shared atomic value that indicates the last + // produced descriptor. Only the kernel updates this value. + producer *atomicbitops.Uint32 + + // consumer points to the shared atomic value that indicates the last + // consumed descriptor. Only we update this value. + consumer *atomicbitops.Uint32 + + // flags points to the shared atomic value that holds flags for the + // queue. + flags *atomicbitops.Uint32 + + // Cached values are used to avoid relatively expensive atomic + // operations. They are used, incremented, and decremented multiple + // times with non-atomic operations, and then "batch-updated" by + // reading or writing atomically to synchronize with the kernel. + + // cachedProducer is updated when we atomically read *producer. + cachedProducer uint32 + // cachedConsumer is used to atomically write *consumer. + cachedConsumer uint32 +} + +// Peek returns the number of buffers available to reuse as well as the index +// at which they start. Peek will only return a buffer once, so callers must +// process any received buffers. +func (cq *CompletionQueue) Peek() (nAvailable, index uint32) { + // Get the number of available buffers and update cachedConsumer to + // reflect that we're going to consume them. + entries := cq.free() + index = cq.cachedConsumer + cq.cachedConsumer += entries + return entries, index +} + +func (cq *CompletionQueue) free() uint32 { + // Return any buffers we know about without incurring an atomic + // operation if possible. + entries := cq.cachedProducer - cq.cachedConsumer + // If we're not aware of any completed packets, refresh the producer + // pointer to see whether the kernel enqueued anything. + if entries == 0 { + cq.cachedProducer = cq.producer.Load() + entries = cq.cachedProducer - cq.cachedConsumer + } + return entries +} + +// Release notifies the kernel that we have consumed nDone packets. +func (cq *CompletionQueue) Release(nDone uint32) { + // We don't have to use an atomic add because only we update this; the + // kernel just reads it. + cq.consumer.Store(cq.consumer.RacyLoad() + nDone) +} + +// Get gets the descriptor at index. +func (cq *CompletionQueue) Get(index uint32) uint64 { + // Use mask to avoid overflowing and loop back around the ring. + return cq.ring[index&cq.mask] +} + +// FreeAll dequeues as many buffers as possible from the queue and returns them +// to the UMEM. +// +// +checklocks:umem.mu +func (cq *CompletionQueue) FreeAll(umem *UMEM) { + available, index := cq.Peek() + if available < 1 { + return + } + for i := uint32(0); i < available; i++ { + umem.FreeFrame(cq.Get(index + i)) + } + cq.Release(available) +} diff --git a/pkg/xdp/fillqueue.go b/pkg/xdp/fillqueue.go new file mode 100644 index 0000000..775977a --- /dev/null +++ b/pkg/xdp/fillqueue.go @@ -0,0 +1,121 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" +) + +// The FillQueue is how a process tells the kernel which buffers are available +// to be filled by incoming packets. +// +// FillQueue is not thread-safe and requires external synchronization +type FillQueue struct { + // mem is the mmap'd area shared with the kernel. Many other fields of + // this struct point into mem. + mem []byte + + // ring is the actual ring buffer. It is a list of frame addresses + // ready for incoming packets. + // + // len(ring) must be a power of 2. + ring []uint64 + + // mask is used whenever indexing into ring. It is always len(ring)-1. + // It prevents index out of bounds errors while allowing the producer + // and consumer pointers to repeatedly "overflow" and loop back around + // the ring. + mask uint32 + + // producer points to the shared atomic value that indicates the last + // produced descriptor. Only we update this value. + producer *atomicbitops.Uint32 + + // consumer points to the shared atomic value that indicates the last + // consumed descriptor. Only the kernel updates this value. + consumer *atomicbitops.Uint32 + + // flags points to the shared atomic value that holds flags for the + // queue. + flags *atomicbitops.Uint32 + + // Cached values are used to avoid relatively expensive atomic + // operations. They are used, incremented, and decremented multiple + // times with non-atomic operations, and then "batch-updated" by + // reading or writing atomically to synchronize with the kernel. + + // cachedProducer is used to atomically write *producer. + cachedProducer uint32 + // cachedConsumer is updated when we atomically read *consumer. + // cachedConsumer is actually len(ring) larger than the real consumer + // value. See free() for details. + cachedConsumer uint32 +} + +// free returns the number of free descriptors in the fill queue. +func (fq *FillQueue) free(toReserve uint32) uint32 { + // Try to find free descriptors without incurring an atomic operation. + // + // cachedConsumer is always len(fq.ring) larger than the real consumer + // value. This lets us, in the common case, compute the number of free + // descriptors simply via fq.cachedConsumer - fq.cachedProducer without + // also adding len(fq.ring). + if available := fq.cachedConsumer - fq.cachedProducer; available >= toReserve { + return available + } + + // If we didn't already have enough descriptors available, check + // whether the kernel has returned some to us. + fq.cachedConsumer = fq.consumer.Load() + fq.cachedConsumer += uint32(len(fq.ring)) + return fq.cachedConsumer - fq.cachedProducer +} + +// Notify updates the producer such that it is visible to the kernel. +func (fq *FillQueue) Notify() { + fq.producer.Store(fq.cachedProducer) +} + +// Set sets the fill queue's descriptor at index to addr. +func (fq *FillQueue) Set(index uint32, addr uint64) { + // Use mask to avoid overflowing and loop back around the ring. + fq.ring[index&fq.mask] = addr +} + +// FillAll posts as many empty buffers as possible for the kernel to fill, then +// notifies the kernel. +// +// +checklocks:umem.mu +func (fq *FillQueue) FillAll(umem *UMEM) { + // Figure out how many buffers and queue slots are available. + available := fq.free(umem.nFreeFrames) + if available == 0 { + return + } + if available > umem.nFreeFrames { + available = umem.nFreeFrames + } + + // Fill the queue as much as possible and notify the kernel. + index := fq.cachedProducer + fq.cachedProducer += available + for i := uint32(0); i < available; i++ { + fq.Set(index+i, umem.AllocFrame()) + } + fq.Notify() +} diff --git a/pkg/xdp/rxqueue.go b/pkg/xdp/rxqueue.go new file mode 100644 index 0000000..b6350fe --- /dev/null +++ b/pkg/xdp/rxqueue.go @@ -0,0 +1,105 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "golang.org/x/sys/unix" +) + +// The RXQueue is how the kernel tells a process which buffers are full with +// incoming packets. +// +// RXQueue is not thread-safe and requires external synchronization +type RXQueue struct { + // mem is the mmap'd area shared with the kernel. Many other fields of + // this struct point into mem. + mem []byte + + // ring is the actual ring buffer. It is a list of XDP descriptors + // pointing to incoming packets. + // + // len(ring) must be a power of 2. + ring []unix.XDPDesc + + // mask is used whenever indexing into ring. It is always len(ring)-1. + // It prevents index out of bounds errors while allowing the producer + // and consumer pointers to repeatedly "overflow" and loop back around + // the ring. + mask uint32 + + // producer points to the shared atomic value that indicates the last + // produced descriptor. Only the kernel updates this value. + producer *atomicbitops.Uint32 + + // consumer points to the shared atomic value that indicates the last + // consumed descriptor. Only we update this value. + consumer *atomicbitops.Uint32 + + // flags points to the shared atomic value that holds flags for the + // queue. + flags *atomicbitops.Uint32 + + // Cached values are used to avoid relatively expensive atomic + // operations. They are used, incremented, and decremented multiple + // times with non-atomic operations, and then "batch-updated" by + // reading or writing atomically to synchronize with the kernel. + + // cachedProducer is updated when we atomically read *producer. + cachedProducer uint32 + // cachedConsumer is used to atomically write *consumer. + cachedConsumer uint32 +} + +// Peek returns the number of packets available to read as well as the index at +// which they start. Peek will only return a packet once, so callers must +// process any received packets. +func (rq *RXQueue) Peek() (nReceived, index uint32) { + // Get the number of available buffers and update cachedConsumer to + // reflect that we're going to consume them. + entries := rq.free() + index = rq.cachedConsumer + rq.cachedConsumer += entries + return entries, index +} + +func (rq *RXQueue) free() uint32 { + // Return any buffers we know about without incurring an atomic + // operation if possible. + entries := rq.cachedProducer - rq.cachedConsumer + // If we're not aware of any RX'd packets, refresh the producer pointer + // to see whether the kernel enqueued anything. + if entries == 0 { + rq.cachedProducer = rq.producer.Load() + entries = rq.cachedProducer - rq.cachedConsumer + } + return entries +} + +// Release notifies the kernel that we have consumed nDone packets. +func (rq *RXQueue) Release(nDone uint32) { + // We don't have to use an atomic add because only we update this; the + // kernel just reads it. + rq.consumer.Store(rq.consumer.RacyLoad() + nDone) +} + +// Get gets the descriptor at index. +func (rq *RXQueue) Get(index uint32) unix.XDPDesc { + // Use mask to avoid overflowing and loop back around the ring. + return rq.ring[index&rq.mask] +} diff --git a/pkg/xdp/txqueue.go b/pkg/xdp/txqueue.go new file mode 100644 index 0000000..66f4d4f --- /dev/null +++ b/pkg/xdp/txqueue.go @@ -0,0 +1,116 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "github.com/sagernet/gvisor/pkg/atomicbitops" + "golang.org/x/sys/unix" +) + +// The TXQueue is how a process tells the kernel which buffers are available to +// be sent via the NIC. +// +// TXQueue is not thread-safe and requires external synchronization +type TXQueue struct { + // sockfd is the underlying AF_XDP socket. + sockfd uint32 + + // mem is the mmap'd area shared with the kernel. Many other fields of + // this struct point into mem. + mem []byte + + // ring is the actual ring buffer. It is a list of XDP descriptors + // pointing to ready-to-transmit packets. + // + // len(ring) must be a power of 2. + ring []unix.XDPDesc + + // mask is used whenever indexing into ring. It is always len(ring)-1. + // It prevents index out of bounds errors while allowing the producer + // and consumer pointers to repeatedly "overflow" and loop back around + // the ring. + mask uint32 + + // producer points to the shared atomic value that indicates the last + // produced descriptor. Only we update this value. + producer *atomicbitops.Uint32 + + // consumer points to the shared atomic value that indicates the last + // consumed descriptor. Only the kernel updates this value. + consumer *atomicbitops.Uint32 + + // flags points to the shared atomic value that holds flags for the + // queue. + flags *atomicbitops.Uint32 + + // Cached values are used to avoid relatively expensive atomic + // operations. They are used, incremented, and decremented multiple + // times with non-atomic operations, and then "batch-updated" by + // reading or writing atomically to synchronize with the kernel. + + // cachedProducer is used to atomically write *producer. + cachedProducer uint32 + // cachedConsumer is updated when we atomically read *consumer. + // cachedConsumer is actually len(ring) larger than the real consumer + // value. See free() for details. + cachedConsumer uint32 +} + +// Reserve reserves descriptors in the queue. If toReserve descriptors cannot +// be reserved, none are reserved. +// +// +checklocks:umem.mu +func (tq *TXQueue) Reserve(umem *UMEM, toReserve uint32) (nReserved, index uint32) { + if umem.nFreeFrames < toReserve || tq.free(toReserve) < toReserve { + return 0, 0 + } + idx := tq.cachedProducer + tq.cachedProducer += toReserve + return toReserve, idx +} + +// free returns the number of free descriptors in the TX queue. +func (tq *TXQueue) free(toReserve uint32) uint32 { + // Try to find free descriptors without incurring an atomic operation. + // + // cachedConsumer is always len(tq.ring) larger than the real consumer + // value. This lets us, in the common case, compute the number of free + // descriptors simply via tq.cachedConsumer - tq.cachedProducer without + // also addign len(tq.ring). + if available := tq.cachedConsumer - tq.cachedProducer; available >= toReserve { + return available + } + + // If we didn't already have enough descriptors available, check + // whether the kernel has returned some to us. + tq.cachedConsumer = tq.consumer.Load() + tq.cachedConsumer += uint32(len(tq.ring)) + return tq.cachedConsumer - tq.cachedProducer +} + +// Notify updates the producer such that it is visible to the kernel. +func (tq *TXQueue) Notify() { + tq.producer.Store(tq.cachedProducer) + tq.kick() +} + +// Set sets the TX queue's descriptor at index to addr. +func (tq *TXQueue) Set(index uint32, desc unix.XDPDesc) { + // Use mask to avoid overflowing and loop back around the ring. + tq.ring[index&tq.mask] = desc +} diff --git a/pkg/xdp/umem.go b/pkg/xdp/umem.go new file mode 100644 index 0000000..b1b2a69 --- /dev/null +++ b/pkg/xdp/umem.go @@ -0,0 +1,107 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "fmt" + + "github.com/sagernet/gvisor/pkg/sync" + "golang.org/x/sys/unix" +) + +// TODO(b/240191988): There's some kind of memory corruption bug that occurs +// occasionally. This occurred even before TX was supported. + +// TODO(b/240191988): We can hold locks for less time if we accept a more +// obtuse API. For example, CompletionQueue.FreeAll doesn't need to hold a +// mutex for its entire duration. + +// UMEM is the shared memory area that the kernel and userspace put packets in. +type UMEM struct { + // mem is the mmap'd area shared with the kernel. + mem []byte + + // sockfd is the underlying AF_XDP socket. + sockfd uint32 + + // frameMask masks the lower bits of an address to get the frame's + // address. + frameMask uint64 + + // mu protects frameAddresses and nFreeFrames. + mu sync.Mutex + + // frameAddresses is a stack of available frame addresses. + // +checklocks:mu + frameAddresses []uint64 + + // nFreeFrames is the number of frames available and is used to index + // into frameAddresses. + // +checklocks:mu + nFreeFrames uint32 +} + +// SockFD returns the underlying AF_XDP socket FD. +func (um *UMEM) SockFD() uint32 { + return um.sockfd +} + +// Lock locks the UMEM. +// +// +checklocksacquire:um.mu +func (um *UMEM) Lock() { + um.mu.Lock() +} + +// Unlock unlocks the UMEM. +// +// +checklocksrelease:um.mu +func (um *UMEM) Unlock() { + um.mu.Unlock() +} + +// FreeFrame returns the frame containing addr to the set of free frames. +// +// The UMEM must be locked during the call to FreeFrame. +// +// +checklocks:um.mu +func (um *UMEM) FreeFrame(addr uint64) { + um.frameAddresses[um.nFreeFrames] = addr + um.nFreeFrames++ +} + +// AllocFrame returns the address of a frame that can be enqueued to the fill +// or TX queue. It will panic if there are no frames left, so callers must call +// it no more than the number of buffers reserved via TXQueue.Reserve(). +// +// The UMEM must be locked during the call to AllocFrame. +// +// +checklocks:um.mu +func (um *UMEM) AllocFrame() uint64 { + um.nFreeFrames-- + return um.frameAddresses[um.nFreeFrames] & um.frameMask +} + +// Get gets the bytes of the packet pointed to by desc. +func (um *UMEM) Get(desc unix.XDPDesc) []byte { + end := desc.Addr + uint64(desc.Len) + if desc.Addr&um.frameMask != (end-1)&um.frameMask { + panic(fmt.Sprintf("UMEM (%+v) access crosses frame boundaries: %+v", um, desc)) + } + return um.mem[desc.Addr:end] +} diff --git a/pkg/xdp/xdp.go b/pkg/xdp/xdp.go new file mode 100644 index 0000000..9ee33c0 --- /dev/null +++ b/pkg/xdp/xdp.go @@ -0,0 +1,323 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +// Package xdp provides tools for working with AF_XDP sockets. +// +// AF_XDP shares a memory area (UMEM) with the kernel to pass packets +// back and forth. Communication is done via a number of queues. +// Briefly, the queues work as follows: +// +// - Receive: Userspace adds a descriptor to the fill queue. The +// descriptor points to an area of the UMEM that the kernel should fill +// with an incoming packet. The packet is filled by the kernel, which +// places a descriptor to the same UMEM area in the RX queue, signifying +// that userspace may read the packet. +// - Transmit: Userspace adds a descriptor to TX queue. The kernel +// sends the packet (stored in UMEM) pointed to by the descriptor. +// Upon completion, the kernel places a descriptor in the completion +// queue to notify userspace that the packet is sent and the UMEM +// area can be reused. +// +// So in short: RX packets move from the fill to RX queue, and TX +// packets move from the TX to completion queue. +// +// Note that the shared UMEM for RX and TX means that packet forwarding +// can be done without copying; only the queues need to be updated to point to +// the packet in UMEM. +package xdp + +import ( + "fmt" + "math/bits" + + "github.com/sagernet/gvisor/pkg/cleanup" + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/memutil" + "golang.org/x/sys/unix" +) + +// A ControlBlock contains all the control structures necessary to use an +// AF_XDP socket. +// +// The ControlBlock and the structures it contains are meant to be used with a +// single RX goroutine and a single TX goroutine. +type ControlBlock struct { + UMEM UMEM + Fill FillQueue + RX RXQueue + TX TXQueue + Completion CompletionQueue +} + +// Opts configure an AF_XDP socket. +type Opts struct { + NFrames uint32 + FrameSize uint32 + NDescriptors uint32 + Bind bool + UseNeedWakeup bool +} + +// DefaultOpts provides recommended default options for initializing an AF_XDP +// socket. AF_XDP setup is extremely finnicky and can fail if incorrect values +// are used. +func DefaultOpts() Opts { + return Opts{ + NFrames: 4096, + // Frames must be 2048 or 4096 bytes, although not all drivers support + // both. + FrameSize: 4096, + NDescriptors: 2048, + } +} + +// New returns an initialized AF_XDP socket bound to a particular interface and +// queue. +func New(ifaceIdx, queueID uint32, opts Opts) (*ControlBlock, error) { + sockfd, err := unix.Socket(unix.AF_XDP, unix.SOCK_RAW, 0) + if err != nil { + return nil, fmt.Errorf("failed to create AF_XDP socket: %v", err) + } + return NewFromSocket(sockfd, ifaceIdx, queueID, opts) +} + +// NewFromSocket takes an AF_XDP socket, initializes it, and binds it to a +// particular interface and queue. +func NewFromSocket(sockfd int, ifaceIdx, queueID uint32, opts Opts) (*ControlBlock, error) { + if opts.FrameSize != 2048 && opts.FrameSize != 4096 { + return nil, fmt.Errorf("invalid frame size %d: must be either 2048 or 4096", opts.FrameSize) + } + if bits.OnesCount32(opts.NDescriptors) != 1 { + return nil, fmt.Errorf("invalid number of descriptors %d: must be a power of 2", opts.NDescriptors) + } + + var cb ControlBlock + + // Create the UMEM area. Use mmap instead of make([[]byte) to ensure + // that the UMEM is page-aligned. Aligning the UMEM keeps individual + // packets from spilling over between pages. + var zerofd uintptr + umemMemory, err := memutil.MapSlice( + 0, + uintptr(opts.NFrames*opts.FrameSize), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_PRIVATE|unix.MAP_ANONYMOUS, + zerofd-1, + 0, + ) + if err != nil { + return nil, fmt.Errorf("failed to mmap umem: %v", err) + } + cleanup := cleanup.Make(func() { + memutil.UnmapSlice(umemMemory) + }) + + if sliceBackingPointer(umemMemory)%uintptr(unix.Getpagesize()) != 0 { + return nil, fmt.Errorf("UMEM is not page aligned (address 0x%x)", sliceBackingPointer(umemMemory)) + } + + cb.UMEM = UMEM{ + mem: umemMemory, + sockfd: uint32(sockfd), + frameAddresses: make([]uint64, opts.NFrames), + nFreeFrames: opts.NFrames, + frameMask: ^(uint64(opts.FrameSize) - 1), + } + + // Fill in each frame address. + for i := range cb.UMEM.frameAddresses { + cb.UMEM.frameAddresses[i] = uint64(i) * uint64(opts.FrameSize) + } + + // Check whether we're likely to fail due to RLIMIT_MEMLOCK. + var rlimit unix.Rlimit + if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlimit); err != nil { + return nil, fmt.Errorf("failed to get rlimit for memlock: %v", err) + } + if rlimit.Cur < uint64(len(cb.UMEM.mem)) { + log.Infof("UMEM size (%d) may exceed RLIMIT_MEMLOCK (%+v) and cause registration to fail", len(cb.UMEM.mem), rlimit) + } + + reg := unix.XDPUmemReg{ + Addr: uint64(sliceBackingPointer(umemMemory)), + Len: uint64(len(umemMemory)), + Size: opts.FrameSize, + // Not useful in the RX path. + Headroom: 0, + // TODO(b/240191988): Investigate use of SHARED flag. + Flags: 0, + } + if err := registerUMEM(sockfd, reg); err != nil { + return nil, fmt.Errorf("failed to register UMEM: %v", err) + } + + // Set the number of descriptors in the fill queue. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_FILL_RING, int(opts.NDescriptors)); err != nil { + return nil, fmt.Errorf("failed to register fill ring: %v", err) + } + // Set the number of descriptors in the completion queue. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_COMPLETION_RING, int(opts.NDescriptors)); err != nil { + return nil, fmt.Errorf("failed to register completion ring: %v", err) + } + // Set the number of descriptors in the RX queue. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_RX_RING, int(opts.NDescriptors)); err != nil { + return nil, fmt.Errorf("failed to register RX queue: %v", err) + } + // Set the number of descriptors in the TX queue. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_TX_RING, int(opts.NDescriptors)); err != nil { + return nil, fmt.Errorf("failed to register TX queue: %v", err) + } + + // Get offset information for the queues. Offsets indicate where, once + // we mmap space for each queue, values in the queue are. They give + // offsets for the shared pointers, a shared flags value, and the + // beginning of the ring of descriptors. + off, err := getOffsets(sockfd) + if err != nil { + return nil, fmt.Errorf("failed to get offsets: %v", err) + } + + // Allocate space for the fill queue. + fillQueueMem, err := memutil.MapSlice( + 0, + uintptr(off.Fr.Desc+uint64(opts.NDescriptors)*sizeOfFillQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE, + uintptr(sockfd), + unix.XDP_UMEM_PGOFF_FILL_RING, + ) + if err != nil { + return nil, fmt.Errorf("failed to mmap fill queue: %v", err) + } + cleanup.Add(func() { + memutil.UnmapSlice(fillQueueMem) + }) + // Setup the fillQueue with offsets into allocated memory. + cb.Fill = FillQueue{ + mem: fillQueueMem, + mask: opts.NDescriptors - 1, + cachedConsumer: opts.NDescriptors, + } + cb.Fill.init(off, opts) + + // Allocate space for the completion queue. + completionQueueMem, err := memutil.MapSlice( + 0, + uintptr(off.Cr.Desc+uint64(opts.NDescriptors)*sizeOfCompletionQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE, + uintptr(sockfd), + unix.XDP_UMEM_PGOFF_COMPLETION_RING, + ) + if err != nil { + return nil, fmt.Errorf("failed to mmap completion queue: %v", err) + } + cleanup.Add(func() { + memutil.UnmapSlice(completionQueueMem) + }) + // Setup the completionQueue with offsets into allocated memory. + cb.Completion = CompletionQueue{ + mem: completionQueueMem, + mask: opts.NDescriptors - 1, + } + cb.Completion.init(off, opts) + + // Allocate space for the RX queue. + rxQueueMem, err := memutil.MapSlice( + 0, + uintptr(off.Rx.Desc+uint64(opts.NDescriptors)*sizeOfRXQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE, + uintptr(sockfd), + unix.XDP_PGOFF_RX_RING, + ) + if err != nil { + return nil, fmt.Errorf("failed to mmap RX queue: %v", err) + } + cleanup.Add(func() { + memutil.UnmapSlice(rxQueueMem) + }) + // Setup the rxQueue with offsets into allocated memory. + cb.RX = RXQueue{ + mem: rxQueueMem, + mask: opts.NDescriptors - 1, + } + cb.RX.init(off, opts) + + // Allocate space for the TX queue. + txQueueMem, err := memutil.MapSlice( + 0, + uintptr(off.Tx.Desc+uint64(opts.NDescriptors)*sizeOfTXQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE, + uintptr(sockfd), + unix.XDP_PGOFF_TX_RING, + ) + if err != nil { + return nil, fmt.Errorf("failed to mmap tx queue: %v", err) + } + cleanup.Add(func() { + memutil.UnmapSlice(txQueueMem) + }) + // Setup the txQueue with offsets into allocated memory. + cb.TX = TXQueue{ + sockfd: uint32(sockfd), + mem: txQueueMem, + mask: opts.NDescriptors - 1, + cachedConsumer: opts.NDescriptors, + } + cb.TX.init(off, opts) + + // In some cases we don't call bind, as we're not in the netns with the + // device. In those cases, another process with the same socket will + // bind for us. + if opts.Bind { + if err := Bind(sockfd, ifaceIdx, queueID, opts.UseNeedWakeup); err != nil { + return nil, fmt.Errorf("failed to bind to interface %d: %v", ifaceIdx, err) + } + } + + cleanup.Release() + return &cb, nil +} + +// Bind binds a socket to a particular network interface and queue. +func Bind(sockfd int, ifindex, queueID uint32, useNeedWakeup bool) error { + var flags uint16 + if useNeedWakeup { + flags |= unix.XDP_USE_NEED_WAKEUP + } + addr := unix.SockaddrXDP{ + // XDP_USE_NEED_WAKEUP lets the driver sleep if there is no + // work to do. It will need to be woken by poll. It is expected + // that this improves performance by preventing the driver from + // burning cycles. + // + // By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct + // the kernel to use zerocopy if available and then fallback to + // copy mode. + Flags: flags, + Ifindex: ifindex, + // AF_XDP sockets are per device RX queue, although multiple + // sockets on multiple queues (or devices) can share a single + // UMEM. + QueueID: queueID, + // We're not using shared mode, so the value here is irrelevant. + SharedUmemFD: 0, + } + return unix.Bind(sockfd, &addr) +} diff --git a/pkg/xdp/xdp_state_autogen.go b/pkg/xdp/xdp_state_autogen.go new file mode 100644 index 0000000..9627301 --- /dev/null +++ b/pkg/xdp/xdp_state_autogen.go @@ -0,0 +1,11 @@ +// automatically generated by stateify. + +//go:build (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) && (amd64 || arm64) +// +build amd64 arm64 +// +build amd64 arm64 +// +build amd64 arm64 +// +build amd64 arm64 +// +build amd64 arm64 +// +build amd64 arm64 + +package xdp diff --git a/pkg/xdp/xdp_unsafe.go b/pkg/xdp/xdp_unsafe.go new file mode 100644 index 0000000..a088b9f --- /dev/null +++ b/pkg/xdp/xdp_unsafe.go @@ -0,0 +1,123 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xdp + +import ( + "fmt" + "reflect" + "unsafe" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "golang.org/x/sys/unix" +) + +func registerUMEM(fd int, reg unix.XDPUmemReg) error { + if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_UMEM_REG, uintptr(unsafe.Pointer(®)), unsafe.Sizeof(reg), 0); errno != 0 { + return fmt.Errorf("failed to setsockopt(XDP_UMEM_REG): errno %d", errno) + } + return nil +} + +func getOffsets(fd int) (unix.XDPMmapOffsets, error) { + var off unix.XDPMmapOffsets + size := unsafe.Sizeof(off) + if _, _, errno := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_MMAP_OFFSETS, uintptr(unsafe.Pointer(&off)), uintptr(unsafe.Pointer(&size)), 0); errno != 0 { + return unix.XDPMmapOffsets{}, fmt.Errorf("failed to get offsets: %v", errno) + } else if unsafe.Sizeof(off) != size { + return unix.XDPMmapOffsets{}, fmt.Errorf("expected optlen of %d, but found %d", unsafe.Sizeof(off), size) + } + return off, nil +} + +func sliceBackingPointer(slice []byte) uintptr { + return uintptr(unsafe.Pointer(&slice[0])) +} + +func sizeOfFillQueueDesc() uint64 { + return uint64(unsafe.Sizeof(uint64(0))) +} + +func sizeOfRXQueueDesc() uint64 { + return uint64(unsafe.Sizeof(unix.XDPDesc{})) +} + +func sizeOfCompletionQueueDesc() uint64 { + return uint64(unsafe.Sizeof(uint64(0))) +} + +func sizeOfTXQueueDesc() uint64 { + return uint64(unsafe.Sizeof(unix.XDPDesc{})) +} + +func (fq *FillQueue) init(off unix.XDPMmapOffsets, opts Opts) { + fillQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&fq.ring)) + fillQueueRingHdr.Data = uintptr(unsafe.Pointer(&fq.mem[off.Fr.Desc])) + fillQueueRingHdr.Len = int(opts.NDescriptors) + fillQueueRingHdr.Cap = fillQueueRingHdr.Len + fq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Producer])) + fq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Consumer])) + fq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Flags])) +} + +func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts Opts) { + rxQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&rq.ring)) + rxQueueRingHdr.Data = uintptr(unsafe.Pointer(&rq.mem[off.Rx.Desc])) + rxQueueRingHdr.Len = int(opts.NDescriptors) + rxQueueRingHdr.Cap = rxQueueRingHdr.Len + rq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Producer])) + rq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Consumer])) + rq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Flags])) + // These probably don't have to be atomic, but we're only loading once + // so better safe than sorry. + rq.cachedProducer = rq.producer.Load() + rq.cachedConsumer = rq.consumer.Load() +} + +func (cq *CompletionQueue) init(off unix.XDPMmapOffsets, opts Opts) { + completionQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cq.ring)) + completionQueueRingHdr.Data = uintptr(unsafe.Pointer(&cq.mem[off.Cr.Desc])) + completionQueueRingHdr.Len = int(opts.NDescriptors) + completionQueueRingHdr.Cap = completionQueueRingHdr.Len + cq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Producer])) + cq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Consumer])) + cq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Flags])) + // These probably don't have to be atomic, but we're only loading once + // so better safe than sorry. + cq.cachedProducer = cq.producer.Load() + cq.cachedConsumer = cq.consumer.Load() +} + +func (tq *TXQueue) init(off unix.XDPMmapOffsets, opts Opts) { + txQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&tq.ring)) + txQueueRingHdr.Data = uintptr(unsafe.Pointer(&tq.mem[off.Tx.Desc])) + txQueueRingHdr.Len = int(opts.NDescriptors) + txQueueRingHdr.Cap = txQueueRingHdr.Len + tq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Producer])) + tq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Consumer])) + tq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Flags])) +} + +// kick notifies the kernel that there are packets to transmit. +func (tq *TXQueue) kick() error { + if tq.flags.RacyLoad()&unix.XDP_RING_NEED_WAKEUP == 0 { + return nil + } + + var msg unix.Msghdr + if _, _, errno := unix.Syscall6(unix.SYS_SENDMSG, uintptr(tq.sockfd), uintptr(unsafe.Pointer(&msg)), unix.MSG_DONTWAIT|unix.MSG_NOSIGNAL, 0, 0, 0); errno != 0 { + return fmt.Errorf("failed to kick TX queue via sendmsg: errno %d", errno) + } + return nil +} diff --git a/pkg/xdp/xdp_unsafe_state_autogen.go b/pkg/xdp/xdp_unsafe_state_autogen.go new file mode 100644 index 0000000..f1d5f1e --- /dev/null +++ b/pkg/xdp/xdp_unsafe_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package xdp diff --git a/reformat.sh b/reformat.sh new file mode 100644 index 0000000..8d3ac05 --- /dev/null +++ b/reformat.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +GO_FILES=$(find . -name "*.go" | grep -v .git) + +gofumpt -l -w $GO_FILES +gofmt -l -w $GO_FILES +gci write $GO_FILES diff --git a/remove-unused.sh b/remove-unused.sh new file mode 100644 index 0000000..09c50e9 --- /dev/null +++ b/remove-unused.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +function remove_unused() { + git rm -rf --ignore-unmatch \ + runsc shim tools webhook \ + pkg/shim \ + pkg/eventchannel \ + pkg/coverage \ + pkg/sentry \ + pkg/metric \ + pkg/hostos \ + pkg/ring0 \ + pkg/prometheus \ + pkg/seccomp \ + pkg/sigframe \ + pkg/bpf \ + pkg/aio \ + pkg/urpc \ + pkg/control \ + pkg/bitmap \ + pkg/p9 \ + pkg/lisafs \ + pkg/erofs \ + pkg/devutil \ + pkg/safemem \ + pkg/usermem \ + pkg/tcpip/nftables +} + +remove_unused +remove_unused + +./reformat.sh + +go mod tidy +git commit -a -m "Remove unused" diff --git a/rename-module.sh b/rename-module.sh new file mode 100644 index 0000000..6872b5d --- /dev/null +++ b/rename-module.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +OLD_MODULE_NAME="gvisor.dev/gvisor" +NEW_MODULE_NAME="github.com/sagernet/gvisor" + +rules=$(cat <.*) + by: $NEW_MODULE_NAME\$PATH +fix: \$NEW_IMPORT +EOF +) + +sg scan --inline-rules "$rules" -U + +sed -i "s|module $OLD_MODULE_NAME|module $NEW_MODULE_NAME|" go.mod + +go mod tidy + +./reformat.sh + +git commit -m "Rename module" -a