diff --git a/go.mod b/go.mod index 7ba1162..30ac698 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,12 @@ module github.com/sagernet/gvisor -go 1.24.1 +go 1.25.0 require ( github.com/google/btree v1.1.2 - golang.org/x/sys v0.26.0 - golang.org/x/time v0.7.0 + github.com/opencontainers/runtime-spec v1.2.1 + golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc + golang.org/x/sys v0.43.0 + golang.org/x/time v0.15.0 + google.golang.org/protobuf v1.36.11 ) diff --git a/go.sum b/go.sum index 7523721..63bcd7c 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,14 @@ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= +github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc= +golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/pkg/abi/gasket/gasket.go b/pkg/abi/gasket/gasket.go index c3829b3..1286bcf 100644 --- a/pkg/abi/gasket/gasket.go +++ b/pkg/abi/gasket/gasket.go @@ -17,6 +17,7 @@ package gasket import ( "fmt" + "structs" "github.com/sagernet/gvisor/pkg/abi/linux" ) @@ -80,6 +81,7 @@ func (i Ioctl) String() string { // // +marshal type GasketInterruptEventFd struct { + _ structs.HostLayout Interrupt uint64 EventFD uint64 } @@ -89,6 +91,7 @@ type GasketInterruptEventFd struct { // // +marshal type GasketPageTableIoctl struct { + _ structs.HostLayout PageTableIndex uint64 Size uint64 HostAddress uint64 @@ -101,6 +104,7 @@ type GasketPageTableIoctl struct { // // +marshal type GasketInterruptMapping struct { + _ structs.HostLayout Interrupt uint64 EventFD uint64 BarIndex uint64 @@ -112,6 +116,7 @@ type GasketInterruptMapping struct { // // +marshal type GasketPageTableDmaBufIoctl struct { + _ structs.HostLayout PageTableIndex uint64 DeviceAddress uint64 DMABufID int32 `marshal:"unaligned"` // Struct ends mid 64bit word. diff --git a/pkg/abi/linux/aio.go b/pkg/abi/linux/aio.go index 044f783..577f66c 100644 --- a/pkg/abi/linux/aio.go +++ b/pkg/abi/linux/aio.go @@ -14,11 +14,15 @@ package linux -import "encoding/binary" +import ( + "encoding/binary" + "structs" +) // AIORing is struct aio_ring, from fs/aio.c, without the trailing // variable-length array. type AIORing struct { + _ structs.HostLayout ID uint32 Nr uint32 Head uint32 @@ -62,6 +66,7 @@ const ( // // +marshal type IOCallback struct { + _ structs.HostLayout Data uint64 Key uint32 _ uint32 @@ -86,6 +91,7 @@ type IOCallback struct { // +marshal // +stateify savable type IOEvent struct { + _ structs.HostLayout Data uint64 Obj uint64 Result int64 diff --git a/pkg/abi/linux/bpf.go b/pkg/abi/linux/bpf.go index 9422fcf..43cd222 100644 --- a/pkg/abi/linux/bpf.go +++ b/pkg/abi/linux/bpf.go @@ -14,11 +14,16 @@ package linux +import ( + "structs" +) + // BPFInstruction is a raw BPF virtual machine instruction. // // +marshal slice:BPFInstructionSlice // +stateify savable type BPFInstruction struct { + _ structs.HostLayout // OpCode is the operation to execute. OpCode uint16 diff --git a/pkg/abi/linux/capability.go b/pkg/abi/linux/capability.go index a98aca8..9f312f8 100644 --- a/pkg/abi/linux/capability.go +++ b/pkg/abi/linux/capability.go @@ -16,6 +16,7 @@ package linux import ( "strings" + "structs" ) // A Capability represents the ability to perform a privileged operation. @@ -254,6 +255,7 @@ const ( // // +marshal type VfsCapData struct { + _ structs.HostLayout MagicEtc uint32 PermittedLo uint32 InheritableLo uint32 @@ -287,6 +289,7 @@ func (c *VfsCapData) ToString() string { // // +marshal type VfsNsCapData struct { + _ structs.HostLayout VfsCapData RootID uint32 } @@ -323,6 +326,7 @@ func (c *VfsNsCapData) ToString() string { // // +marshal type CapUserHeader struct { + _ structs.HostLayout Version uint32 Pid int32 } @@ -331,6 +335,7 @@ type CapUserHeader struct { // // +marshal slice:CapUserDataSlice type CapUserData struct { + _ structs.HostLayout Effective uint32 Permitted uint32 Inheritable uint32 diff --git a/pkg/abi/linux/clone.go b/pkg/abi/linux/clone.go index ae1276d..e565624 100644 --- a/pkg/abi/linux/clone.go +++ b/pkg/abi/linux/clone.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Clone constants per clone(2). const ( CSIGNAL = 0xff @@ -57,6 +61,7 @@ const ( // // +marshal type CloneArgs struct { + _ structs.HostLayout Flags uint64 Pidfd uint64 ChildTID uint64 diff --git a/pkg/abi/linux/ebpf.go b/pkg/abi/linux/ebpf.go new file mode 100644 index 0000000..0ead4fe --- /dev/null +++ b/pkg/abi/linux/ebpf.go @@ -0,0 +1,303 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linux + +import ( + "structs" + + "github.com/sagernet/gvisor/pkg/marshal" +) + +// EBPFInstruction is the userspace representation of an eBPF instruction that has not +// been validated. +// +// +marshal slice:EBPFInstructionSlice +// +stateify savable +type EBPFInstruction struct { + _ structs.HostLayout + + Code uint8 + Registers uint8 // LE: 4 LSBs are destination, 4 MSBs are source + Offset int16 + Immediate int32 +} + +// Constants defining eBPF-related limits. +const ( + // Maximum instruction count in an eBPF program + BPF_COMPLEXITY_LIMIT_INSNS = 1_000_000 + + // Maximum length of an eBPF program's name + BPF_OBJ_NAME_LEN = 16 + + // Size of an EbpfInstruction + BPF_INSTRUCTION_SIZE = 8 + + // Maximum number of cgroup eBPF programs per attachment type. + BPF_CGROUP_MAX_PROGS = 64 +) + +// Valid values for `cmd` for bpf(2). +const ( + BPF_MAP_CREATE = iota + BPF_MAP_LOOKUP_ELEM + BPF_MAP_UPDATE_ELEM + BPF_MAP_DELETE_ELEM + BPF_MAP_GET_NEXT_KEY + BPF_PROG_LOAD + BPF_OBJ_PIN + BPF_OBJ_GET + BPF_PROG_ATTACH + BPF_PROG_DETACH + BPF_PROG_TEST_RUN + BPF_PROG_GET_NEXT_ID + BPF_MAP_GET_NEXT_ID + BPF_PROG_GET_FD_BY_ID + BPF_MAP_GET_FD_BY_ID + BPF_OBJ_GET_INFO_BY_FD + BPF_PROG_QUERY + BPF_RAW_TRACEPOINT_OPEN + BPF_BTF_LOAD + BPF_BTF_GET_FD_BY_ID + BPF_TASK_FD_QUERY + BPF_MAP_LOOKUP_AND_DELETE_ELEM + BPF_MAP_FREEZE + BPF_BTF_GET_NEXT_ID + BPF_MAP_LOOKUP_BATCH + BPF_MAP_LOOKUP_AND_DELETE_BATCH + BPF_MAP_UPDATE_BATCH + BPF_MAP_DELETE_BATCH + BPF_LINK_CREATE + BPF_LINK_UPDATE + BPF_LINK_GET_FD_BY_ID + BPF_LINK_GET_NEXT_ID + BPF_ENABLE_STATS + BPF_ITER_CREATE + BPF_LINK_DETACH + BPF_PROG_BIND_MAP + BPF_TOKEN_CREATE + BPF_PROG_STREAM_READ_BY_FD + BPF_PROG_ASSOC_STRUCT_OPS + + BPF_PROG_RUN = BPF_PROG_TEST_RUN +) + +// BPFProgramType represents an type for an eBPF program. +type BPFProgramType uint + +// Valid types of eBPF programs. +const ( + BPF_PROG_TYPE_UNSPEC BPFProgramType = iota + BPF_PROG_TYPE_SOCKET_FILTER + BPF_PROG_TYPE_KPROBE + BPF_PROG_TYPE_SCHED_CLS + BPF_PROG_TYPE_SCHED_ACT + BPF_PROG_TYPE_TRACEPOINT + BPF_PROG_TYPE_XDP + BPF_PROG_TYPE_PERF_EVENT + BPF_PROG_TYPE_CGROUP_SKB + BPF_PROG_TYPE_CGROUP_SOCK + BPF_PROG_TYPE_LWT_IN + BPF_PROG_TYPE_LWT_OUT + BPF_PROG_TYPE_LWT_XMIT + BPF_PROG_TYPE_SOCK_OPS + BPF_PROG_TYPE_SK_SKB + BPF_PROG_TYPE_CGROUP_DEVICE + BPF_PROG_TYPE_SK_MSG + BPF_PROG_TYPE_RAW_TRACEPOINT + BPF_PROG_TYPE_CGROUP_SOCK_ADDR + BPF_PROG_TYPE_LWT_SEG6LOCAL + BPF_PROG_TYPE_LIRC_MODE2 + BPF_PROG_TYPE_SK_REUSEPORT + BPF_PROG_TYPE_FLOW_DISSECTOR + BPF_PROG_TYPE_CGROUP_SYSCTL + BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE + BPF_PROG_TYPE_CGROUP_SOCKOPT + BPF_PROG_TYPE_TRACING + BPF_PROG_TYPE_STRUCT_OPS + BPF_PROG_TYPE_EXT + BPF_PROG_TYPE_LSM + BPF_PROG_TYPE_SK_LOOKUP + BPF_PROG_TYPE_SYSCALL + BPF_PROG_TYPE_NETFILTER +) + +// BPFAttachType represents an attachment type for an eBPF program. +type BPFAttachType uint + +// All valid attachment types for eBPF programs. +const ( + BPF_CGROUP_INET_INGRESS BPFAttachType = iota + BPF_CGROUP_INET_EGRESS + BPF_CGROUP_INET_SOCK_CREATE + BPF_CGROUP_SOCK_OPS + BPF_SK_SKB_STREAM_PARSER + BPF_SK_SKB_STREAM_VERDICT + BPF_CGROUP_DEVICE + BPF_SK_MSG_VERDICT + BPF_CGROUP_INET4_BIND + BPF_CGROUP_INET6_BIND + BPF_CGROUP_INET4_CONNECT + BPF_CGROUP_INET6_CONNECT + BPF_CGROUP_INET4_POST_BIND + BPF_CGROUP_INET6_POST_BIND + BPF_CGROUP_UDP4_SENDMSG + BPF_CGROUP_UDP6_SENDMSG + BPF_LIRC_MODE2 + BPF_FLOW_DISSECTOR + BPF_CGROUP_SYSCTL + BPF_CGROUP_UDP4_RECVMSG + BPF_CGROUP_UDP6_RECVMSG + BPF_CGROUP_GETSOCKOPT + BPF_CGROUP_SETSOCKOPT + BPF_TRACE_RAW_TP + BPF_TRACE_FENTRY + BPF_TRACE_FEXIT + BPF_MODIFY_RETURN + BPF_LSM_MAC + BPF_TRACE_ITER + BPF_CGROUP_INET4_GETPEERNAME + BPF_CGROUP_INET6_GETPEERNAME + BPF_CGROUP_INET4_GETSOCKNAME + BPF_CGROUP_INET6_GETSOCKNAME + BPF_XDP_DEVMAP + BPF_CGROUP_INET_SOCK_RELEASE + BPF_XDP_CPUMAP + BPF_SK_LOOKUP + BPF_XDP + BPF_SK_SKB_VERDICT + BPF_SK_REUSEPORT_SELECT + BPF_SK_REUSEPORT_SELECT_OR_MIGRATE + BPF_PERF_EVENT + BPF_TRACE_KPROBE_MULTI + BPF_LSM_CGROUP + BPF_STRUCT_OPS + BPF_NETFILTER + BPF_TCX_INGRESS + BPF_TCX_EGRESS + BPF_TRACE_UPROBE_MULTI + BPF_CGROUP_UNIX_CONNECT + BPF_CGROUP_UNIX_SENDMSG + BPF_CGROUP_UNIX_RECVMSG + BPF_CGROUP_UNIX_GETPEERNAME + BPF_CGROUP_UNIX_GETSOCKNAME + BPF_NETKIT_PRIMARY + BPF_NETKIT_PEER + BPF_TRACE_KPROBE_SESSION + BPF_TRACE_UPROBE_SESSION + BPF_TRACE_FSESSION +) + +// BPFAttr represents the parameters to a bpf(2) call. +type BPFAttr interface { + marshal.Marshallable + + implementsBPFAttr() +} + +func (a *BPFAttrProgLoad) implementsBPFAttr() {} +func (a *BPFAttrProgQuery) implementsBPFAttr() {} +func (a *BPFAttrProgAttach) implementsBPFAttr() {} + +// BPFAttrProgLoad contains parameters for a BPF_PROG_LOAD command. +// +// +marshal +type BPFAttrProgLoad struct { + _ structs.HostLayout + + ProgType uint32 + InstructionCount uint32 + Instructions uint64 + License uint64 + LogLevel uint32 + LogSize uint32 + LogBuf uint64 + KernVersion uint32 + ProgFlags uint32 + ProgName [BPF_OBJ_NAME_LEN]byte + ProgInterfaceIndex uint32 + ExpectedAttachType uint32 + ProgBTFFD uint32 + FuncInfoRecSize uint32 + FuncInfo uint64 + FuncInfoCount uint32 + LineInfoRecSize uint32 + LineInfo uint64 + LineInfoCount uint32 + AttachBTFID uint32 + AttachFD uint32 // union of either attach_prog_fd or attach_btf_obj_fd + CoreReloCount uint32 + FDArray uint64 + CoreRelos uint64 + CoreReloRecSize uint32 + LogTrueSize uint32 + ProgTokenFD int32 + FDArrayCount uint32 + Signature uint64 + SignatureSize uint32 + KeyringID int32 +} + +// BPFAttrProgQuery contains parameters for a BPF_PROG_QUERY command. +// +// +marshal +type BPFAttrProgQuery struct { + _ structs.HostLayout + + Target uint32 // union of either target_fd or target_ifindex + AttachType uint32 + QueryFlags uint32 + AttachFlags uint32 + ProgIDs uint64 + Count uint32 // union of either prog_cnt or count + _ uint32 // padding + ProgAttachFlags uint64 + LinkIDs uint64 + LinkAttachFlags uint64 + Revision uint64 +} + +// BPFAttrProgAttach contains parameters for a BPF_PROG_ATTACH command. +// +// +marshal +type BPFAttrProgAttach struct { + _ structs.HostLayout + + Target uint32 // union of either target_fd or target_ifindex + AttachBPFFD uint32 + AttachType uint32 + AttachFlags uint32 + ReplaceBPFFD uint32 + Relative uint32 // union of either relative_fd or relative_id + ExpectedRevision uint64 +} + +// BPF_ATTR_SIZE is the size of union bpf_attr, which is the largest of +// the sub-command attribute structures. +// +// BPF_ATTR_SIZE is immutable. +var BPF_ATTR_SIZE = max((*BPFAttrProgLoad)(nil).SizeBytes(), (*BPFAttrProgQuery)(nil).SizeBytes(), (*BPFAttrProgAttach)(nil).SizeBytes()) + +// eBPF-related flags +const ( + BPF_F_ALLOW_OVERRIDE = 1 << iota + BPF_F_ALLOW_MULTI + BPF_F_REPLACE + BPF_F_BEFORE + BPF_F_AFTER + BPF_F_ID + BPF_F_PREORDER + BPF_F_LINK = 1 << 13 +) diff --git a/pkg/abi/linux/elf.go b/pkg/abi/linux/elf.go index c571354..b2360f8 100644 --- a/pkg/abi/linux/elf.go +++ b/pkg/abi/linux/elf.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Linux auxiliary vector entry types. const ( // AT_NULL is the end of the auxiliary vector. @@ -111,6 +115,7 @@ const ( // // +marshal type ElfHeader64 struct { + _ structs.HostLayout Ident [16]byte // File identification. Type uint16 // File type. Machine uint16 // Machine architecture. @@ -131,6 +136,7 @@ type ElfHeader64 struct { // // +marshal type ElfSection64 struct { + _ structs.HostLayout Name uint32 // Section name (index into the section header string table). Type uint32 // Section type. Flags uint64 // Section flags. @@ -147,6 +153,7 @@ type ElfSection64 struct { // // +marshal type ElfProg64 struct { + _ structs.HostLayout Type uint32 // Entry type. Flags uint32 // Access permission flags. Off uint64 // File offset of contents. diff --git a/pkg/abi/linux/epoll_amd64.go b/pkg/abi/linux/epoll_amd64.go index 7d5b9fd..a87b892 100644 --- a/pkg/abi/linux/epoll_amd64.go +++ b/pkg/abi/linux/epoll_amd64.go @@ -17,10 +17,15 @@ package linux +import ( + "structs" +) + // EpollEvent is equivalent to struct epoll_event from epoll(2). // // +marshal slice:EpollEventSlice type EpollEvent struct { + _ structs.HostLayout Events uint32 // Linux makes struct epoll_event::data a __u64. We represent it as // [2]int32 because, on amd64, Linux also makes struct epoll_event diff --git a/pkg/abi/linux/epoll_arm64.go b/pkg/abi/linux/epoll_arm64.go index 5e5960d..80c8dd8 100644 --- a/pkg/abi/linux/epoll_arm64.go +++ b/pkg/abi/linux/epoll_arm64.go @@ -17,10 +17,15 @@ package linux +import ( + "structs" +) + // EpollEvent is equivalent to struct epoll_event from epoll(2). // // +marshal slice:EpollEventSlice type EpollEvent struct { + _ structs.HostLayout Events uint32 // Linux makes struct epoll_event a __u64, necessitating 4 bytes of padding // here. diff --git a/pkg/abi/linux/errqueue.go b/pkg/abi/linux/errqueue.go index 4f06f8e..d9c7677 100644 --- a/pkg/abi/linux/errqueue.go +++ b/pkg/abi/linux/errqueue.go @@ -15,6 +15,8 @@ package linux import ( + "structs" + "github.com/sagernet/gvisor/pkg/marshal" ) @@ -31,6 +33,7 @@ const ( // // +marshal type SockExtendedErr struct { + _ structs.HostLayout Errno uint32 Origin uint8 Type uint8 @@ -54,6 +57,7 @@ type SockErrCMsg interface { // // +marshal type SockErrCMsgIPv4 struct { + _ structs.HostLayout SockExtendedErr Offender SockAddrInet } @@ -76,6 +80,7 @@ func (*SockErrCMsgIPv4) CMsgType() uint32 { // // +marshal type SockErrCMsgIPv6 struct { + _ structs.HostLayout SockExtendedErr Offender SockAddrInet6 } diff --git a/pkg/abi/linux/fcntl.go b/pkg/abi/linux/fcntl.go index 4853024..15f25f2 100644 --- a/pkg/abi/linux/fcntl.go +++ b/pkg/abi/linux/fcntl.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Commands from linux/fcntl.h. const ( F_DUPFD = 0 @@ -54,6 +58,7 @@ const ( // // +marshal type Flock struct { + _ structs.HostLayout Type int16 Whence int16 _ [4]byte @@ -74,6 +79,7 @@ const ( // // +marshal type FOwnerEx struct { + _ structs.HostLayout Type int32 PID int32 } diff --git a/pkg/abi/linux/file.go b/pkg/abi/linux/file.go index 01a5b59..32f9614 100644 --- a/pkg/abi/linux/file.go +++ b/pkg/abi/linux/file.go @@ -17,6 +17,7 @@ package linux import ( "fmt" "strings" + "structs" "github.com/sagernet/gvisor/pkg/abi" ) @@ -42,6 +43,20 @@ const ( O_TMPFILE = 0o20000000 // __O_TMPFILE in Linux ) +// Constants for file mode (struct file::f_mode in Linux). +const ( + // FMODE_READ indicates the file is open for reading. + // It is set when MayReadFileWithOpenFlags(flags) is true. + FMODE_READ = 0x1 + // FMODE_WRITE indicates the file is open for writing. + // It is set when MayWriteFileWithOpenFlags(flags) is true. When set, + // the FileDescription holds a write count on vd.mount. + FMODE_WRITE = 0x2 + // FMODE_CREATED is set when a file is newly created by an open + // operation (O_CREAT and the file did not already exist). + FMODE_CREATED = 0x100000 +) + // Constants for fstatat(2). const ( AT_SYMLINK_NOFOLLOW = 0x100 @@ -85,6 +100,64 @@ const ( UMOUNT_NOFOLLOW = 0x8 ) +// Constants for fsopen(2). +const ( + FSOPEN_CLOEXEC = 0x1 +) + +// Constants for fsconfig(2). +const ( + FSCONFIG_SET_FLAG = 0x0 + FSCONFIG_SET_STRING = 0x1 + FSCONFIG_SET_BINARY = 0x2 + FSCONFIG_SET_PATH = 0x3 + FSCONFIG_SET_PATH_EMPTY = 0x4 + FSCONFIG_SET_FD = 0x5 + FSCONFIG_CMD_CREATE = 0x6 + FSCONFIG_CMD_RECONFIGURE = 0x7 + FSCONFIG_CMD_CREATE_EXCL = 0x8 +) + +// Constants for fsmount(2). +const ( + FSMOUNT_CLOEXEC = 0x1 +) + +// Constants for move_mount(2). +const ( + MOVE_MOUNT_F_SYMLINKS = 0x00000001 + MOVE_MOUNT_F_AUTOMOUNTS = 0x00000002 + MOVE_MOUNT_F_EMPTY_PATH = 0x00000004 + MOVE_MOUNT_T_SYMLINKS = 0x00000010 + MOVE_MOUNT_T_AUTOMOUNTS = 0x00000020 + MOVE_MOUNT_T_EMPTY_PATH = 0x00000040 + MOVE_MOUNT_SET_GROUP = 0x00000100 + MOVE_MOUNT_BENEATH = 0x00000200 +) + +// Constants for mount_setattr(2). +const ( + MOUNT_ATTR_RDONLY = 0x00000001 + MOUNT_ATTR_NOSUID = 0x00000002 + MOUNT_ATTR_NODEV = 0x00000004 + MOUNT_ATTR_NOEXEC = 0x00000008 + MOUNT_ATTR__ATIME = 0x00000070 + MOUNT_ATTR_RELATIME = 0x00000000 + MOUNT_ATTR_NOATIME = 0x00000010 + MOUNT_ATTR_STRICTATIME = 0x00000020 + MOUNT_ATTR_NODIRATIME = 0x00000080 + MOUNT_ATTR_IDMAP = 0x00100000 + MOUNT_ATTR_NOSYMFOLLOW = 0x00200000 + AT_RECURSIVE = 0x8000 +) + +// Constants for open_tree(2). +const ( + OPEN_TREE_CLONE = (1 << 0) + OPEN_TREE_NAMESPACE = (1 << 1) + OPEN_TREE_CLOEXEC = O_CLOEXEC +) + // Constants for unlinkat(2). const ( AT_REMOVEDIR = 0x200 @@ -265,6 +338,7 @@ const ( STATX_BLOCKS = 0x00000400 STATX_BASIC_STATS = 0x000007ff STATX_BTIME = 0x00000800 + STATX_MNT_ID = 0x00001000 STATX_ALL = 0x00000fff STATX__RESERVED = 0x80000000 ) @@ -284,6 +358,7 @@ const ( // // +marshal boundCheck slice:StatxSlice type Statx struct { + _ structs.HostLayout Mask uint32 Blksize uint32 Attributes uint64 @@ -304,12 +379,13 @@ type Statx struct { RdevMinor uint32 DevMajor uint32 DevMinor uint32 + MntID uint64 } // String implements fmt.Stringer.String. func (s *Statx) String() string { - return fmt.Sprintf("Statx{Mask: %#x, Mode: %s, UID: %d, GID: %d, Ino: %d, DevMajor: %d, DevMinor: %d, Size: %d, Blocks: %d, Blksize: %d, Nlink: %d, Atime: %s, Btime: %s, Ctime: %s, Mtime: %s, Attributes: %d, AttributesMask: %d, RdevMajor: %d, RdevMinor: %d}", - s.Mask, FileMode(s.Mode), s.UID, s.GID, s.Ino, s.DevMajor, s.DevMinor, s.Size, s.Blocks, s.Blksize, s.Nlink, s.Atime.ToTime(), s.Btime.ToTime(), s.Ctime.ToTime(), s.Mtime.ToTime(), s.Attributes, s.AttributesMask, s.RdevMajor, s.RdevMinor) + return fmt.Sprintf("Statx{Mask: %#x, Mode: %s, UID: %d, GID: %d, Ino: %d, DevMajor: %d, DevMinor: %d, Size: %d, Blocks: %d, Blksize: %d, Nlink: %d, Atime: %s, Btime: %s, Ctime: %s, Mtime: %s, Attributes: %d, AttributesMask: %d, RdevMajor: %d, RdevMinor: %d, MntId: %d}", + s.Mask, FileMode(s.Mode), s.UID, s.GID, s.Ino, s.DevMajor, s.DevMinor, s.Size, s.Blocks, s.Blksize, s.Nlink, s.Atime.ToTime(), s.Btime.ToTime(), s.Ctime.ToTime(), s.Mtime.ToTime(), s.Attributes, s.AttributesMask, s.RdevMajor, s.RdevMinor, s.MntID) } // SizeOfStatx is the size of a Statx struct. @@ -340,6 +416,19 @@ func (m FileMode) IsDir() bool { return m.FileType() == S_IFDIR } +// IsSpecialFile returns true if m is the mode of a "special file": a character +// or block device, FIFO, or socket. +// +// Analogous to include/linux/fs.h:special_file(). +func (m FileMode) IsSpecialFile() bool { + switch m.FileType() { + case ModeCharacterDevice, ModeBlockDevice, ModeNamedPipe, ModeSocket: + return true + default: + return false + } +} + // String returns a string representation of m. func (m FileMode) String() string { var s []string diff --git a/pkg/abi/linux/file_amd64.go b/pkg/abi/linux/file_amd64.go index da6bb96..a21da4b 100644 --- a/pkg/abi/linux/file_amd64.go +++ b/pkg/abi/linux/file_amd64.go @@ -17,6 +17,10 @@ package linux +import ( + "structs" +) + // Constants for open(2). const ( O_DIRECT = 0o00040000 @@ -29,6 +33,7 @@ const ( // // +marshal type Stat struct { + _ structs.HostLayout Dev uint64 Ino uint64 Nlink uint64 diff --git a/pkg/abi/linux/file_arm64.go b/pkg/abi/linux/file_arm64.go index 215d96c..fe00362 100644 --- a/pkg/abi/linux/file_arm64.go +++ b/pkg/abi/linux/file_arm64.go @@ -17,6 +17,10 @@ package linux +import ( + "structs" +) + // Constants for open(2). const ( O_DIRECTORY = 0o00040000 @@ -29,6 +33,7 @@ const ( // // +marshal type Stat struct { + _ structs.HostLayout Dev uint64 Ino uint64 Mode uint32 diff --git a/pkg/abi/linux/fs.go b/pkg/abi/linux/fs.go index e1c4a13..a2089c5 100644 --- a/pkg/abi/linux/fs.go +++ b/pkg/abi/linux/fs.go @@ -14,12 +14,20 @@ package linux +import ( + "math" + "structs" + + "github.com/sagernet/gvisor/pkg/hostarch" +) + // Filesystem types used in statfs(2). // // See linux/magic.h. const ( ANON_INODE_FS_MAGIC = 0x09041934 CGROUP_SUPER_MAGIC = 0x27e0eb + CGROUP2_SUPER_MAGIC = 0x63677270 DEVPTS_SUPER_MAGIC = 0x00001cd1 EXT_SUPER_MAGIC = 0xef53 FUSE_SUPER_MAGIC = 0x65735546 @@ -60,6 +68,7 @@ const ( // // +marshal type Statfs struct { + _ structs.HostLayout // Type is one of the filesystem magic values, defined above. Type uint64 @@ -127,3 +136,8 @@ const ( WHITEOUT_MODE = 0 WHITEOUT_DEV = 0 ) + +// MAX_RW_COUNT is the maximum size in bytes of a single read or write. +// Reads and writes that exceed this size may be truncated. +// (Linux: include/linux/fs.h:MAX_RW_COUNT) +var MAX_RW_COUNT = int(hostarch.PageRoundDown(uint32(math.MaxInt32))) diff --git a/pkg/abi/linux/fuse.go b/pkg/abi/linux/fuse.go index 1110579..2e9e45b 100644 --- a/pkg/abi/linux/fuse.go +++ b/pkg/abi/linux/fuse.go @@ -15,6 +15,7 @@ package linux import ( + "structs" "time" "github.com/sagernet/gvisor/pkg/marshal/primitive" @@ -93,6 +94,7 @@ const ( // +marshal // +stateify savable type FUSEHeaderIn struct { + _ structs.HostLayout // Len specifies the total length of the data, including this header. Len uint32 @@ -127,6 +129,7 @@ var SizeOfFUSEHeaderIn = uint32((*FUSEHeaderIn)(nil).SizeBytes()) // +marshal // +stateify savable type FUSEHeaderOut struct { + _ structs.HostLayout // Len specifies the total length of the data, including this header. Len uint32 @@ -172,9 +175,10 @@ const ( // Constants relevant to FUSE operations. const ( - FUSE_NAME_MAX = 1024 - FUSE_PAGE_SIZE = 4096 - FUSE_DIRENT_ALIGN = 8 + FUSE_NAME_MAX = 1024 + FUSE_PAGE_SIZE = 4096 + FUSE_DIRENT_ALIGN = 8 + FUSE_FSYNC_FDATASYNC = 1 << 0 ) // FUSEInitIn is the request sent by the kernel to the daemon, @@ -182,6 +186,7 @@ const ( // // +marshal type FUSEInitIn struct { + _ structs.HostLayout // Major version supported by kernel. Major uint32 @@ -201,6 +206,7 @@ type FUSEInitIn struct { // // +marshal type FUSEInitOut struct { + _ structs.HostLayout // Major version supported by daemon. Major uint32 @@ -249,6 +255,7 @@ type FUSEInitOut struct { // // +marshal type FUSEStatfsOut struct { + _ structs.HostLayout // Blocks is the maximum number of data blocks the filesystem may store, in // units of BlockSize. Blocks uint64 @@ -290,6 +297,7 @@ const FUSE_GETATTR_FH = (1 << 0) // // +marshal type FUSEGetAttrIn struct { + _ structs.HostLayout // GetAttrFlags specifies whether getattr request is sent with a nodeid or // with a file handle. GetAttrFlags uint32 @@ -305,6 +313,7 @@ type FUSEGetAttrIn struct { // // +marshal type FUSEAttr struct { + _ structs.HostLayout // Ino is the inode number of this file. Ino uint64 @@ -376,6 +385,7 @@ func (a FUSEAttr) CTimeNsec() int64 { // // +marshal type FUSEAttrOut struct { + _ structs.HostLayout // AttrValid and AttrValidNsec describe the attribute cache duration AttrValid uint64 @@ -394,6 +404,7 @@ type FUSEAttrOut struct { // // +marshal type FUSEEntryOut struct { + _ structs.HostLayout // NodeID is the ID for current inode. NodeID uint64 @@ -445,6 +456,7 @@ func (s *CString) SizeBytes() int { // // +marshal dynamic type FUSELookupIn struct { + _ structs.HostLayout // Name is a file name to be looked up. Name CString } @@ -475,6 +487,12 @@ const ( FOPEN_KEEP_CACHE = 1 << 1 // FOPEN_NONSEEKABLE indicates the file cannot be seeked. FOPEN_NONSEEKABLE = 1 << 2 + // FOPEN_CACHE_DIR indicated to allow caching this directory + FOPEN_CACHE_DIR = 1 << 3 + // FOPEN_STREAM indicates the file is stream-like (no file position at all) + FOPEN_STREAM = 1 << 4 + // FOPEN_NOFLUSH indicates the file does not need to be flushed on close. + FOPEN_NOFLUSH = 1 << 5 ) // FUSEOpenIn is the request sent by the kernel to the daemon, @@ -482,6 +500,7 @@ const ( // // +marshal type FUSEOpenIn struct { + _ structs.HostLayout // Flags of this open request. Flags uint32 @@ -493,6 +512,7 @@ type FUSEOpenIn struct { // // +marshal type FUSEOpenOut struct { + _ structs.HostLayout // Fh is the file handler for opened files. Fh uint64 @@ -507,6 +527,7 @@ type FUSEOpenOut struct { // // +marshal type FUSECreateOut struct { + _ structs.HostLayout FUSEEntryOut FUSEOpenOut } @@ -521,6 +542,7 @@ const ( // // +marshal type FUSEReadIn struct { + _ structs.HostLayout // Fh is the file handle in userspace. Fh uint64 @@ -553,6 +575,7 @@ type FUSEReadIn struct { // // +marshal type FUSEWriteIn struct { + _ structs.HostLayout // Fh is the file handle in userspace. Fh uint64 @@ -583,8 +606,9 @@ var SizeOfFUSEWriteIn = uint32((*FUSEWriteIn)(nil).SizeBytes()) // // +marshal dynamic type FUSEWritePayloadIn struct { + _ structs.HostLayout Header FUSEWriteIn - Payload primitive.ByteSlice + Payload primitive.ByteSlice `hostlayout:"ignore"` } // SizeBytes implements marshal.Marshallable.SizeBytes. @@ -612,6 +636,7 @@ func (r *FUSEWritePayloadIn) UnmarshalBytes(src []byte) []byte { // // +marshal type FUSEWriteOut struct { + _ structs.HostLayout // Size is the number of bytes written. Size uint32 @@ -623,6 +648,7 @@ type FUSEWriteOut struct { // // +marshal type FUSEReleaseIn struct { + _ structs.HostLayout // Fh is the file handler for the file to be released. Fh uint64 @@ -641,6 +667,7 @@ type FUSEReleaseIn struct { // // +marshal type FUSECreateMeta struct { + _ structs.HostLayout // Flags of the creating file. Flags uint32 @@ -656,6 +683,7 @@ type FUSECreateMeta struct { // // +marshal dynamic type FUSERenameIn struct { + _ structs.HostLayout Newdir primitive.Uint64 Oldname CString Newname CString @@ -683,6 +711,7 @@ func (r *FUSERenameIn) SizeBytes() int { // // +marshal dynamic type FUSECreateIn struct { + _ structs.HostLayout // CreateMeta contains mode, rdev and umash fields for FUSE_MKNODS. CreateMeta FUSECreateMeta @@ -711,6 +740,7 @@ func (r *FUSECreateIn) SizeBytes() int { // // +marshal type FUSEMknodMeta struct { + _ structs.HostLayout // Mode of the inode to create. Mode uint32 @@ -728,6 +758,7 @@ type FUSEMknodMeta struct { // // +marshal dynamic type FUSEMknodIn struct { + _ structs.HostLayout // MknodMeta contains mode, rdev and umash fields for FUSE_MKNODS. MknodMeta FUSEMknodMeta // Name is the name of the node to create. @@ -755,6 +786,7 @@ func (r *FUSEMknodIn) SizeBytes() int { // // +marshal dynamic type FUSESymlinkIn struct { + _ structs.HostLayout // Name of symlink to create. Name CString @@ -782,6 +814,7 @@ func (r *FUSESymlinkIn) SizeBytes() int { // // +marshal dynamic type FUSELinkIn struct { + _ structs.HostLayout // OldNodeID is the ID of the inode that is being linked to. OldNodeID primitive.Uint64 // Name of the new hard link to create. @@ -807,7 +840,9 @@ func (r *FUSELinkIn) SizeBytes() int { // FUSEEmptyIn is used by operations without request body. // // +marshal dynamic -type FUSEEmptyIn struct{} +type FUSEEmptyIn struct { + _ structs.HostLayout +} // MarshalBytes implements marshal.Marshallable.MarshalBytes. func (r *FUSEEmptyIn) MarshalBytes(buf []byte) []byte { @@ -829,6 +864,7 @@ func (r *FUSEEmptyIn) SizeBytes() int { // // +marshal type FUSEMkdirMeta struct { + _ structs.HostLayout // Mode of the directory of create. Mode uint32 // Umask is the user file creation mask. @@ -840,6 +876,7 @@ type FUSEMkdirMeta struct { // // +marshal dynamic type FUSEMkdirIn struct { + _ structs.HostLayout // MkdirMeta contains Mode and Umask of the directory to create. MkdirMeta FUSEMkdirMeta // Name of the directory to create. @@ -867,6 +904,7 @@ func (r *FUSEMkdirIn) SizeBytes() int { // // +marshal dynamic type FUSERmDirIn struct { + _ structs.HostLayout // Name is a directory name to be removed. Name CString } @@ -886,12 +924,94 @@ func (r *FUSERmDirIn) SizeBytes() int { return r.Name.SizeBytes() } +// FUSEGetXattrHdr contains the static fields of FUSEGetXattrIn. +// +// +marshal +type FUSEGetXattrHdr struct { + _ structs.HostLayout + Size uint32 + _ uint32 +} + +// FUSEGetXattrIn contains the arguments for FUSE_GETXATTR. +// +// +marshal dynamic +type FUSEGetXattrIn struct { + _ structs.HostLayout + Hdr FUSEGetXattrHdr + Name CString +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSEGetXattrIn) MarshalBytes(buf []byte) []byte { + buf = r.Hdr.MarshalBytes(buf) + return r.Name.MarshalBytes(buf) +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSEGetXattrIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSEGetXattrIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSEGetXattrIn) SizeBytes() int { + return r.Hdr.SizeBytes() + r.Name.SizeBytes() +} + +// FUSEGetXattrOut is the reply sent by the daemon to the kernel +// for FUSE_GETXATTR and FUSE_LISTXATTR when the input size was 0. +// +// +marshal +type FUSEGetXattrOut struct { + _ structs.HostLayout + Size uint32 + _ uint32 +} + +// FUSESetXattrHdr contains the static fields of FUSESetXattrIn. +// +// +marshal +type FUSESetXattrHdr struct { + _ structs.HostLayout + Size uint32 + Flags uint32 +} + +// FUSESetXattrIn contains the arguments for FUSE_SETXATTR. +// +// +marshal dynamic +type FUSESetXattrIn struct { + _ structs.HostLayout + Hdr FUSESetXattrHdr + Name CString + Value []byte `hostlayout:"ignore"` +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (r *FUSESetXattrIn) MarshalBytes(buf []byte) []byte { + buf = r.Hdr.MarshalBytes(buf) + buf = r.Name.MarshalBytes(buf) + copy(buf, r.Value) + return buf[len(r.Value):] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (r *FUSESetXattrIn) UnmarshalBytes(buf []byte) []byte { + panic("Unimplemented, FUSESetXattrIn is never unmarshalled") +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (r *FUSESetXattrIn) SizeBytes() int { + return r.Hdr.SizeBytes() + r.Name.SizeBytes() + len(r.Value) +} + // FUSEDirents is a list of Dirents received from the FUSE daemon server. // It is used for FUSE_READDIR. // // +marshal dynamic type FUSEDirents struct { - Dirents []*FUSEDirent + _ structs.HostLayout + Dirents []*FUSEDirent `hostlayout:"ignore"` } // FUSEDirent is a Dirent received from the FUSE daemon server. @@ -899,10 +1019,11 @@ type FUSEDirents struct { // // +marshal dynamic type FUSEDirent struct { + _ structs.HostLayout // Meta contains all the static fields of FUSEDirent. Meta FUSEDirentMeta // Name is the filename of the dirent. - Name string + Name string `hostlayout:"ignore"` } // FUSEDirentMeta contains all the static fields of FUSEDirent. @@ -910,6 +1031,7 @@ type FUSEDirent struct { // // +marshal type FUSEDirentMeta struct { + _ structs.HostLayout // Inode of the dirent. Ino uint64 // Offset of the dirent. @@ -937,23 +1059,17 @@ func (r *FUSEDirents) MarshalBytes(buf []byte) []byte { // UnmarshalBytes deserializes FUSEDirents from the src buffer. func (r *FUSEDirents) UnmarshalBytes(src []byte) []byte { - for { - if len(src) <= (*FUSEDirentMeta)(nil).SizeBytes() { + for len(src) >= (*FUSEDirentMeta)(nil).SizeBytes() { + var dirent FUSEDirent + rem := dirent.UnmarshalBytes(src) + if len(rem) == len(src) || len(dirent.Name) == 0 { break } - - // Its unclear how many dirents there are in src. Each dirent is dynamically - // sized and so we can't make assumptions about how many dirents we can allocate. if r.Dirents == nil { r.Dirents = make([]*FUSEDirent, 0) } - - // We have to allocate a struct for each dirent - there must be a better way - // to do this. Linux allocates 1 page to store all the dirents and then - // simply reads them from the page. - var dirent FUSEDirent - src = dirent.UnmarshalBytes(src) r.Dirents = append(r.Dirents, &dirent) + src = rem } return src } @@ -985,12 +1101,18 @@ func (r *FUSEDirent) shiftNextDirent(buf []byte) []byte { // UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. func (r *FUSEDirent) UnmarshalBytes(src []byte) []byte { + if len(src) < (*FUSEDirentMeta)(nil).SizeBytes() { + return src + } srcP := r.Meta.UnmarshalBytes(src) - if r.Meta.NameLen > FUSE_NAME_MAX || r.Meta.NameLen > uint32(len(srcP)) { - // The name is too long and therefore invalid. We don't - // need to unmarshal the name since it'll be thrown away. - return r.shiftNextDirent(src) + // Calculate the 8-byte aligned size of this directory entry record. + recLen := (r.Meta.SizeBytes() + int(r.Meta.NameLen) + (FUSE_DIRENT_ALIGN - 1)) & ^(FUSE_DIRENT_ALIGN - 1) + // If the name is invalid or if the record straddles the end of the source + // buffer (making it incomplete), return the buffer unconsumed. This leaves + // the offset at the start of this entry so it can be re-fetched whole. + if r.Meta.NameLen == 0 || r.Meta.NameLen > FUSE_NAME_MAX || recLen > len(src) { + return src } buf := make([]byte, r.Meta.NameLen) @@ -1021,6 +1143,7 @@ const ( // // +marshal type FUSESetAttrIn struct { + _ structs.HostLayout // Valid indicates which attributes are modified by this request. Valid uint32 @@ -1072,6 +1195,7 @@ type FUSESetAttrIn struct { // // +marshal dynamic type FUSEUnlinkIn struct { + _ structs.HostLayout // Name of the node to unlink. Name CString } @@ -1096,6 +1220,7 @@ func (r *FUSEUnlinkIn) SizeBytes() int { // // +marshal type FUSEFsyncIn struct { + _ structs.HostLayout Fh uint64 FsyncFlags uint32 @@ -1109,6 +1234,7 @@ type FUSEFsyncIn struct { // // +marshal type FUSEAccessIn struct { + _ structs.HostLayout Mask uint32 // padding _ uint32 @@ -1119,6 +1245,7 @@ type FUSEAccessIn struct { // // +marshal type FUSEFallocateIn struct { + _ structs.HostLayout Fh uint64 Offset uint64 Length uint64 @@ -1132,6 +1259,7 @@ type FUSEFallocateIn struct { // // +marshal type FUSEFlushIn struct { + _ structs.HostLayout Fh uint64 _ uint32 // unused _ uint32 // padding diff --git a/pkg/abi/linux/futex.go b/pkg/abi/linux/futex.go index 8138088..8af0398 100644 --- a/pkg/abi/linux/futex.go +++ b/pkg/abi/linux/futex.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // From and . // Flags are used in syscall futex(2). const ( @@ -71,6 +75,7 @@ const ROBUST_LIST_LIMIT = 2048 // // +marshal type RobustListHead struct { + _ structs.HostLayout List uint64 FutexOffset uint64 ListOpPending uint64 diff --git a/pkg/abi/linux/ioctl.go b/pkg/abi/linux/ioctl.go index 6ed8d2a..a97c880 100644 --- a/pkg/abi/linux/ioctl.go +++ b/pkg/abi/linux/ioctl.go @@ -19,9 +19,13 @@ package linux // These are ordered by request number (low byte). const ( TCGETS = 0x00005401 + TCGETS2 = 0x802c542a TCSETS = 0x00005402 + TCSETS2 = 0x402c542b TCSETSW = 0x00005403 + TCSETSW2 = 0x402c542c TCSETSF = 0x00005404 + TCSETSF2 = 0x402c542d TCSBRK = 0x00005409 TIOCEXCL = 0x0000540c TIOCNXCL = 0x0000540d @@ -56,7 +60,7 @@ const ( TIOCCONS = 0x0000541d TIOCSSERIAL = 0x0000541f TIOCGEXCL = 0x80045440 - TIOCGPTPEER = 0x80045441 + TIOCGPTPEER = 0x00005441 TIOCGICOUNT = 0x0000545d FIONCLEX = 0x00005450 FIOCLEX = 0x00005451 @@ -152,6 +156,13 @@ func IOC_SIZE(nr uint32) uint32 { return (nr >> IOC_SIZESHIFT) & ((1 << IOC_SIZEBITS) - 1) } +// TCFLSH queue selector arguments. +const ( + TCIFLUSH = 0 + TCOFLUSH = 1 + TCIOFLUSH = 2 +) + /* Used for packet mode */ const ( TIOCPKT_DATA = 0 @@ -184,3 +195,15 @@ const ( KCOV_MODE_TRACE_PC = 2 KCOV_MODE_TRACE_CMP = 3 ) + +// File clone/dedup ioctls from include/uapi/linux/fs.h. +var ( + FICLONE = IOW(0x94, 9, 4) + FICLONERANGE = IOW(0x94, 13, 32) + FIDEDUPERANGE = IOWR(0x94, 54, 24) +) + +// FUSE_DEV_IOC_CLONE from include/uapi/linux/fuse.h. +var ( + FUSE_DEV_IOC_CLONE = IOR(229, 0, 4) +) diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go index 0eeb62a..c23873c 100644 --- a/pkg/abi/linux/iouring.go +++ b/pkg/abi/linux/iouring.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Constants for io_uring_setup(2). See include/uapi/linux/io_uring.h. const ( IORING_SETUP_IOPOLL = (1 << 0) @@ -75,6 +79,7 @@ type IORingIndex uint32 // // +marshal type IOSqRingOffsets struct { + _ structs.HostLayout Head uint32 // Offset to io_rings.sq.head Tail uint32 // Offset to io_rings.sq.tail RingMask uint32 // Offset to io_rings.sq_ring_mask @@ -92,6 +97,7 @@ type IOSqRingOffsets struct { // // +marshal type IOCqRingOffsets struct { + _ structs.HostLayout Head uint32 // Offset to io_rings.cq.head Tail uint32 // Offset to io_rings.cq.tail RingMask uint32 // Offset to io_rings.cq_ring_mask @@ -108,6 +114,7 @@ type IOCqRingOffsets struct { // // +marshal type IOUringParams struct { + _ structs.HostLayout SqEntries uint32 CqEntries uint32 Flags uint32 @@ -128,6 +135,7 @@ type IOUringParams struct { // +marshal // +stateify savable type IOUringCqe struct { + _ structs.HostLayout UserData uint64 Res int32 Flags uint32 @@ -139,6 +147,7 @@ type IOUringCqe struct { // +marshal // +stateify savable type IOUring struct { + _ structs.HostLayout // Both head and tail should be cacheline aligned. And we assume that // cacheline size is 64 bytes. Head uint32 @@ -154,6 +163,7 @@ type IOUring struct { // +marshal // +stateify savable type IORings struct { + _ structs.HostLayout Sq IOUring Cq IOUring SqRingMask uint32 @@ -177,6 +187,7 @@ type IORings struct { // +marshal // +stateify savable type IOUringSqe struct { + _ structs.HostLayout Opcode uint8 Flags uint8 IoPrio uint16 diff --git a/pkg/abi/linux/ipc.go b/pkg/abi/linux/ipc.go index 57af6e8..0ca900d 100644 --- a/pkg/abi/linux/ipc.go +++ b/pkg/abi/linux/ipc.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Control commands used with semctl, shmctl, and msgctl. // // Source: include/uapi/linux/ipc.h. @@ -46,6 +50,7 @@ const ( // // +marshal type IPCPerm struct { + _ structs.HostLayout Key uint32 UID uint32 GID uint32 diff --git a/pkg/abi/linux/limits.go b/pkg/abi/linux/limits.go index c74dfcd..65817da 100644 --- a/pkg/abi/linux/limits.go +++ b/pkg/abi/linux/limits.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Resources for getrlimit(2)/setrlimit(2)/prlimit(2). const ( RLIMIT_CPU = 0 @@ -36,6 +40,7 @@ const ( // RLimit corresponds to Linux's struct rlimit. type RLimit struct { + _ structs.HostLayout // Cur specifies the soft limit. Cur uint64 // Max specifies the hard limit. @@ -69,20 +74,20 @@ const ( // InitRLimits is a map of initial rlimits set by Linux in // include/asm-generic/resource.h. var InitRLimits = map[int]RLimit{ - RLIMIT_CPU: {RLimInfinity, RLimInfinity}, - RLIMIT_FSIZE: {RLimInfinity, RLimInfinity}, - RLIMIT_DATA: {RLimInfinity, RLimInfinity}, - RLIMIT_STACK: {DefaultStackSoftLimit, RLimInfinity}, - RLIMIT_CORE: {0, RLimInfinity}, - RLIMIT_RSS: {RLimInfinity, RLimInfinity}, - RLIMIT_NPROC: {DefaultNprocLimit, DefaultNprocLimit}, - RLIMIT_NOFILE: {DefaultNofileSoftLimit, DefaultNofileHardLimit}, - RLIMIT_MEMLOCK: {DefaultMemlockLimit, DefaultMemlockLimit}, - RLIMIT_AS: {RLimInfinity, RLimInfinity}, - RLIMIT_LOCKS: {RLimInfinity, RLimInfinity}, - RLIMIT_SIGPENDING: {0, 0}, - RLIMIT_MSGQUEUE: {DefaultMsgqueueLimit, DefaultMsgqueueLimit}, - RLIMIT_NICE: {0, 0}, - RLIMIT_RTPRIO: {0, 0}, - RLIMIT_RTTIME: {RLimInfinity, RLimInfinity}, + RLIMIT_CPU: {Cur: RLimInfinity, Max: RLimInfinity}, + RLIMIT_FSIZE: {Cur: RLimInfinity, Max: RLimInfinity}, + RLIMIT_DATA: {Cur: RLimInfinity, Max: RLimInfinity}, + RLIMIT_STACK: {Cur: DefaultStackSoftLimit, Max: RLimInfinity}, + RLIMIT_CORE: {Cur: 0, Max: RLimInfinity}, + RLIMIT_RSS: {Cur: RLimInfinity, Max: RLimInfinity}, + RLIMIT_NPROC: {Cur: DefaultNprocLimit, Max: DefaultNprocLimit}, + RLIMIT_NOFILE: {Cur: DefaultNofileSoftLimit, Max: DefaultNofileHardLimit}, + RLIMIT_MEMLOCK: {Cur: DefaultMemlockLimit, Max: DefaultMemlockLimit}, + RLIMIT_AS: {Cur: RLimInfinity, Max: RLimInfinity}, + RLIMIT_LOCKS: {Cur: RLimInfinity, Max: RLimInfinity}, + RLIMIT_SIGPENDING: {Cur: 0, Max: 0}, + RLIMIT_MSGQUEUE: {Cur: DefaultMsgqueueLimit, Max: DefaultMsgqueueLimit}, + RLIMIT_NICE: {Cur: 0, Max: 0}, + RLIMIT_RTPRIO: {Cur: 0, Max: 0}, + RLIMIT_RTTIME: {Cur: RLimInfinity, Max: RLimInfinity}, } diff --git a/pkg/abi/linux/linux.go b/pkg/abi/linux/linux.go index 3b4abec..dba15ac 100644 --- a/pkg/abi/linux/linux.go +++ b/pkg/abi/linux/linux.go @@ -16,6 +16,10 @@ // Linux kernel. package linux +import ( + "structs" +) + // NumSoftIRQ is the number of software IRQs, exposed via /proc/stat. // // Defined in linux/interrupt.h. @@ -25,6 +29,7 @@ const NumSoftIRQ = 10 // // +marshal type Sysinfo struct { + _ structs.HostLayout Uptime int64 Loads [3]uint64 TotalRAM uint64 diff --git a/pkg/abi/linux/linux_abi_autogen_unsafe.go b/pkg/abi/linux/linux_abi_autogen_unsafe.go index 20a0129..af3ee06 100644 --- a/pkg/abi/linux/linux_abi_autogen_unsafe.go +++ b/pkg/abi/linux/linux_abi_autogen_unsafe.go @@ -15,6 +15,9 @@ import ( // Marshallable types used by this file. var ( + _ marshal.Marshallable = (*BPFAttrProgAttach)(nil) + _ marshal.Marshallable = (*BPFAttrProgLoad)(nil) + _ marshal.Marshallable = (*BPFAttrProgQuery)(nil) _ marshal.Marshallable = (*BPFInstruction)(nil) _ marshal.Marshallable = (*CString)(nil) _ marshal.Marshallable = (*CapUserData)(nil) @@ -25,6 +28,7 @@ var ( _ marshal.Marshallable = (*ControlMessageHeader)(nil) _ marshal.Marshallable = (*ControlMessageIPPacketInfo)(nil) _ marshal.Marshallable = (*ControlMessageIPv6PacketInfo)(nil) + _ marshal.Marshallable = (*EBPFInstruction)(nil) _ marshal.Marshallable = (*ElfHeader64)(nil) _ marshal.Marshallable = (*ElfProg64)(nil) _ marshal.Marshallable = (*ElfSection64)(nil) @@ -49,6 +53,9 @@ var ( _ marshal.Marshallable = (*FUSEFlushIn)(nil) _ marshal.Marshallable = (*FUSEFsyncIn)(nil) _ marshal.Marshallable = (*FUSEGetAttrIn)(nil) + _ marshal.Marshallable = (*FUSEGetXattrHdr)(nil) + _ marshal.Marshallable = (*FUSEGetXattrIn)(nil) + _ marshal.Marshallable = (*FUSEGetXattrOut)(nil) _ marshal.Marshallable = (*FUSEHeaderIn)(nil) _ marshal.Marshallable = (*FUSEHeaderOut)(nil) _ marshal.Marshallable = (*FUSEInitIn)(nil) @@ -68,6 +75,8 @@ var ( _ marshal.Marshallable = (*FUSERenameIn)(nil) _ marshal.Marshallable = (*FUSERmDirIn)(nil) _ marshal.Marshallable = (*FUSESetAttrIn)(nil) + _ marshal.Marshallable = (*FUSESetXattrHdr)(nil) + _ marshal.Marshallable = (*FUSESetXattrIn)(nil) _ marshal.Marshallable = (*FUSEStatfsOut)(nil) _ marshal.Marshallable = (*FUSESymlinkIn)(nil) _ marshal.Marshallable = (*FUSEUnlinkIn)(nil) @@ -91,6 +100,7 @@ var ( _ marshal.Marshallable = (*IOUringSqe)(nil) _ marshal.Marshallable = (*IP6TEntry)(nil) _ marshal.Marshallable = (*IP6TIP)(nil) + _ marshal.Marshallable = (*IP6TRejectInfo)(nil) _ marshal.Marshallable = (*IP6TReplace)(nil) _ marshal.Marshallable = (*IPCPerm)(nil) _ marshal.Marshallable = (*IPTEntry)(nil) @@ -98,6 +108,7 @@ var ( _ marshal.Marshallable = (*IPTGetinfo)(nil) _ marshal.Marshallable = (*IPTIP)(nil) _ marshal.Marshallable = (*IPTOwnerInfo)(nil) + _ marshal.Marshallable = (*IPTRejectInfo)(nil) _ marshal.Marshallable = (*IPTReplace)(nil) _ marshal.Marshallable = (*Inet6Addr)(nil) _ marshal.Marshallable = (*Inet6MulticastRequest)(nil) @@ -112,6 +123,7 @@ var ( _ marshal.Marshallable = (*KernelIP6TGetEntries)(nil) _ marshal.Marshallable = (*KernelIPTEntry)(nil) _ marshal.Marshallable = (*KernelIPTGetEntries)(nil) + _ marshal.Marshallable = (*KernelTermios)(nil) _ marshal.Marshallable = (*Linger)(nil) _ marshal.Marshallable = (*MqAttr)(nil) _ marshal.Marshallable = (*MsgBuf)(nil) @@ -127,11 +139,14 @@ var ( _ marshal.Marshallable = (*NfNATIPV4Range)(nil) _ marshal.Marshallable = (*NumaPolicy)(nil) _ marshal.Marshallable = (*PollFD)(nil) + _ marshal.Marshallable = (*PosixACLXattr)(nil) + _ marshal.Marshallable = (*PosixACLXattrEntry)(nil) _ marshal.Marshallable = (*RSeqCriticalSection)(nil) _ marshal.Marshallable = (*RobustListHead)(nil) _ marshal.Marshallable = (*RouteMessage)(nil) _ marshal.Marshallable = (*RtAttr)(nil) _ marshal.Marshallable = (*Rusage)(nil) + _ marshal.Marshallable = (*SchedAttr)(nil) _ marshal.Marshallable = (*SeccompData)(nil) _ marshal.Marshallable = (*SeccompNotif)(nil) _ marshal.Marshallable = (*SeccompNotifResp)(nil) @@ -174,6 +189,7 @@ var ( _ marshal.Marshallable = (*Utime)(nil) _ marshal.Marshallable = (*UtsName)(nil) _ marshal.Marshallable = (*VFIODeviceInfo)(nil) + _ marshal.Marshallable = (*VFIODeviceInfoMin)(nil) _ marshal.Marshallable = (*VFIOIommuType1DmaMap)(nil) _ marshal.Marshallable = (*VFIOIommuType1DmaUnmap)(nil) _ marshal.Marshallable = (*VFIOIrqInfo)(nil) @@ -181,13 +197,14 @@ var ( _ marshal.Marshallable = (*VFIORegionInfo)(nil) _ marshal.Marshallable = (*VfsCapData)(nil) _ marshal.Marshallable = (*VfsNsCapData)(nil) - _ marshal.Marshallable = (*WindowSize)(nil) _ marshal.Marshallable = (*Winsize)(nil) + _ marshal.Marshallable = (*XTCTTargetInfoV0)(nil) _ marshal.Marshallable = (*XTCounters)(nil) _ marshal.Marshallable = (*XTEntryMatch)(nil) _ marshal.Marshallable = (*XTEntryTarget)(nil) _ marshal.Marshallable = (*XTErrorTarget)(nil) _ marshal.Marshallable = (*XTGetRevision)(nil) + _ marshal.Marshallable = (*XTMarkMtinfo1)(nil) _ marshal.Marshallable = (*XTMultiport)(nil) _ marshal.Marshallable = (*XTMultiportV1)(nil) _ marshal.Marshallable = (*XTNATTargetV0)(nil) @@ -638,6 +655,56 @@ func UnmarshalUnsafeBPFInstructionSlice(dst []BPFInstruction, src []byte) []byte return src[size*count:] } +// ReadBPFInstructionSlice reads a []BPFInstruction. It returns the number of bytes read +func ReadBPFInstructionSlice(src io.Reader, dst []BPFInstruction) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*BPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteBPFInstructionSlice is like BPFInstruction.WriteTo, but for a []BPFInstruction. +func WriteBPFInstructionSlice(dst io.Writer, src []BPFInstruction) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*BPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (c *CapUserData) SizeBytes() int { return 12 @@ -820,6 +887,56 @@ func UnmarshalUnsafeCapUserDataSlice(dst []CapUserData, src []byte) []byte { return src[size*count:] } +// ReadCapUserDataSlice reads a []CapUserData. It returns the number of bytes read +func ReadCapUserDataSlice(src io.Reader, dst []CapUserData) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*CapUserData)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteCapUserDataSlice is like CapUserData.WriteTo, but for a []CapUserData. +func WriteCapUserDataSlice(dst io.Writer, src []CapUserData) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*CapUserData)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (c *CapUserHeader) SizeBytes() int { return 8 @@ -1308,6 +1425,725 @@ func (c *CloneArgs) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (a *BPFAttrProgAttach) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (a *BPFAttrProgAttach) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Target)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachBPFFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ReplaceBPFFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Relative)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.ExpectedRevision)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (a *BPFAttrProgAttach) UnmarshalBytes(src []byte) []byte { + a.Target = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachBPFFD = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ReplaceBPFFD = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.Relative = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ExpectedRevision = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (a *BPFAttrProgAttach) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (a *BPFAttrProgAttach) MarshalUnsafe(dst []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(a), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (a *BPFAttrProgAttach) UnmarshalUnsafe(src []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(a), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (a *BPFAttrProgAttach) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (a *BPFAttrProgAttach) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyOutN(cc, addr, a.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (a *BPFAttrProgAttach) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (a *BPFAttrProgAttach) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyInN(cc, addr, a.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (a *BPFAttrProgAttach) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (a *BPFAttrProgLoad) SizeBytes() int { + return 152 + + 1*BPF_OBJ_NAME_LEN +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (a *BPFAttrProgLoad) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ProgType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.InstructionCount)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Instructions)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.License)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.LogLevel)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.LogSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.LogBuf)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.KernVersion)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ProgFlags)) + dst = dst[4:] + for idx := 0; idx < BPF_OBJ_NAME_LEN; idx++ { + dst[0] = byte(a.ProgName[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ProgInterfaceIndex)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ExpectedAttachType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ProgBTFFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.FuncInfoRecSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.FuncInfo)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.FuncInfoCount)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.LineInfoRecSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.LineInfo)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.LineInfoCount)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachBTFID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.CoreReloCount)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.FDArray)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.CoreRelos)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.CoreReloRecSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.LogTrueSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.ProgTokenFD)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.FDArrayCount)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Signature)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.SignatureSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.KeyringID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (a *BPFAttrProgLoad) UnmarshalBytes(src []byte) []byte { + a.ProgType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.InstructionCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.Instructions = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.License = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.LogLevel = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.LogSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.LogBuf = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.KernVersion = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ProgFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < BPF_OBJ_NAME_LEN; idx++ { + a.ProgName[idx] = src[0] + src = src[1:] + } + a.ProgInterfaceIndex = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ExpectedAttachType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ProgBTFFD = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.FuncInfoRecSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.FuncInfo = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.FuncInfoCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.LineInfoRecSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.LineInfo = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.LineInfoCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachBTFID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachFD = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.CoreReloCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.FDArray = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.CoreRelos = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.CoreReloRecSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.LogTrueSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ProgTokenFD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.FDArrayCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.Signature = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.SignatureSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.KeyringID = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (a *BPFAttrProgLoad) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (a *BPFAttrProgLoad) MarshalUnsafe(dst []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(a), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (a *BPFAttrProgLoad) UnmarshalUnsafe(src []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(a), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (a *BPFAttrProgLoad) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (a *BPFAttrProgLoad) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyOutN(cc, addr, a.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (a *BPFAttrProgLoad) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (a *BPFAttrProgLoad) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyInN(cc, addr, a.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (a *BPFAttrProgLoad) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (a *BPFAttrProgQuery) SizeBytes() int { + return 64 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (a *BPFAttrProgQuery) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Target)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.QueryFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.AttachFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.ProgIDs)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(a.Count)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.ProgAttachFlags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.LinkIDs)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.LinkAttachFlags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(a.Revision)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (a *BPFAttrProgQuery) UnmarshalBytes(src []byte) []byte { + a.Target = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.QueryFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.AttachFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + a.ProgIDs = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Count = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + a.ProgAttachFlags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.LinkIDs = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.LinkAttachFlags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + a.Revision = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (a *BPFAttrProgQuery) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (a *BPFAttrProgQuery) MarshalUnsafe(dst []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(a), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (a *BPFAttrProgQuery) UnmarshalUnsafe(src []byte) []byte { + size := a.SizeBytes() + gohacks.Memmove(unsafe.Pointer(a), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (a *BPFAttrProgQuery) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (a *BPFAttrProgQuery) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyOutN(cc, addr, a.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (a *BPFAttrProgQuery) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (a *BPFAttrProgQuery) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyInN(cc, addr, a.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (a *BPFAttrProgQuery) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(a))) + hdr.Len = a.SizeBytes() + hdr.Cap = a.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that a + // must live until the use above. + runtime.KeepAlive(a) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (e *EBPFInstruction) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (e *EBPFInstruction) MarshalBytes(dst []byte) []byte { + dst[0] = byte(e.Code) + dst = dst[1:] + dst[0] = byte(e.Registers) + dst = dst[1:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(e.Offset)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(e.Immediate)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (e *EBPFInstruction) UnmarshalBytes(src []byte) []byte { + e.Code = uint8(src[0]) + src = src[1:] + e.Registers = uint8(src[0]) + src = src[1:] + e.Offset = int16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + e.Immediate = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (e *EBPFInstruction) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (e *EBPFInstruction) MarshalUnsafe(dst []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(e), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (e *EBPFInstruction) UnmarshalUnsafe(src []byte) []byte { + size := e.SizeBytes() + gohacks.Memmove(unsafe.Pointer(e), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (e *EBPFInstruction) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (e *EBPFInstruction) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyOutN(cc, addr, e.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (e *EBPFInstruction) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (e *EBPFInstruction) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return e.CopyInN(cc, addr, e.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (e *EBPFInstruction) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(e))) + hdr.Len = e.SizeBytes() + hdr.Cap = e.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that e + // must live until the use above. + runtime.KeepAlive(e) // escapes: replaced by intrinsic. + return int64(length), err +} + +// CopyEBPFInstructionSliceIn copies in a slice of EBPFInstruction objects from the task's memory. +func CopyEBPFInstructionSliceIn(cc marshal.CopyContext, addr hostarch.Addr, dst []EBPFInstruction) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*EBPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyInBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// CopyEBPFInstructionSliceOut copies a slice of EBPFInstruction objects to the task's memory. +func CopyEBPFInstructionSliceOut(cc marshal.CopyContext, addr hostarch.Addr, src []EBPFInstruction) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*EBPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := cc.CopyOutBytes(addr, buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + +// MarshalUnsafeEBPFInstructionSlice is like EBPFInstruction.MarshalUnsafe, but for a []EBPFInstruction. +func MarshalUnsafeEBPFInstructionSlice(src []EBPFInstruction, dst []byte) []byte { + count := len(src) + if count == 0 { + return dst + } + + size := (*EBPFInstruction)(nil).SizeBytes() + buf := dst[:size*count] + gohacks.Memmove(unsafe.Pointer(&buf[0]), unsafe.Pointer(&src[0]), uintptr(len(buf))) + return dst[size*count:] +} + +// UnmarshalUnsafeEBPFInstructionSlice is like EBPFInstruction.UnmarshalUnsafe, but for a []EBPFInstruction. +func UnmarshalUnsafeEBPFInstructionSlice(dst []EBPFInstruction, src []byte) []byte { + count := len(dst) + if count == 0 { + return src + } + + size := (*EBPFInstruction)(nil).SizeBytes() + buf := src[:size*count] + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&buf[0]), uintptr(len(buf))) + return src[size*count:] +} + +// ReadEBPFInstructionSlice reads a []EBPFInstruction. It returns the number of bytes read +func ReadEBPFInstructionSlice(src io.Reader, dst []EBPFInstruction) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*EBPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteEBPFInstructionSlice is like EBPFInstruction.WriteTo, but for a []EBPFInstruction. +func WriteEBPFInstructionSlice(dst io.Writer, src []EBPFInstruction) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*EBPFInstruction)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (e *ElfHeader64) SizeBytes() int { return 48 + @@ -2438,7 +3274,7 @@ func (m *FileMode) WriteTo(writer io.Writer) (int64, error) { // SizeBytes implements marshal.Marshallable.SizeBytes. func (s *Statx) SizeBytes() int { - return 80 + + return 88 + (*StatxTimestamp)(nil).SizeBytes() + (*StatxTimestamp)(nil).SizeBytes() + (*StatxTimestamp)(nil).SizeBytes() + @@ -2483,6 +3319,8 @@ func (s *Statx) MarshalBytes(dst []byte) []byte { dst = dst[4:] hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.DevMinor)) dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.MntID)) + dst = dst[8:] return dst } @@ -2524,6 +3362,8 @@ func (s *Statx) UnmarshalBytes(src []byte) []byte { src = src[4:] s.DevMinor = uint32(hostarch.ByteOrder.Uint32(src[:4])) src = src[4:] + s.MntID = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] return src } @@ -2778,6 +3618,86 @@ func UnmarshalUnsafeStatxSlice(dst []Statx, src []byte) []byte { return src[size*count:] } +// ReadStatxSlice reads a []Statx. It returns the number of bytes read +func ReadStatxSlice(src io.Reader, dst []Statx) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Statx)(nil).SizeBytes() + + if !dst[0].Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := make([]byte, size) + length := 0 + for idx := 0; idx < count; idx++ { + n, err := io.ReadFull(src, buf) + length += n + if err != nil { + return length, err + } + dst[idx].UnmarshalBytes(buf) + } + return length, nil + } + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteStatxSlice is like Statx.WriteTo, but for a []Statx. +func WriteStatxSlice(dst io.Writer, src []Statx) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Statx)(nil).SizeBytes() + + if !src[0].Packed() { + // Type Statx doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, size) + length := 0 + for idx := 0; idx < count; idx++ { + src[idx].MarshalBytes(buf) + n, err := dst.Write(buf) + length += n + if err != nil { + return length, err + } + } + return length, nil + } + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (s *Statfs) SizeBytes() int { return 80 + @@ -4585,6 +5505,272 @@ func (f *FUSEGetAttrIn) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEGetXattrHdr) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEGetXattrHdr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Size)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEGetXattrHdr) UnmarshalBytes(src []byte) []byte { + f.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEGetXattrHdr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEGetXattrHdr) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEGetXattrHdr) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEGetXattrHdr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEGetXattrHdr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEGetXattrHdr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEGetXattrHdr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEGetXattrHdr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSEGetXattrIn) Packed() bool { + // Type FUSEGetXattrIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSEGetXattrIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSEGetXattrIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSEGetXattrIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSEGetXattrIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSEGetXattrIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEGetXattrIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSEGetXattrIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSEGetXattrIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSEGetXattrIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSEGetXattrIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSEGetXattrIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSEGetXattrIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSEGetXattrOut) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSEGetXattrOut) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Size)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSEGetXattrOut) UnmarshalBytes(src []byte) []byte { + f.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSEGetXattrOut) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSEGetXattrOut) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSEGetXattrOut) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSEGetXattrOut) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSEGetXattrOut) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSEGetXattrOut) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSEGetXattrOut) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSEGetXattrOut) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (f *FUSEHeaderIn) SizeBytes() int { return 28 + @@ -6509,6 +7695,170 @@ func (f *FUSESetAttrIn) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (f *FUSESetXattrHdr) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (f *FUSESetXattrHdr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(f.Flags)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (f *FUSESetXattrHdr) UnmarshalBytes(src []byte) []byte { + f.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + f.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (f *FUSESetXattrHdr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (f *FUSESetXattrHdr) MarshalUnsafe(dst []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(f), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (f *FUSESetXattrHdr) UnmarshalUnsafe(src []byte) []byte { + size := f.SizeBytes() + gohacks.Memmove(unsafe.Pointer(f), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (f *FUSESetXattrHdr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (f *FUSESetXattrHdr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyOutN(cc, addr, f.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (f *FUSESetXattrHdr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (f *FUSESetXattrHdr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return f.CopyInN(cc, addr, f.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (f *FUSESetXattrHdr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(f))) + hdr.Len = f.SizeBytes() + hdr.Cap = f.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that f + // must live until the use above. + runtime.KeepAlive(f) // escapes: replaced by intrinsic. + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (r *FUSESetXattrIn) Packed() bool { + // Type FUSESetXattrIn is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (r *FUSESetXattrIn) MarshalUnsafe(dst []byte) []byte { + // Type FUSESetXattrIn doesn't have a packed layout in memory, fallback to MarshalBytes. + return r.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (r *FUSESetXattrIn) UnmarshalUnsafe(src []byte) []byte { + // Type FUSESetXattrIn doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return r.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (r *FUSESetXattrIn) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSESetXattrIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + r.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (r *FUSESetXattrIn) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyOutN(cc, addr, r.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (r *FUSESetXattrIn) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type FUSESetXattrIn doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(r.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + r.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (r *FUSESetXattrIn) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return r.CopyInN(cc, addr, r.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (r *FUSESetXattrIn) WriteTo(writer io.Writer) (int64, error) { + // Type FUSESetXattrIn doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, r.SizeBytes()) + r.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (f *FUSEStatfsOut) SizeBytes() int { return 56 + @@ -9790,6 +11140,104 @@ func (en *ExtensionName) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IP6TRejectInfo) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IP6TRejectInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.With)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IP6TRejectInfo) UnmarshalBytes(src []byte) []byte { + i.With = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IP6TRejectInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IP6TRejectInfo) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IP6TRejectInfo) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IP6TRejectInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IP6TRejectInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IP6TRejectInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IP6TRejectInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IP6TRejectInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (i *IPTEntry) SizeBytes() int { return 12 + @@ -10530,6 +11978,104 @@ func (i *IPTOwnerInfo) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IPTRejectInfo) SizeBytes() int { + return 4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IPTRejectInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.With)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IPTRejectInfo) UnmarshalBytes(src []byte) []byte { + i.With = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IPTRejectInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IPTRejectInfo) MarshalUnsafe(dst []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IPTRejectInfo) UnmarshalUnsafe(src []byte) []byte { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IPTRejectInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IPTRejectInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IPTRejectInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IPTRejectInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IPTRejectInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (i *IPTReplace) SizeBytes() int { return 24 + @@ -11185,6 +12731,171 @@ func (tn *TableName) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTCTTargetInfoV0) SizeBytes() int { + return 12 + + (*XTEntryTarget)(nil).SizeBytes() + + 1*16 + + 1*4 + + 1*8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTCTTargetInfoV0) MarshalBytes(dst []byte) []byte { + dst = x.Target.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.Flags)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(x.Zone)) + dst = dst[2:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.CTEvents)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.ExpEvents)) + dst = dst[4:] + for idx := 0; idx < 16; idx++ { + dst[0] = byte(x.Helper[idx]) + dst = dst[1:] + } + // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} + dst = dst[1*(4):] + // Padding: dst[:sizeof(byte)*8] ~= [8]byte{0} + dst = dst[1*(8):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTCTTargetInfoV0) UnmarshalBytes(src []byte) []byte { + src = x.Target.UnmarshalUnsafe(src) + x.Flags = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.Zone = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + x.CTEvents = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.ExpEvents = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 16; idx++ { + x.Helper[idx] = src[0] + src = src[1:] + } + // Padding: ~ copy([4]byte(x._), src[:sizeof(byte)*4]) + src = src[1*(4):] + // Padding: ~ copy([8]byte(x._), src[:sizeof(byte)*8]) + src = src[1*(8):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTCTTargetInfoV0) Packed() bool { + return x.Target.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTCTTargetInfoV0) MarshalUnsafe(dst []byte) []byte { + if x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] + } + // Type XTCTTargetInfoV0 doesn't have a packed layout in memory, fallback to MarshalBytes. + return x.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTCTTargetInfoV0) UnmarshalUnsafe(src []byte) []byte { + if x.Target.Packed() { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type XTCTTargetInfoV0 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return x.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTCTTargetInfoV0) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Target.Packed() { + // Type XTCTTargetInfoV0 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + x.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTCTTargetInfoV0) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTCTTargetInfoV0) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !x.Target.Packed() { + // Type XTCTTargetInfoV0 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(x.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + x.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTCTTargetInfoV0) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTCTTargetInfoV0) WriteTo(writer io.Writer) (int64, error) { + if !x.Target.Packed() { + // Type XTCTTargetInfoV0 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, x.SizeBytes()) + x.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (x *XTCounters) SizeBytes() int { return 16 @@ -11835,6 +13546,117 @@ func (x *XTGetRevision) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (x *XTMarkMtinfo1) SizeBytes() int { + return 9 + + 1*3 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (x *XTMarkMtinfo1) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.Mark)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(x.Mask)) + dst = dst[4:] + dst[0] = byte(x.Invert) + dst = dst[1:] + // Padding: dst[:sizeof(byte)*3] ~= [3]byte{0} + dst = dst[1*(3):] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (x *XTMarkMtinfo1) UnmarshalBytes(src []byte) []byte { + x.Mark = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.Mask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + x.Invert = uint8(src[0]) + src = src[1:] + // Padding: ~ copy([3]byte(x._), src[:sizeof(byte)*3]) + src = src[1*(3):] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (x *XTMarkMtinfo1) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (x *XTMarkMtinfo1) MarshalUnsafe(dst []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(x), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (x *XTMarkMtinfo1) UnmarshalUnsafe(src []byte) []byte { + size := x.SizeBytes() + gohacks.Memmove(unsafe.Pointer(x), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (x *XTMarkMtinfo1) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (x *XTMarkMtinfo1) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyOutN(cc, addr, x.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (x *XTMarkMtinfo1) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (x *XTMarkMtinfo1) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return x.CopyInN(cc, addr, x.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (x *XTMarkMtinfo1) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(x))) + hdr.Len = x.SizeBytes() + hdr.Cap = x.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that x + // must live until the use above. + runtime.KeepAlive(x) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (x *XTMultiport) SizeBytes() int { return 2 + @@ -15271,6 +17093,56 @@ func UnmarshalUnsafePollFDSlice(dst []PollFD, src []byte) []byte { return src[size*count:] } +// ReadPollFDSlice reads a []PollFD. It returns the number of bytes read +func ReadPollFDSlice(src io.Reader, dst []PollFD) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*PollFD)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WritePollFDSlice is like PollFD.WriteTo, but for a []PollFD. +func WritePollFDSlice(dst io.Writer, src []PollFD) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*PollFD)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (r *RSeqCriticalSection) SizeBytes() int { return 32 @@ -15574,6 +17446,140 @@ func (r *Rusage) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (s *SchedAttr) SizeBytes() int { + return 56 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (s *SchedAttr) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SchedPolicy)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SchedFlags)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SchedNice)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SchedPriority)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SchedRuntime)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SchedDeadline)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(s.SchedPeriod)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SchedUtilMin)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(s.SchedUtilMax)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (s *SchedAttr) UnmarshalBytes(src []byte) []byte { + s.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SchedPolicy = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SchedFlags = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SchedNice = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SchedPriority = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SchedRuntime = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SchedDeadline = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SchedPeriod = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + s.SchedUtilMin = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + s.SchedUtilMax = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (s *SchedAttr) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (s *SchedAttr) MarshalUnsafe(dst []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(s), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (s *SchedAttr) UnmarshalUnsafe(src []byte) []byte { + size := s.SizeBytes() + gohacks.Memmove(unsafe.Pointer(s), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (s *SchedAttr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (s *SchedAttr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyOutN(cc, addr, s.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (s *SchedAttr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (s *SchedAttr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return s.CopyInN(cc, addr, s.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (s *SchedAttr) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(s))) + hdr.Len = s.SizeBytes() + hdr.Cap = s.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that s + // must live until the use above. + runtime.KeepAlive(s) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (sd *SeccompData) SizeBytes() int { return 16 + @@ -16363,6 +18369,56 @@ func UnmarshalUnsafeSembufSlice(dst []Sembuf, src []byte) []byte { return src[size*count:] } +// ReadSembufSlice reads a []Sembuf. It returns the number of bytes read +func ReadSembufSlice(src io.Reader, dst []Sembuf) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Sembuf)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteSembufSlice is like Sembuf.WriteTo, but for a []Sembuf. +func WriteSembufSlice(dst io.Writer, src []Sembuf) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Sembuf)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (s *ShmInfo) SizeBytes() int { return 44 + @@ -20301,120 +22357,120 @@ func (i *ItimerVal) WriteTo(writer io.Writer) (int64, error) { } // SizeBytes implements marshal.Marshallable.SizeBytes. -func (i *Itimerspec) SizeBytes() int { +func (its *Itimerspec) SizeBytes() int { return 0 + (*Timespec)(nil).SizeBytes() + (*Timespec)(nil).SizeBytes() } // MarshalBytes implements marshal.Marshallable.MarshalBytes. -func (i *Itimerspec) MarshalBytes(dst []byte) []byte { - dst = i.Interval.MarshalUnsafe(dst) - dst = i.Value.MarshalUnsafe(dst) +func (its *Itimerspec) MarshalBytes(dst []byte) []byte { + dst = its.Interval.MarshalUnsafe(dst) + dst = its.Value.MarshalUnsafe(dst) return dst } // UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. -func (i *Itimerspec) UnmarshalBytes(src []byte) []byte { - src = i.Interval.UnmarshalUnsafe(src) - src = i.Value.UnmarshalUnsafe(src) +func (its *Itimerspec) UnmarshalBytes(src []byte) []byte { + src = its.Interval.UnmarshalUnsafe(src) + src = its.Value.UnmarshalUnsafe(src) return src } // Packed implements marshal.Marshallable.Packed. // //go:nosplit -func (i *Itimerspec) Packed() bool { - return i.Interval.Packed() && i.Value.Packed() +func (its *Itimerspec) Packed() bool { + return its.Interval.Packed() && its.Value.Packed() } // MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. -func (i *Itimerspec) MarshalUnsafe(dst []byte) []byte { - if i.Interval.Packed() && i.Value.Packed() { - size := i.SizeBytes() - gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) +func (its *Itimerspec) MarshalUnsafe(dst []byte) []byte { + if its.Interval.Packed() && its.Value.Packed() { + size := its.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(its), uintptr(size)) return dst[size:] } // Type Itimerspec doesn't have a packed layout in memory, fallback to MarshalBytes. - return i.MarshalBytes(dst) + return its.MarshalBytes(dst) } // UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. -func (i *Itimerspec) UnmarshalUnsafe(src []byte) []byte { - if i.Interval.Packed() && i.Value.Packed() { - size := i.SizeBytes() - gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) +func (its *Itimerspec) UnmarshalUnsafe(src []byte) []byte { + if its.Interval.Packed() && its.Value.Packed() { + size := its.SizeBytes() + gohacks.Memmove(unsafe.Pointer(its), unsafe.Pointer(&src[0]), uintptr(size)) return src[size:] } // Type Itimerspec doesn't have a packed layout in memory, fallback to UnmarshalBytes. - return i.UnmarshalBytes(src) + return its.UnmarshalBytes(src) } // CopyOutN implements marshal.Marshallable.CopyOutN. -func (i *Itimerspec) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { - if !i.Interval.Packed() && i.Value.Packed() { +func (its *Itimerspec) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !its.Interval.Packed() && its.Value.Packed() { // Type Itimerspec doesn't have a packed layout in memory, fall back to MarshalBytes. - buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. - i.MarshalBytes(buf) // escapes: fallback. - return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + buf := cc.CopyScratchBuffer(its.SizeBytes()) // escapes: okay. + its.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. } // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) - hdr.Len = i.SizeBytes() - hdr.Cap = i.SizeBytes() + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(its))) + hdr.Len = its.SizeBytes() + hdr.Cap = its.SizeBytes() length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. - // Since we bypassed the compiler's escape analysis, indicate that i + // Since we bypassed the compiler's escape analysis, indicate that its // must live until the use above. - runtime.KeepAlive(i) // escapes: replaced by intrinsic. + runtime.KeepAlive(its) // escapes: replaced by intrinsic. return length, err } // CopyOut implements marshal.Marshallable.CopyOut. -func (i *Itimerspec) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { - return i.CopyOutN(cc, addr, i.SizeBytes()) +func (its *Itimerspec) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return its.CopyOutN(cc, addr, its.SizeBytes()) } // CopyInN implements marshal.Marshallable.CopyInN. -func (i *Itimerspec) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { - if !i.Interval.Packed() && i.Value.Packed() { +func (its *Itimerspec) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !its.Interval.Packed() && its.Value.Packed() { // Type Itimerspec doesn't have a packed layout in memory, fall back to UnmarshalBytes. - buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + buf := cc.CopyScratchBuffer(its.SizeBytes()) // escapes: okay. length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. // Unmarshal unconditionally. If we had a short copy-in, this results in a // partially unmarshalled struct. - i.UnmarshalBytes(buf) // escapes: fallback. + its.UnmarshalBytes(buf) // escapes: fallback. return length, err } // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) - hdr.Len = i.SizeBytes() - hdr.Cap = i.SizeBytes() + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(its))) + hdr.Len = its.SizeBytes() + hdr.Cap = its.SizeBytes() length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. - // Since we bypassed the compiler's escape analysis, indicate that i + // Since we bypassed the compiler's escape analysis, indicate that its // must live until the use above. - runtime.KeepAlive(i) // escapes: replaced by intrinsic. + runtime.KeepAlive(its) // escapes: replaced by intrinsic. return length, err } // CopyIn implements marshal.Marshallable.CopyIn. -func (i *Itimerspec) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { - return i.CopyInN(cc, addr, i.SizeBytes()) +func (its *Itimerspec) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return its.CopyInN(cc, addr, its.SizeBytes()) } // WriteTo implements io.WriterTo.WriteTo. -func (i *Itimerspec) WriteTo(writer io.Writer) (int64, error) { - if !i.Interval.Packed() && i.Value.Packed() { +func (its *Itimerspec) WriteTo(writer io.Writer) (int64, error) { + if !its.Interval.Packed() && its.Value.Packed() { // Type Itimerspec doesn't have a packed layout in memory, fall back to MarshalBytes. - buf := make([]byte, i.SizeBytes()) - i.MarshalBytes(buf) + buf := make([]byte, its.SizeBytes()) + its.MarshalBytes(buf) length, err := writer.Write(buf) return int64(length), err } @@ -20422,14 +22478,14 @@ func (i *Itimerspec) WriteTo(writer io.Writer) (int64, error) { // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) - hdr.Len = i.SizeBytes() - hdr.Cap = i.SizeBytes() + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(its))) + hdr.Len = its.SizeBytes() + hdr.Cap = its.SizeBytes() length, err := writer.Write(buf) - // Since we bypassed the compiler's escape analysis, indicate that i + // Since we bypassed the compiler's escape analysis, indicate that its // must live until the use above. - runtime.KeepAlive(i) // escapes: replaced by intrinsic. + runtime.KeepAlive(its) // escapes: replaced by intrinsic. return int64(length), err } @@ -20915,6 +22971,56 @@ func UnmarshalUnsafeTimespecSlice(dst []Timespec, src []byte) []byte { return src[size*count:] } +// ReadTimespecSlice reads a []Timespec. It returns the number of bytes read +func ReadTimespecSlice(src io.Reader, dst []Timespec) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Timespec)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteTimespecSlice is like Timespec.WriteTo, but for a []Timespec. +func WriteTimespecSlice(dst io.Writer, src []Timespec) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Timespec)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (tv *Timeval) SizeBytes() int { return 16 @@ -21093,6 +23199,56 @@ func UnmarshalUnsafeTimevalSlice(dst []Timeval, src []byte) []byte { return src[size*count:] } +// ReadTimevalSlice reads a []Timeval. It returns the number of bytes read +func ReadTimevalSlice(src io.Reader, dst []Timeval) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*Timeval)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteTimevalSlice is like Timeval.WriteTo, but for a []Timeval. +func WriteTimevalSlice(dst io.Writer, src []Timeval) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*Timeval)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (t *Tms) SizeBytes() int { return 0 + @@ -21334,6 +23490,137 @@ func (u *Utime) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (t *KernelTermios) SizeBytes() int { + return 25 + + 1*NumControlCharacters +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (t *KernelTermios) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.InputFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.OutputFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.ControlFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.LocalFlags)) + dst = dst[4:] + dst[0] = byte(t.LineDiscipline) + dst = dst[1:] + for idx := 0; idx < NumControlCharacters; idx++ { + dst[0] = byte(t.ControlCharacters[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.InputSpeed)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(t.OutputSpeed)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (t *KernelTermios) UnmarshalBytes(src []byte) []byte { + t.InputFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.OutputFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.ControlFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LocalFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.LineDiscipline = uint8(src[0]) + src = src[1:] + for idx := 0; idx < NumControlCharacters; idx++ { + t.ControlCharacters[idx] = uint8(src[0]) + src = src[1:] + } + t.InputSpeed = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + t.OutputSpeed = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (t *KernelTermios) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (t *KernelTermios) MarshalUnsafe(dst []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(t), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (t *KernelTermios) UnmarshalUnsafe(src []byte) []byte { + size := t.SizeBytes() + gohacks.Memmove(unsafe.Pointer(t), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (t *KernelTermios) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (t *KernelTermios) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyOutN(cc, addr, t.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (t *KernelTermios) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (t *KernelTermios) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return t.CopyInN(cc, addr, t.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (t *KernelTermios) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(t))) + hdr.Len = t.SizeBytes() + hdr.Cap = t.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that t + // must live until the use above. + runtime.KeepAlive(t) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (t *Termios) SizeBytes() int { return 17 + @@ -21457,113 +23744,6 @@ func (t *Termios) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } -// SizeBytes implements marshal.Marshallable.SizeBytes. -func (w *WindowSize) SizeBytes() int { - return 4 + - 1*4 -} - -// MarshalBytes implements marshal.Marshallable.MarshalBytes. -func (w *WindowSize) MarshalBytes(dst []byte) []byte { - hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Rows)) - dst = dst[2:] - hostarch.ByteOrder.PutUint16(dst[:2], uint16(w.Cols)) - dst = dst[2:] - // Padding: dst[:sizeof(byte)*4] ~= [4]byte{0} - dst = dst[1*(4):] - return dst -} - -// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. -func (w *WindowSize) UnmarshalBytes(src []byte) []byte { - w.Rows = uint16(hostarch.ByteOrder.Uint16(src[:2])) - src = src[2:] - w.Cols = uint16(hostarch.ByteOrder.Uint16(src[:2])) - src = src[2:] - // Padding: ~ copy([4]byte(w._), src[:sizeof(byte)*4]) - src = src[1*(4):] - return src -} - -// Packed implements marshal.Marshallable.Packed. -// -//go:nosplit -func (w *WindowSize) Packed() bool { - return true -} - -// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. -func (w *WindowSize) MarshalUnsafe(dst []byte) []byte { - size := w.SizeBytes() - gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(w), uintptr(size)) - return dst[size:] -} - -// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. -func (w *WindowSize) UnmarshalUnsafe(src []byte) []byte { - size := w.SizeBytes() - gohacks.Memmove(unsafe.Pointer(w), unsafe.Pointer(&src[0]), uintptr(size)) - return src[size:] -} - -// CopyOutN implements marshal.Marshallable.CopyOutN. -func (w *WindowSize) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { - // Construct a slice backed by dst's underlying memory. - var buf []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) - hdr.Len = w.SizeBytes() - hdr.Cap = w.SizeBytes() - - length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. - // Since we bypassed the compiler's escape analysis, indicate that w - // must live until the use above. - runtime.KeepAlive(w) // escapes: replaced by intrinsic. - return length, err -} - -// CopyOut implements marshal.Marshallable.CopyOut. -func (w *WindowSize) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { - return w.CopyOutN(cc, addr, w.SizeBytes()) -} - -// CopyInN implements marshal.Marshallable.CopyInN. -func (w *WindowSize) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { - // Construct a slice backed by dst's underlying memory. - var buf []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) - hdr.Len = w.SizeBytes() - hdr.Cap = w.SizeBytes() - - length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. - // Since we bypassed the compiler's escape analysis, indicate that w - // must live until the use above. - runtime.KeepAlive(w) // escapes: replaced by intrinsic. - return length, err -} - -// CopyIn implements marshal.Marshallable.CopyIn. -func (w *WindowSize) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { - return w.CopyInN(cc, addr, w.SizeBytes()) -} - -// WriteTo implements io.WriterTo.WriteTo. -func (w *WindowSize) WriteTo(writer io.Writer) (int64, error) { - // Construct a slice backed by dst's underlying memory. - var buf []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(w))) - hdr.Len = w.SizeBytes() - hdr.Cap = w.SizeBytes() - - length, err := writer.Write(buf) - // Since we bypassed the compiler's escape analysis, indicate that w - // must live until the use above. - runtime.KeepAlive(w) // escapes: replaced by intrinsic. - return int64(length), err -} - // SizeBytes implements marshal.Marshallable.SizeBytes. func (w *Winsize) SizeBytes() int { return 8 @@ -21824,19 +24004,13 @@ func (u *UtsName) WriteTo(writer io.Writer) (int64, error) { // SizeBytes implements marshal.Marshallable.SizeBytes. func (v *VFIODeviceInfo) SizeBytes() int { - return 24 + return 8 + + (*VFIODeviceInfoMin)(nil).SizeBytes() } // MarshalBytes implements marshal.Marshallable.MarshalBytes. func (v *VFIODeviceInfo) MarshalBytes(dst []byte) []byte { - hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) - dst = dst[4:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) - dst = dst[4:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.NumRegions)) - dst = dst[4:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.NumIrqs)) - dst = dst[4:] + dst = v.VFIODeviceInfoMin.MarshalUnsafe(dst) hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.CapOffset)) dst = dst[4:] hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.pad)) @@ -21846,14 +24020,7 @@ func (v *VFIODeviceInfo) MarshalBytes(dst []byte) []byte { // UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. func (v *VFIODeviceInfo) UnmarshalBytes(src []byte) []byte { - v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] - v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] - v.NumRegions = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] - v.NumIrqs = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] + src = v.VFIODeviceInfoMin.UnmarshalUnsafe(src) v.CapOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) src = src[4:] v.pad = uint32(hostarch.ByteOrder.Uint32(src[:4])) @@ -21865,25 +24032,40 @@ func (v *VFIODeviceInfo) UnmarshalBytes(src []byte) []byte { // //go:nosplit func (v *VFIODeviceInfo) Packed() bool { - return true + return v.VFIODeviceInfoMin.Packed() } // MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. func (v *VFIODeviceInfo) MarshalUnsafe(dst []byte) []byte { - size := v.SizeBytes() - gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) - return dst[size:] + if v.VFIODeviceInfoMin.Packed() { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] + } + // Type VFIODeviceInfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return v.MarshalBytes(dst) } // UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. func (v *VFIODeviceInfo) UnmarshalUnsafe(src []byte) []byte { - size := v.SizeBytes() - gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) - return src[size:] + if v.VFIODeviceInfoMin.Packed() { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type VFIODeviceInfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return v.UnmarshalBytes(src) } // CopyOutN implements marshal.Marshallable.CopyOutN. func (v *VFIODeviceInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !v.VFIODeviceInfoMin.Packed() { + // Type VFIODeviceInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(v.SizeBytes()) // escapes: okay. + v.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) @@ -21905,6 +24087,16 @@ func (v *VFIODeviceInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (in // CopyInN implements marshal.Marshallable.CopyInN. func (v *VFIODeviceInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !v.VFIODeviceInfoMin.Packed() { + // Type VFIODeviceInfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(v.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + v.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) @@ -21926,6 +24118,124 @@ func (v *VFIODeviceInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int // WriteTo implements io.WriterTo.WriteTo. func (v *VFIODeviceInfo) WriteTo(writer io.Writer) (int64, error) { + if !v.VFIODeviceInfoMin.Packed() { + // Type VFIODeviceInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, v.SizeBytes()) + v.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (v *VFIODeviceInfoMin) SizeBytes() int { + return 16 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (v *VFIODeviceInfoMin) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Argsz)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.NumRegions)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.NumIrqs)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (v *VFIODeviceInfoMin) UnmarshalBytes(src []byte) []byte { + v.Argsz = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.NumRegions = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + v.NumIrqs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (v *VFIODeviceInfoMin) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (v *VFIODeviceInfoMin) MarshalUnsafe(dst []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(v), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (v *VFIODeviceInfoMin) UnmarshalUnsafe(src []byte) []byte { + size := v.SizeBytes() + gohacks.Memmove(unsafe.Pointer(v), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (v *VFIODeviceInfoMin) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (v *VFIODeviceInfoMin) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyOutN(cc, addr, v.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (v *VFIODeviceInfoMin) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(v))) + hdr.Len = v.SizeBytes() + hdr.Cap = v.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that v + // must live until the use above. + runtime.KeepAlive(v) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (v *VFIODeviceInfoMin) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return v.CopyInN(cc, addr, v.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (v *VFIODeviceInfoMin) WriteTo(writer io.Writer) (int64, error) { // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) @@ -22401,7 +24711,7 @@ func (v *VFIORegionInfo) MarshalBytes(dst []byte) []byte { dst = dst[4:] hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.Index)) dst = dst[4:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.capOffset)) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(v.CapOffset)) dst = dst[4:] hostarch.ByteOrder.PutUint64(dst[:8], uint64(v.Size)) dst = dst[8:] @@ -22418,7 +24728,7 @@ func (v *VFIORegionInfo) UnmarshalBytes(src []byte) []byte { src = src[4:] v.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) src = src[4:] - v.capOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) + v.CapOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) src = src[4:] v.Size = uint64(hostarch.ByteOrder.Uint64(src[:8])) src = src[8:] @@ -22505,3 +24815,127 @@ func (v *VFIORegionInfo) WriteTo(writer io.Writer) (int64, error) { runtime.KeepAlive(v) // escapes: replaced by intrinsic. return int64(length), err } + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (a *PosixACLXattr) Packed() bool { + // Type PosixACLXattr is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (a *PosixACLXattr) MarshalUnsafe(dst []byte) []byte { + // Type PosixACLXattr doesn't have a packed layout in memory, fallback to MarshalBytes. + return a.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (a *PosixACLXattr) UnmarshalUnsafe(src []byte) []byte { + // Type PosixACLXattr doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return a.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (a *PosixACLXattr) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type PosixACLXattr doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(a.SizeBytes()) // escapes: okay. + a.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (a *PosixACLXattr) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyOutN(cc, addr, a.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (a *PosixACLXattr) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type PosixACLXattr doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(a.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + a.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (a *PosixACLXattr) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyInN(cc, addr, a.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (a *PosixACLXattr) WriteTo(writer io.Writer) (int64, error) { + // Type PosixACLXattr doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, a.SizeBytes()) + a.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (a *PosixACLXattrEntry) Packed() bool { + // Type PosixACLXattrEntry is dynamic so it might have slice/string headers. Hence, it is not packed. + return false +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (a *PosixACLXattrEntry) MarshalUnsafe(dst []byte) []byte { + // Type PosixACLXattrEntry doesn't have a packed layout in memory, fallback to MarshalBytes. + return a.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (a *PosixACLXattrEntry) UnmarshalUnsafe(src []byte) []byte { + // Type PosixACLXattrEntry doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return a.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +// +//go:nosplit +func (a *PosixACLXattrEntry) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type PosixACLXattrEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(a.SizeBytes()) // escapes: okay. + a.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (a *PosixACLXattrEntry) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyOutN(cc, addr, a.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +// +//go:nosplit +func (a *PosixACLXattrEntry) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Type PosixACLXattrEntry doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(a.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + a.UnmarshalBytes(buf) // escapes: fallback. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (a *PosixACLXattrEntry) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return a.CopyInN(cc, addr, a.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (a *PosixACLXattrEntry) WriteTo(writer io.Writer) (int64, error) { + // Type PosixACLXattrEntry doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, a.SizeBytes()) + a.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err +} diff --git a/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go b/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go index 3b47f4b..5b2cd48 100644 --- a/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go +++ b/pkg/abi/linux/linux_amd64_abi_autogen_unsafe.go @@ -216,6 +216,56 @@ func UnmarshalUnsafeEpollEventSlice(dst []EpollEvent, src []byte) []byte { return src[size*count:] } +// ReadEpollEventSlice reads a []EpollEvent. It returns the number of bytes read +func ReadEpollEventSlice(src io.Reader, dst []EpollEvent) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteEpollEventSlice is like EpollEvent.WriteTo, but for a []EpollEvent. +func WriteEpollEventSlice(dst io.Writer, src []EpollEvent) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (s *Stat) SizeBytes() int { return 72 + diff --git a/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go b/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go index 884e946..2950ce4 100644 --- a/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go +++ b/pkg/abi/linux/linux_arm64_abi_autogen_unsafe.go @@ -220,6 +220,56 @@ func UnmarshalUnsafeEpollEventSlice(dst []EpollEvent, src []byte) []byte { return src[size*count:] } +// ReadEpollEventSlice reads a []EpollEvent. It returns the number of bytes read +func ReadEpollEventSlice(src io.Reader, dst []EpollEvent) (int, error) { + count := len(dst) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&dst) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := io.ReadFull(src, buf) + // Since we bypassed the compiler's escape analysis, indicate that dst + // must live until the use above. + runtime.KeepAlive(dst) // escapes: replaced by intrinsic. + return length, err +} + +// WriteEpollEventSlice is like EpollEvent.WriteTo, but for a []EpollEvent. +func WriteEpollEventSlice(dst io.Writer, src []EpollEvent) (int, error) { + count := len(src) + if count == 0 { + return 0, nil + } + size := (*EpollEvent)(nil).SizeBytes() + + ptr := unsafe.Pointer(&src) + val := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data)) + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(val) + hdr.Len = size * count + hdr.Cap = size * count + + length, err := dst.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that src + // must live until the use above. + runtime.KeepAlive(src) // escapes: replaced by intrinsic. + return length, err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (s *Stat) SizeBytes() int { return 72 + diff --git a/pkg/abi/linux/linux_state_autogen.go b/pkg/abi/linux/linux_state_autogen.go index d212e77..30cfc72 100644 --- a/pkg/abi/linux/linux_state_autogen.go +++ b/pkg/abi/linux/linux_state_autogen.go @@ -76,6 +76,40 @@ func (b *BPFInstruction) StateLoad(ctx context.Context, stateSourceObject state. stateSourceObject.Load(3, &b.K) } +func (e *EBPFInstruction) StateTypeName() string { + return "pkg/abi/linux.EBPFInstruction" +} + +func (e *EBPFInstruction) StateFields() []string { + return []string{ + "Code", + "Registers", + "Offset", + "Immediate", + } +} + +func (e *EBPFInstruction) beforeSave() {} + +// +checklocksignore +func (e *EBPFInstruction) StateSave(stateSinkObject state.Sink) { + e.beforeSave() + stateSinkObject.Save(0, &e.Code) + stateSinkObject.Save(1, &e.Registers) + stateSinkObject.Save(2, &e.Offset) + stateSinkObject.Save(3, &e.Immediate) +} + +func (e *EBPFInstruction) afterLoad(context.Context) {} + +// +checklocksignore +func (e *EBPFInstruction) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &e.Code) + stateSourceObject.Load(1, &e.Registers) + stateSourceObject.Load(2, &e.Offset) + stateSourceObject.Load(3, &e.Immediate) +} + func (f *FUSEHeaderIn) StateTypeName() string { return "pkg/abi/linux.FUSEHeaderIn" } @@ -505,6 +539,40 @@ func (i *ICMP6Filter) StateLoad(ctx context.Context, stateSourceObject state.Sou stateSourceObject.Load(0, &i.Filter) } +func (w *Winsize) StateTypeName() string { + return "pkg/abi/linux.Winsize" +} + +func (w *Winsize) StateFields() []string { + return []string{ + "Row", + "Col", + "Xpixel", + "Ypixel", + } +} + +func (w *Winsize) beforeSave() {} + +// +checklocksignore +func (w *Winsize) StateSave(stateSinkObject state.Sink) { + w.beforeSave() + stateSinkObject.Save(0, &w.Row) + stateSinkObject.Save(1, &w.Col) + stateSinkObject.Save(2, &w.Xpixel) + stateSinkObject.Save(3, &w.Ypixel) +} + +func (w *Winsize) afterLoad(context.Context) {} + +// +checklocksignore +func (w *Winsize) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &w.Row) + stateSourceObject.Load(1, &w.Col) + stateSourceObject.Load(2, &w.Xpixel) + stateSourceObject.Load(3, &w.Ypixel) +} + func (t *KernelTermios) StateTypeName() string { return "pkg/abi/linux.KernelTermios" } @@ -551,37 +619,10 @@ func (t *KernelTermios) StateLoad(ctx context.Context, stateSourceObject state.S stateSourceObject.Load(7, &t.OutputSpeed) } -func (w *WindowSize) StateTypeName() string { - return "pkg/abi/linux.WindowSize" -} - -func (w *WindowSize) StateFields() []string { - return []string{ - "Rows", - "Cols", - } -} - -func (w *WindowSize) beforeSave() {} - -// +checklocksignore -func (w *WindowSize) StateSave(stateSinkObject state.Sink) { - w.beforeSave() - stateSinkObject.Save(0, &w.Rows) - stateSinkObject.Save(1, &w.Cols) -} - -func (w *WindowSize) afterLoad(context.Context) {} - -// +checklocksignore -func (w *WindowSize) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &w.Rows) - stateSourceObject.Load(1, &w.Cols) -} - func init() { state.Register((*IOEvent)(nil)) state.Register((*BPFInstruction)(nil)) + state.Register((*EBPFInstruction)(nil)) state.Register((*FUSEHeaderIn)(nil)) state.Register((*FUSEHeaderOut)(nil)) state.Register((*IOUringCqe)(nil)) @@ -594,6 +635,6 @@ func init() { state.Register((*ControlMessageIPPacketInfo)(nil)) state.Register((*ControlMessageIPv6PacketInfo)(nil)) state.Register((*ICMP6Filter)(nil)) + state.Register((*Winsize)(nil)) state.Register((*KernelTermios)(nil)) - state.Register((*WindowSize)(nil)) } diff --git a/pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go b/pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go deleted file mode 100644 index 399c294..0000000 --- a/pkg/abi/linux/linux_unsafe_abi_autogen_unsafe.go +++ /dev/null @@ -1,3 +0,0 @@ -// Automatically generated marshal implementation. See tools/go_marshal. - -package linux diff --git a/pkg/abi/linux/mqueue.go b/pkg/abi/linux/mqueue.go index 79b463e..e6d9eeb 100644 --- a/pkg/abi/linux/mqueue.go +++ b/pkg/abi/linux/mqueue.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Default values for POSIX message queues. Source: // include/linux/ipc_namespace.h const ( @@ -47,6 +51,7 @@ const ( // // +marshal type MqAttr struct { + _ structs.HostLayout MqFlags int64 // Message queue flags. MqMaxmsg int64 // Maximum number of messages. MqMsgsize int64 // Maximum message size. diff --git a/pkg/abi/linux/msgqueue.go b/pkg/abi/linux/msgqueue.go index f283a66..550dfef 100644 --- a/pkg/abi/linux/msgqueue.go +++ b/pkg/abi/linux/msgqueue.go @@ -15,6 +15,8 @@ package linux import ( + "structs" + "github.com/sagernet/gvisor/pkg/marshal/primitive" ) @@ -55,6 +57,7 @@ const ( // // +marshal type MsqidDS struct { + _ structs.HostLayout MsgPerm IPCPerm // IPC permissions. MsgStime TimeT // Last msgsnd time. MsgRtime TimeT // Last msgrcv time. @@ -72,8 +75,9 @@ type MsqidDS struct { // // +marshal dynamic type MsgBuf struct { + _ structs.HostLayout Type primitive.Int64 - Text primitive.ByteSlice + Text primitive.ByteSlice `hostlayout:"ignore"` } // SizeBytes implements marshal.Marshallable.SizeBytes. @@ -97,6 +101,7 @@ func (b *MsgBuf) UnmarshalBytes(src []byte) []byte { // // +marshal type MsgInfo struct { + _ structs.HostLayout MsgPool int32 MsgMap int32 MsgMax int32 diff --git a/pkg/abi/linux/netdevice.go b/pkg/abi/linux/netdevice.go index 924bd88..4001547 100644 --- a/pkg/abi/linux/netdevice.go +++ b/pkg/abi/linux/netdevice.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + const ( // IFNAMSIZ is the size of the name field for IFReq. IFNAMSIZ = 16 @@ -23,6 +27,7 @@ const ( // // +marshal type IFReq struct { + _ structs.HostLayout // IFName is an encoded name, normally null-terminated. This should be // accessed via the Name and SetName functions. IFName [IFNAMSIZ]byte @@ -66,6 +71,7 @@ var SizeOfIFReq = (*IFReq)(nil).SizeBytes() // IFMap contains interface hardware parameters. type IFMap struct { + _ structs.HostLayout MemStart uint64 MemEnd uint64 BaseAddr int16 @@ -80,6 +86,7 @@ type IFMap struct { // // +marshal type IFConf struct { + _ structs.HostLayout Len int32 _ [4]byte // Pad to sizeof(struct ifconf). Ptr uint64 @@ -106,6 +113,7 @@ const ( // // +marshal type EthtoolGFeatures struct { + _ structs.HostLayout Cmd uint32 Size uint32 } @@ -116,6 +124,7 @@ type EthtoolGFeatures struct { // // +marshal type EthtoolGetFeaturesBlock struct { + _ structs.HostLayout Available uint32 Requested uint32 Active uint32 diff --git a/pkg/abi/linux/netfilter.go b/pkg/abi/linux/netfilter.go index 5033ee4..f479f89 100644 --- a/pkg/abi/linux/netfilter.go +++ b/pkg/abi/linux/netfilter.go @@ -15,6 +15,8 @@ package linux import ( + "structs" + "github.com/sagernet/gvisor/pkg/marshal" "github.com/sagernet/gvisor/pkg/marshal/primitive" ) @@ -109,6 +111,7 @@ const ( // // +marshal type IPTEntry struct { + _ structs.HostLayout // IP is used to filter packets based on the IP header. IP IPTIP @@ -148,13 +151,14 @@ const SizeOfIPTEntry = 112 // // +marshal dynamic type KernelIPTEntry struct { + _ structs.HostLayout Entry IPTEntry // Elems holds the data for all this rule's matches followed by the // target. It is variable length -- users have to iterate over any // matches and use TargetOffset and NextOffset to make sense of the // data. - Elems primitive.ByteSlice + Elems primitive.ByteSlice `hostlayout:"ignore"` } // SizeBytes implements marshal.Marshallable.SizeBytes. @@ -182,6 +186,7 @@ var _ marshal.Marshallable = (*KernelIPTEntry)(nil) // // +marshal type IPTIP struct { + _ structs.HostLayout // Src is the source IP address. Src InetAddr @@ -246,6 +251,7 @@ const SizeOfIPTIP = 84 // // +marshal type XTCounters struct { + _ structs.HostLayout // Pcnt is the packet count. Pcnt uint64 @@ -267,6 +273,7 @@ const SizeOfXTCounters = 16 // // +marshal type XTEntryMatch struct { + _ structs.HostLayout MatchSize uint16 Name ExtensionName Revision uint8 @@ -281,8 +288,9 @@ const SizeOfXTEntryMatch = 32 // KernelXTEntryMatch is identical to XTEntryMatch, but contains // variable-length Data field. type KernelXTEntryMatch struct { + _ structs.HostLayout XTEntryMatch - Data []byte + Data []byte `hostlayout:"ignore"` } // XTGetRevision corresponds to xt_get_revision in @@ -290,6 +298,7 @@ type KernelXTEntryMatch struct { // // +marshal type XTGetRevision struct { + _ structs.HostLayout Name ExtensionName Revision uint8 } @@ -308,6 +317,7 @@ const SizeOfXTGetRevision = 30 // // +marshal type XTEntryTarget struct { + _ structs.HostLayout TargetSize uint16 Name ExtensionName Revision uint8 @@ -322,8 +332,9 @@ const SizeOfXTEntryTarget = 32 // KernelXTEntryTarget is identical to XTEntryTarget, but contains a // variable-length Data field. type KernelXTEntryTarget struct { + _ structs.HostLayout XTEntryTarget - Data []byte + Data []byte `hostlayout:"ignore"` } // XTStandardTarget is a built-in target, one of ACCEPT, DROP, JUMP, QUEUE, @@ -332,6 +343,7 @@ type KernelXTEntryTarget struct { // // +marshal type XTStandardTarget struct { + _ structs.HostLayout Target XTEntryTarget // A positive verdict indicates a jump, and is the offset from the // start of the table to jump to. A negative value means one of the @@ -350,6 +362,7 @@ const SizeOfXTStandardTarget = 40 // // +marshal type XTErrorTarget struct { + _ structs.HostLayout Target XTEntryTarget Name ErrorName _ [2]byte @@ -379,6 +392,7 @@ const ( // // +marshal type NfNATIPV4Range struct { + _ structs.HostLayout Flags uint32 MinIP [4]byte MaxIP [4]byte @@ -391,6 +405,7 @@ type NfNATIPV4Range struct { // // +marshal type NfNATIPV4MultiRangeCompat struct { + _ structs.HostLayout RangeSize uint32 RangeIPV4 NfNATIPV4Range } @@ -400,6 +415,7 @@ type NfNATIPV4MultiRangeCompat struct { // // +marshal type XTRedirectTarget struct { + _ structs.HostLayout Target XTEntryTarget NfRange NfNATIPV4MultiRangeCompat _ [4]byte @@ -413,6 +429,7 @@ const SizeOfXTRedirectTarget = 56 // // +marshal type XTNATTargetV0 struct { + _ structs.HostLayout Target XTEntryTarget NfRange NfNATIPV4MultiRangeCompat _ [4]byte @@ -425,6 +442,7 @@ const SizeOfXTNATTargetV0 = 56 // // +marshal type XTNATTargetV1 struct { + _ structs.HostLayout Target XTEntryTarget Range NFNATRange } @@ -436,6 +454,7 @@ const SizeOfXTNATTargetV1 = SizeOfXTEntryTarget + SizeOfNFNATRange // // +marshal type XTNATTargetV2 struct { + _ structs.HostLayout Target XTEntryTarget Range NFNATRange2 } @@ -443,11 +462,33 @@ type XTNATTargetV2 struct { // SizeOfXTNATTargetV2 is the size of an XTNATTargetV2. const SizeOfXTNATTargetV2 = SizeOfXTEntryTarget + SizeOfNFNATRange2 +// XTCTTargetInfoV0 corresponds to struct xt_ct_target_info (revision 0) in +// include/uapi/linux/netfilter/xt_CT.h. The CT target is used in the raw +// table for conntrack zone assignment. The trailing padding accounts for the +// kernel-internal nf_conn pointer that is 8-byte aligned. +// +// +marshal +type XTCTTargetInfoV0 struct { + _ structs.HostLayout + Target XTEntryTarget + Flags uint16 + Zone uint16 + CTEvents uint32 + ExpEvents uint32 + Helper [16]byte + _ [4]byte // padding for 8-byte alignment of ct pointer + _ [8]byte // space for kernel nf_conn pointer (unused in userspace) +} + +// SizeOfXTCTTargetInfoV0 is the size of an XTCTTargetInfoV0. +const SizeOfXTCTTargetInfoV0 = 72 + // IPTGetinfo is the argument for the IPT_SO_GET_INFO sockopt. It corresponds // to struct ipt_getinfo in include/uapi/linux/netfilter_ipv4/ip_tables.h. // // +marshal type IPTGetinfo struct { + _ structs.HostLayout Name TableName ValidHooks uint32 HookEntry [NF_INET_NUMHOOKS]uint32 @@ -465,6 +506,7 @@ const SizeOfIPTGetinfo = 84 // // +marshal type IPTGetEntries struct { + _ structs.HostLayout Name TableName Size uint32 _ [4]byte @@ -482,8 +524,9 @@ const SizeOfIPTGetEntries = 40 // // +marshal dynamic type KernelIPTGetEntries struct { + _ structs.HostLayout IPTGetEntries - Entrytable []KernelIPTEntry + Entrytable []KernelIPTEntry `hostlayout:"ignore"` } // SizeBytes implements marshal.Marshallable.SizeBytes. @@ -521,6 +564,7 @@ var _ marshal.Marshallable = (*KernelIPTGetEntries)(nil) // // +marshal type IPTReplace struct { + _ structs.HostLayout Name TableName ValidHooks uint32 NumEntries uint32 @@ -582,6 +626,7 @@ func goString(cstring []byte) string { // // +marshal type XTTCP struct { + _ structs.HostLayout // SourcePortStart specifies the inclusive start of the range of source // ports to which the matcher applies. SourcePortStart uint16 @@ -637,6 +682,7 @@ const ( // // +marshal type XTUDP struct { + _ structs.HostLayout // SourcePortStart is the inclusive start of the range of source ports // to which the matcher applies. SourcePortStart uint16 @@ -679,6 +725,7 @@ const ( // // +marshal type IPTOwnerInfo struct { + _ structs.HostLayout // UID is user id which created the packet. UID uint32 @@ -711,6 +758,7 @@ const SizeOfIPTOwnerInfo = 34 // // +marshal type XTOwnerMatchInfo struct { + _ structs.HostLayout UIDMin uint32 UIDMax uint32 GIDMin uint32 @@ -753,6 +801,7 @@ const ( // // +marshal type XTMultiport struct { + _ structs.HostLayout // Flags indicates whether the match applies to // source ports, destination ports, or either, as // defined by "enum xt_multiport_flags". @@ -774,6 +823,7 @@ type XTMultiport struct { // // +marshal type XTMultiportV1 struct { + _ structs.HostLayout // Fields same as "XTMultiport". Flags uint8 Count uint8 @@ -794,3 +844,68 @@ const SizeOfXTMultiport = 2 + (XT_MULTI_PORTS * 2) // SizeOfXTMultiportV1 is the size of XTMultiportV1 (in bytes). const SizeOfXTMultiportV1 = SizeOfXTMultiport + XT_MULTI_PORTS + 1 + +// XTMarkMtinfo1 holds data for matching packets against a mark. +// It corresponds to struct xt_mark_mtinfo1 in include/uapi/linux/netfilter/xt_mark.h. +// +// +marshal +type XTMarkMtinfo1 struct { + _ structs.HostLayout + Mark uint32 + Mask uint32 + Invert uint8 + _ [3]byte +} + +// SizeOfXTMarkMtinfo1 is the size of XTMarkMtinfo1. +const SizeOfXTMarkMtinfo1 = 12 + +// Ref: include/uapi/linux/netfilter_ipv4/ipt_REJECT.h:enum ipt_reject_with +const ( + IPT_ICMP_NET_UNREACHABLE = iota + IPT_ICMP_HOST_UNREACHABLE + IPT_ICMP_PROT_UNREACHABLE + IPT_ICMP_PORT_UNREACHABLE + IPT_ICMP_ECHOREPLY + IPT_ICMP_NET_PROHIBITED + IPT_ICMP_HOST_PROHIBITED + IPT_TCP_RESET + IPT_ICMP_ADMIN_PROHIBITED +) + +// Ref: include/uapi/linux/netfilter_ipv6/ip6t_REJECT.h:enum ip6t_reject_with +const ( + IP6T_ICMP6_NO_ROUTE = iota + IP6T_ICMP6_ADM_PROHIBITED + IP6T_ICMP6_NOT_NEIGHBOUR + IP6T_ICMP6_ADDR_UNREACH + IP6T_ICMP6_PORT_UNREACH + IP6T_ICMP6_ECHOREPLY + IP6T_TCP_RESET + IP6T_ICMP6_POLICY_FAIL + IP6T_ICMP6_REJECT_ROUTE +) + +// IPTRejectInfo is the argument for the IPT_REJECT target. It corresponds to +// struct ipt_reject_info in include/uapi/linux/netfilter_ipv4/ipt_REJECT.h. +// +// +marshal +type IPTRejectInfo struct { + _ structs.HostLayout + With uint32 +} + +// SizeOfIPTRejectInfo is the size of an IPTRejectInfo. +const SizeOfIPTRejectInfo = 4 + +// IP6TRejectInfo is the argument for the IP6T_REJECT target. It corresponds to +// struct ip6t_reject_info in include/uapi/linux/netfilter_ipv6/ip6t_REJECT.h. +// +// +marshal +type IP6TRejectInfo struct { + _ structs.HostLayout + With uint32 +} + +// SizeOfIP6TRejectInfo is the size of an IP6TRejectInfo. +const SizeOfIP6TRejectInfo = 4 diff --git a/pkg/abi/linux/netfilter_ipv6.go b/pkg/abi/linux/netfilter_ipv6.go index 691e604..31b521b 100644 --- a/pkg/abi/linux/netfilter_ipv6.go +++ b/pkg/abi/linux/netfilter_ipv6.go @@ -16,6 +16,7 @@ package linux import ( "math" + "structs" "github.com/sagernet/gvisor/pkg/marshal" "github.com/sagernet/gvisor/pkg/marshal/primitive" @@ -69,6 +70,7 @@ const IP6T_ORIGINAL_DST = 80 // // +marshal type IP6TReplace struct { + _ structs.HostLayout Name TableName ValidHooks uint32 NumEntries uint32 @@ -90,8 +92,9 @@ const SizeOfIP6TReplace = 96 // // +marshal dynamic type KernelIP6TGetEntries struct { + _ structs.HostLayout IPTGetEntries - Entrytable []KernelIP6TEntry + Entrytable []KernelIP6TEntry `hostlayout:"ignore"` } // SizeBytes implements marshal.Marshallable.SizeBytes. @@ -128,6 +131,7 @@ var _ marshal.Marshallable = (*KernelIP6TGetEntries)(nil) // // +marshal type IP6TEntry struct { + _ structs.HostLayout // IPv6 is used to filter packets based on the IPv6 header. IPv6 IP6TIP @@ -169,13 +173,14 @@ const SizeOfIP6TEntry = 168 // // +marshal dynamic type KernelIP6TEntry struct { + _ structs.HostLayout Entry IP6TEntry // Elems holds the data for all this rule's matches followed by the // target. It is variable length -- users have to iterate over any // matches and use TargetOffset and NextOffset to make sense of the // data. - Elems primitive.ByteSlice + Elems primitive.ByteSlice `hostlayout:"ignore"` } // SizeBytes implements marshal.Marshallable.SizeBytes. @@ -203,6 +208,7 @@ var _ marshal.Marshallable = (*KernelIP6TEntry)(nil) // // +marshal type IP6TIP struct { + _ structs.HostLayout // Src is the source IP address. Src Inet6Addr @@ -286,6 +292,7 @@ const ( // // +marshal type NFNATRange struct { + _ structs.HostLayout Flags uint32 MinAddr Inet6Addr MaxAddr Inet6Addr @@ -301,6 +308,7 @@ const SizeOfNFNATRange = 40 // // +marshal type NFNATRange2 struct { + _ structs.HostLayout Flags uint32 MinAddr Inet6Addr MaxAddr Inet6Addr diff --git a/pkg/abi/linux/netlink.go b/pkg/abi/linux/netlink.go index 2be0b75..46717a0 100644 --- a/pkg/abi/linux/netlink.go +++ b/pkg/abi/linux/netlink.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Netlink protocols, from uapi/linux/netlink.h. const ( NETLINK_ROUTE = 0 @@ -43,6 +47,7 @@ const ( // // +marshal type SockAddrNetlink struct { + _ structs.HostLayout Family uint16 _ uint16 PortID uint32 @@ -56,6 +61,7 @@ const SockAddrNetlinkSize = 12 // // +marshal type NetlinkMessageHeader struct { + _ structs.HostLayout Length uint32 Type uint16 Flags uint16 @@ -118,6 +124,7 @@ const NLMSG_ALIGNTO = 4 // // +marshal type NetlinkAttrHeader struct { + _ structs.HostLayout Length uint16 Type uint16 } @@ -136,6 +143,33 @@ const NetlinkAttrHeaderSize = 4 // uapi/linux/netlink.h. const NLA_ALIGNTO = 4 +// Standard attribute types to specify validation policy, from +// include/net/netlink.h. +const ( + NLA_UNSPEC = iota + NLA_U8 + NLA_U16 + NLA_U32 + NLA_U64 + NLA_STRING + NLA_FLAG + NLA_MSECS + NLA_NESTED + NLA_NESTED_ARRAY + NLA_NUL_STRING + NLA_BINARY + NLA_S8 + NLA_S16 + NLA_S32 + NLA_S64 + NLA_BITFIELD32 + NLA_REJECT + NLA_BE16 + NLA_BE32 + __NLA_TYPE_MAX + NLA_TYPE_MAX = __NLA_TYPE_MAX - 1 +) + // Socket options, from uapi/linux/netlink.h. const ( NETLINK_ADD_MEMBERSHIP = 1 @@ -154,6 +188,12 @@ const ( // // +marshal type NetlinkErrorMessage struct { + _ structs.HostLayout Error int32 Header NetlinkMessageHeader } + +// RTNetlink multicast groups, from uapi/linux/rtnetlink.h. +const ( + RTNLGRP_LINK = 1 +) diff --git a/pkg/abi/linux/netlink_netfilter.go b/pkg/abi/linux/netlink_netfilter.go index ebf178c..0758452 100644 --- a/pkg/abi/linux/netlink_netfilter.go +++ b/pkg/abi/linux/netlink_netfilter.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Group describes Netlink Netfilter groups, from uapi/linux/netfilter/nfnetlink.h. // Users bind to specific groups to receive processing logs from those groups. type Group uint16 @@ -38,6 +42,7 @@ const ( // // +marshal type NetFilterGenMsg struct { + _ structs.HostLayout Family uint8 Version uint8 ResourceID uint16 diff --git a/pkg/abi/linux/netlink_route.go b/pkg/abi/linux/netlink_route.go index b3deba1..20caae8 100644 --- a/pkg/abi/linux/netlink_route.go +++ b/pkg/abi/linux/netlink_route.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Netlink message types for NETLINK_ROUTE sockets, from uapi/linux/rtnetlink.h. const ( RTM_NEWLINK = 16 @@ -88,6 +92,7 @@ const ( // // +marshal type InterfaceInfoMessage struct { + _ structs.HostLayout Family uint8 _ uint8 Type uint16 @@ -187,6 +192,7 @@ const ( // // +marshal type InterfaceAddrMessage struct { + _ structs.HostLayout Family uint8 PrefixLen uint8 Flags uint8 @@ -221,6 +227,7 @@ const ( // // +marshal type RouteMessage struct { + _ structs.HostLayout Family uint8 DstLen uint8 SrcLen uint8 @@ -369,6 +376,7 @@ const ( // // +marshal type RtAttr struct { + _ structs.HostLayout Len uint16 Type uint16 } diff --git a/pkg/abi/linux/nf_tables.go b/pkg/abi/linux/nf_tables.go index dcd1d9e..4f6477e 100644 --- a/pkg/abi/linux/nf_tables.go +++ b/pkg/abi/linux/nf_tables.go @@ -28,6 +28,7 @@ const ( NFT_OBJ_MAXNAMELEN = NFT_NAME_MAXLEN NFT_USERDATA_MAXLEN = 256 NFT_OSF_MAXGENRELEN = 16 + NFT_SET_EXPR_MAX = 2 ) // 16-byte Registers that can be used to maintain state for rules. @@ -336,6 +337,22 @@ const ( NFTA_IMMEDIATE_MAX = __NFTA_IMMEDIATE_MAX - 1 ) +// NfTablePayloadAttributes represents the netfilter payload attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_PAYLOAD_UNSPEC uint16 = iota + NFTA_PAYLOAD_DREG + NFTA_PAYLOAD_BASE + NFTA_PAYLOAD_OFFSET + NFTA_PAYLOAD_LEN + NFTA_PAYLOAD_SREG + NFTA_PAYLOAD_CSUM_TYPE + NFTA_PAYLOAD_CSUM_OFFSET + NFTA_PAYLOAD_CSUM_FLAGS + __NFTA_PAYLOAD_MAX + NFTA_PAYLOAD_MAX = __NFTA_PAYLOAD_MAX - 1 +) + // Nf table relational operators. // Used by the nft comparison operation to compare values in registers. // These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. @@ -348,6 +365,17 @@ const ( NFT_CMP_GTE // greater than or equal to ) +// Nf table cmp expression netlink attributes. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_CMP_UNSPEC uint16 = iota + NFTA_CMP_SREG + NFTA_CMP_OP + NFTA_CMP_DATA + __NFTA_CMP_MAX + NFTA_CMP_MAX = __NFTA_CMP_MAX - 1 +) + // Nf table range operators. // Used by the nft range operation to compare values in registers. // These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. @@ -392,6 +420,21 @@ const ( NFT_BITWISE_RSHIFT // right-shift operation ) +// Nf table bitwise expression netlink attributes. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_BITWISE_UNSPEC uint16 = iota + NFTA_BITWISE_SREG + NFTA_BITWISE_DREG + NFTA_BITWISE_LEN + NFTA_BITWISE_MASK + NFTA_BITWISE_XOR + NFTA_BITWISE_OP + NFTA_BITWISE_DATA + __NFTA_BITWISE_MAX + NFTA_BITWISE_MAX = __NFTA_BITWISE_MAX - 1 +) + // Nf table route expression keys. // Used by the nft route operation to determine the routing data to retrieve. // These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. @@ -462,3 +505,323 @@ const ( NFT_META_SDIFNAME // Slave device interface name NFT_META_BRI_BROUTE // Packet br_netfilter_broute bit ) + +// Nf table meta expression netlink attributes +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_META_UNSPEC = iota + NFTA_META_DREG + NFTA_META_KEY + NFTA_META_SREG + __NFTA_META_MAX + NFTA_META_MAX = __NFTA_META_MAX - 1 +) + +// Nf table counter expression netlink attributes. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_COUNTER_UNSPEC = iota + NFTA_COUNTER_BYTES + NFTA_COUNTER_PACKETS + NFTA_COUNTER_PAD + __NFTA_COUNTER_MAX + NFTA_COUNTER_MAX = __NFTA_COUNTER_MAX - 1 +) + +// Nftables Generation Attributes +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_GEN_UNSPEC uint16 = iota + NFTA_GEN_ID + NFTA_GEN_PROC_PID + NFTA_GEN_PROC_NAME + __NFTA_GEN_MAX + NFTA_GEN_MAX = __NFTA_GEN_MAX - 1 +) + +// Nf table nat expression netlink attributes. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_NAT_UNSPEC uint16 = iota + NFTA_NAT_TYPE + NFTA_NAT_FAMILY + NFTA_NAT_REG_ADDR_MIN + NFTA_NAT_REG_ADDR_MAX + NFTA_NAT_REG_PROTO_MIN + NFTA_NAT_REG_PROTO_MAX + NFTA_NAT_FLAGS + __NFTA_NAT_MAX + NFTA_NAT_MAX = __NFTA_NAT_MAX - 1 +) + +// NfTableSetFlags represents the netfilter set flags. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_SET_ANONYMOUS = uint16(0x1) + NFT_SET_CONSTANT = uint16(0x2) + NFT_SET_INTERVAL = uint16(0x4) + NFT_SET_MAP = uint16(0x8) + NFT_SET_TIMEOUT = uint16(0x10) + NFT_SET_EVAL = uint16(0x20) + NFT_SET_OBJECT = uint16(0x40) + NFT_SET_CONCAT = uint16(0x80) + NFT_SET_EXPR = uint16(0x100) +) + +// NfTableSetAttributes represents the netfilter set attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_SET_UNSPEC uint16 = iota + NFTA_SET_TABLE + NFTA_SET_NAME + NFTA_SET_FLAGS + NFTA_SET_KEY_TYPE + NFTA_SET_KEY_LEN + NFTA_SET_DATA_TYPE + NFTA_SET_DATA_LEN + NFTA_SET_POLICY + NFTA_SET_DESC + NFTA_SET_ID + NFTA_SET_TIMEOUT + NFTA_SET_GC_INTERVAL + NFTA_SET_USERDATA + NFTA_SET_PAD + NFTA_SET_OBJ_TYPE + NFTA_SET_HANDLE + NFTA_SET_EXPR + NFTA_SET_EXPRESSIONS + __NFTA_SET_MAX + NFTA_SET_MAX = __NFTA_SET_MAX - 1 +) + +// NfTableSetPolicies represents the netfilter set policies. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_SET_POL_PERFORMANCE uint32 = iota // prefer high performance over low memory use + NFT_SET_POL_MEMORY // prefer low memory use over high performance +) + +// NfTableSetDescAttributes represents the netfilter set description attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_SET_DESC_UNSPEC uint16 = iota + NFTA_SET_DESC_SIZE + NFTA_SET_DESC_CONCAT + __NFTA_SET_DESC_MAX + NFTA_SET_DESC_MAX = __NFTA_SET_DESC_MAX - 1 +) + +// NfTableSetFieldAttributes represents the netfilter set field attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_SET_FIELD_UNSPEC uint16 = iota + NFTA_SET_FIELD_LEN + __NFTA_SET_FIELD_MAX + NFTA_SET_FIELD_MAX = __NFTA_SET_FIELD_MAX - 1 +) + +// NfTableObjectAttributes represents the netfilter object attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_OBJECT_UNSPEC uint32 = iota + NFT_OBJECT_COUNTER + NFT_OBJECT_QUOTA + NFT_OBJECT_CT_HELPER + NFT_OBJECT_LIMIT + NFT_OBJECT_CONNLIMIT + NFT_OBJECT_TUNNEL + NFT_OBJECT_CT_TIMEOUT + NFT_OBJECT_SECMARK + NFT_OBJECT_CT_EXPECT + NFT_OBJECT_SYNPROXY + __NFT_OBJECT_MAX + NFT_OBJECT_MAX = __NFT_OBJECT_MAX - 1 +) + +// NfTableSetElemListAttributes represents the netfilter set element list attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_SET_ELEM_LIST_UNSPEC uint16 = iota + NFTA_SET_ELEM_LIST_TABLE + NFTA_SET_ELEM_LIST_SET + NFTA_SET_ELEM_LIST_ELEMENTS + NFTA_SET_ELEM_LIST_SET_ID + __NFTA_SET_ELEM_LIST_MAX + NFTA_SET_ELEM_LIST_MAX = __NFTA_SET_ELEM_LIST_MAX - 1 +) + +// NfTableSetElemAttributes represents the netfilter set element attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_SET_ELEM_UNSPEC uint16 = iota + NFTA_SET_ELEM_KEY + NFTA_SET_ELEM_DATA + NFTA_SET_ELEM_FLAGS + NFTA_SET_ELEM_TIMEOUT + NFTA_SET_ELEM_EXPIRATION + NFTA_SET_ELEM_USERDATA + NFTA_SET_ELEM_EXPR + NFTA_SET_ELEM_PAD + NFTA_SET_ELEM_OBJREF + NFTA_SET_ELEM_KEY_END + NFTA_SET_ELEM_EXPRESSIONS + __NFTA_SET_ELEM_MAX + NFTA_SET_ELEM_MAX = __NFTA_SET_ELEM_MAX - 1 +) + +// NfTableSetElemFlags represents the netfilter set element flags. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_SET_ELEM_INTERVAL_END = uint16(0x1) + NFT_SET_ELEM_CATCHALL = uint16(0x2) +) + +// NfTableLookupAttributes represents the netfilter lookup attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_LOOKUP_UNSPEC uint16 = iota + NFTA_LOOKUP_SET + NFTA_LOOKUP_SREG + NFTA_LOOKUP_DREG + NFTA_LOOKUP_SET_ID + NFTA_LOOKUP_FLAGS + __NFTA_LOOKUP_MAX + NFTA_LOOKUP_MAX = __NFTA_LOOKUP_MAX - 1 +) +const NFT_LOOKUP_F_INV = uint32(1 << 0) + +// NfTable fib expression netlink attributes. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_FIB_UNSPEC uint16 = iota + NFTA_FIB_DREG + NFTA_FIB_RESULT + NFTA_FIB_FLAGS + __NFTA_FIB_MAX +) + +const NFTA_FIB_MAX = __NFTA_FIB_MAX - 1 + +// NfTable fib result types. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_FIB_RESULT_UNSPEC = iota + NFT_FIB_RESULT_OIF + NFT_FIB_RESULT_OIFNAME + NFT_FIB_RESULT_ADDRTYPE + __NFT_FIB_RESULT_MAX +) + +const NFT_FIB_RESULT_MAX = __NFT_FIB_RESULT_MAX - 1 + +// NfTable fib flags. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_FIB_F_SADDR = 1 << 0 + NFTA_FIB_F_DADDR = 1 << 1 + NFTA_FIB_F_MARK = 1 << 2 + NFTA_FIB_F_IIF = 1 << 3 + NFTA_FIB_F_OIF = 1 << 4 + NFTA_FIB_F_PRESENT = 1 << 5 +) + +// Nf table ct expression keys. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFT_CT_STATE = iota + NFT_CT_DIRECTION + NFT_CT_STATUS + NFT_CT_MARK + NFT_CT_SECMARK + NFT_CT_EXPIRATION + NFT_CT_HELPER + NFT_CT_L3PROTOCOL + NFT_CT_SRC + NFT_CT_DST + NFT_CT_PROTOCOL + NFT_CT_PROTO_SRC + NFT_CT_PROTO_DST + NFT_CT_LABELS + NFT_CT_PKTS + NFT_CT_BYTES + NFT_CT_AVGPKT + NFT_CT_ZONE + NFT_CT_EVENTMASK + NFT_CT_SRC_IP + NFT_CT_DST_IP + NFT_CT_SRC_IP6 + NFT_CT_DST_IP6 + NFT_CT_ID + __NFT_CT_MAX + NFT_CT_MAX = __NFT_CT_MAX - 1 +) + +// Nf table ct expression netlink attributes. +// These correspond to values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_CT_UNSPEC uint16 = iota + NFTA_CT_DREG + NFTA_CT_KEY + NFTA_CT_DIRECTION + NFTA_CT_SREG + __NFTA_CT_MAX + NFTA_CT_MAX = __NFTA_CT_MAX - 1 +) + +// IPCTInfo represents the state of a connection. +// Used with NF_CT_STATE to represent the state of a connection. +// Ref: enum include/uapi/linux/netfilter/nf_conntrack.h:ip_conntrack_info +type IPCTInfo int + +const ( + // IP_CT_ESTABLISHED represents an established connection (either direction). + IP_CT_ESTABLISHED IPCTInfo = iota + // IP_CT_RELATED represents a connection related to an existing connection, + // or an ICMP error (in either direction). + IP_CT_RELATED + // IP_CT_NEW represents a new connection to track. + IP_CT_NEW + // IP_CT_IS_REPLY indicates reply direction. + IP_CT_IS_REPLY + // IP_CT_ESTABLISHED_REPLY represents an established connection in the reply direction. + IP_CT_ESTABLISHED_REPLY = IP_CT_ESTABLISHED + IP_CT_IS_REPLY + // IP_CT_RELATED_REPLY represents a connection related to an existing connection, + // or an ICMP error in the reply direction. + IP_CT_RELATED_REPLY = IP_CT_RELATED + IP_CT_IS_REPLY + // IP_CT_NUMBER is the number of distinct IP_CT types. + IP_CT_NUMBER = 5 + // IP_CT_NEW_REPLY is for userspace compatibility. + IP_CT_NEW_REPLY = IP_CT_NUMBER + // IP_CT_UNTRACKED represents an untracked connection. + IP_CT_UNTRACKED = 7 +) + +// Conntrack states. +const ( + // NF_CT_STATE_INVALID_BIT represents an invalid connection state. + NF_CT_STATE_INVALID_BIT = 1 << 0 + // NF_CT_STATE_UNTRACKED_BIT represents an untracked connection state. + NF_CT_STATE_UNTRACKED_BIT = 1 << 6 +) + +// From include/uapi/linux/netfilter/nf_conntrack_common.h. +const ( + IP_CT_DIR_ORIGINAL uint8 = iota + IP_CT_DIR_REPLY + IP_CT_DIR_MAX +) + +// Nf table masq expression netlink attributes. +// These correspond to enum values in include/uapi/linux/netfilter/nf_tables.h. +const ( + NFTA_MASQ_UNSPEC uint16 = iota + NFTA_MASQ_FLAGS + NFTA_MASQ_REG_PROTO_MIN + NFTA_MASQ_REG_PROTO_MAX + __NFTA_MASQ_MAX + NFTA_MASQ_MAX = __NFTA_MASQ_MAX - 1 +) + +// SizeOfNfConntrackManProto is the size of the nf_conntrack_man_proto in bytes. +// Ref: include/uapi/linux/netfilter/nf_conntrack_tuple_common.h:nf_conntrack_man_proto. +const SizeOfNfConntrackManProto = 2 diff --git a/pkg/sync/runtime_constants_go125.go b/pkg/abi/linux/personality.go similarity index 51% rename from pkg/sync/runtime_constants_go125.go rename to pkg/abi/linux/personality.go index 1dcd70c..a4313fb 100644 --- a/pkg/sync/runtime_constants_go125.go +++ b/pkg/abi/linux/personality.go @@ -1,4 +1,4 @@ -// Copyright 2023 The gVisor Authors. +// Copyright 2026 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,14 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -// https://go.dev/cl/670497 (1.25) adds a new wait reason, adjusting the value of waitReasonSemacquire. -//go:build go1.25 +package linux -package sync - -// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. +// Personality flags, used by personality(2), +// from include/uapi/linux/personality.h. const ( - WaitReasonSelect uint8 = 9 // +checkconst runtime waitReasonSelect - WaitReasonChanReceive uint8 = 14 // +checkconst runtime waitReasonChanReceive - WaitReasonSemacquire uint8 = 19 // +checkconst runtime waitReasonSemacquire + SHORT_INODE = 0x1000000 + WHOLE_SECONDS = 0x2000000 + PER_LINUX = 0x0000 + PER_BSD = 0x0006 + PER_HPUX = 0x0010 ) + +// NOTE: All of the above flags are non-security-sensitive and may be copied +// from parent task to child task. However, this is not the case for all +// personality bits. If adding more, check PER_CLEAR_ON_SETID and ensure that +// these are cleared on suid/sgid execs. diff --git a/pkg/goid/goid_123_amd64.s b/pkg/abi/linux/pidfd.go similarity index 62% rename from pkg/goid/goid_123_amd64.s rename to pkg/abi/linux/pidfd.go index 21c61a7..4cbbfd7 100644 --- a/pkg/goid/goid_123_amd64.s +++ b/pkg/abi/linux/pidfd.go @@ -1,4 +1,4 @@ -// Copyright 2020 The gVisor Authors. +// Copyright 2026 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.23 && !go1.25 +package linux -#include "textflag.h" +// Flags for pidfd_open() from include/uapi/linux/pidfd.h. +const ( + PIDFD_NONBLOCK = O_NONBLOCK + PIDFD_THREAD = O_EXCL +) -#define GOID_OFFSET 160 // +checkoffset runtime g.goid - -// func goid() int64 -TEXT ·goid(SB),NOSPLIT|NOFRAME,$0-8 - MOVQ (TLS), R14 - MOVQ GOID_OFFSET(R14), R14 - MOVQ R14, ret+0(FP) - RET +// Flags for pidfd_send_signal(). +const ( + PIDFD_SIGNAL_THREAD = 1 << 0 + PIDFD_SIGNAL_THREAD_GROUP = 1 << 1 + PIDFD_SIGNAL_PROCESS_GROUP = 1 << 2 +) diff --git a/pkg/abi/linux/poll.go b/pkg/abi/linux/poll.go index 3443a57..1dda0de 100644 --- a/pkg/abi/linux/poll.go +++ b/pkg/abi/linux/poll.go @@ -14,10 +14,15 @@ package linux +import ( + "structs" +) + // PollFD is struct pollfd, used by poll(2)/ppoll(2), from uapi/asm-generic/poll.h. // // +marshal slice:PollFDSlice type PollFD struct { + _ structs.HostLayout FD int32 Events int16 REvents int16 @@ -42,3 +47,9 @@ const ( POLLFREE = 0x4000 POLL_BUSY_LOOP = 0x8000 ) + +const ( + // ReventsOffsetInPollFD is the byte offset of the REvents field within + // linux.PollFD. + ReventsOffsetInPollFD = 6 // +checkoffset . PollFD.REvents +) diff --git a/pkg/abi/linux/prctl.go b/pkg/abi/linux/prctl.go index c1928ae..ad4989c 100644 --- a/pkg/abi/linux/prctl.go +++ b/pkg/abi/linux/prctl.go @@ -34,6 +34,12 @@ const ( // PR_SET_KEEPCAPS sets the value of the keep capabilities flag. PR_SET_KEEPCAPS = 8 + // PR_GET_SECUREBITS gets the securebits flags of the calling thread. + PR_GET_SECUREBITS = 27 + + // PR_SET_SECUREBITS sets the securebits flags of the calling thread. + PR_SET_SECUREBITS = 28 + // PR_GET_TIMING gets the process' timing method. PR_GET_TIMING = 13 @@ -159,6 +165,21 @@ const ( // specified) to ptrace the current task. PR_SET_PTRACER = 0x59616d61 PR_SET_PTRACER_ANY = -1 + + PR_SET_TAGGED_ADDR_CTRL = 55 + PR_GET_TAGGED_ADDR_CTRL = 56 + PR_TAGGED_ADDR_ENABLE = (1 << 0) + + // PR_CAP_AMBIENT controls ambient capabilities. + PR_CAP_AMBIENT = 47 + + PR_CAP_AMBIENT_IS_SET = 1 + PR_CAP_AMBIENT_RAISE = 2 + PR_CAP_AMBIENT_LOWER = 3 + PR_CAP_AMBIENT_CLEAR_ALL = 4 + + // SECBIT_* flags are used to control securebits. + SECBIT_KEEP_CAPS = 1 << 4 ) // From diff --git a/pkg/abi/linux/ptrace_amd64.go b/pkg/abi/linux/ptrace_amd64.go index e970b5b..3c25724 100644 --- a/pkg/abi/linux/ptrace_amd64.go +++ b/pkg/abi/linux/ptrace_amd64.go @@ -17,12 +17,17 @@ package linux +import ( + "structs" +) + // PtraceRegs is the set of CPU registers exposed by ptrace. Source: // syscall.PtraceRegs. // // +marshal // +stateify savable type PtraceRegs struct { + _ structs.HostLayout R15 uint64 R14 uint64 R13 uint64 diff --git a/pkg/abi/linux/ptrace_arm64.go b/pkg/abi/linux/ptrace_arm64.go index 87c197d..83bb988 100644 --- a/pkg/abi/linux/ptrace_arm64.go +++ b/pkg/abi/linux/ptrace_arm64.go @@ -17,6 +17,10 @@ package linux +import ( + "structs" +) + const ( // PSR bits PSR_MODE_EL0t = 0x00000000 @@ -54,6 +58,7 @@ const ( // +marshal // +stateify savable type PtraceRegs struct { + _ structs.HostLayout Regs [31]uint64 Sp uint64 Pc uint64 diff --git a/pkg/abi/linux/rseq.go b/pkg/abi/linux/rseq.go index 76253ba..e98e8b4 100644 --- a/pkg/abi/linux/rseq.go +++ b/pkg/abi/linux/rseq.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Flags passed to rseq(2). // // Defined in include/uapi/linux/rseq.h. @@ -45,6 +49,7 @@ const ( // // +marshal type RSeqCriticalSection struct { + _ structs.HostLayout // Version is the version of this structure. Version 0 is defined here. Version uint32 @@ -88,6 +93,7 @@ const ( // // In userspace, this structure is always aligned to 32 bytes. type RSeq struct { + _ structs.HostLayout // CPUIDStart contains the current CPU ID if rseq is initialized. // // This field should only be read by the thread which registered this diff --git a/pkg/abi/linux/rusage.go b/pkg/abi/linux/rusage.go index e29d0ac..8ebb353 100644 --- a/pkg/abi/linux/rusage.go +++ b/pkg/abi/linux/rusage.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // Flags that may be used with wait4(2) and getrusage(2). const ( // wait4(2) uses this to aggregate RUSAGE_SELF and RUSAGE_CHILDREN. @@ -29,6 +33,7 @@ const ( // // +marshal type Rusage struct { + _ structs.HostLayout UTime Timeval STime Timeval MaxRSS int64 diff --git a/pkg/abi/linux/sched.go b/pkg/abi/linux/sched.go index 2a67921..163bab8 100644 --- a/pkg/abi/linux/sched.go +++ b/pkg/abi/linux/sched.go @@ -14,6 +14,8 @@ package linux +import "structs" + // Scheduling policies, exposed by sched_getscheduler(2)/sched_setscheduler(2). const ( SCHED_NORMAL = 0 @@ -35,3 +37,75 @@ const ( PRIO_PROCESS = 0x0 PRIO_USER = 0x2 ) + +// SchedAttr represents struct sched_attr, as used by sched_setattr(2) and sched_getattr(2). +// +// +marshal +type SchedAttr struct { + _ structs.HostLayout + + Size uint32 + + SchedPolicy uint32 + SchedFlags uint64 + SchedNice int32 + SchedPriority uint32 + + // For SCHED_DEADLINE + SchedRuntime uint64 + SchedDeadline uint64 + SchedPeriod uint64 + + // Utilization hints + SchedUtilMin uint32 + SchedUtilMax uint32 +} + +// Sizes for different versions of the SchedAttr struct. +const ( + SCHED_ATTR_SIZE_VER0 = 48 + SCHED_ATTR_SIZE_VER1 = 56 + + SCHED_ATTR_SIZE_LATEST = SCHED_ATTR_SIZE_VER1 +) + +// Flags for sched_setattr. +const ( + SCHED_FLAG_RESET_ON_FORK = 0x01 + SCHED_FLAG_RECLAIM = 0x02 + SCHED_FLAG_DL_OVERRUN = 0x04 + SCHED_FLAG_KEEP_POLICY = 0x08 + SCHED_FLAG_KEEP_PARAMS = 0x10 + SCHED_FLAG_UTIL_CLAMP_MIN = 0x20 + SCHED_FLAG_UTIL_CLAMP_MAX = 0x40 +) + +// I/O priority target types. +const ( + IOPRIO_WHO_PROCESS = 1 + IOPRIO_WHO_PGRP = 2 + IOPRIO_WHO_USER = 3 +) + +// I/O priority classes. +const ( + IOPRIO_CLASS_NONE = 0 + IOPRIO_CLASS_RT = 1 + IOPRIO_CLASS_BE = 2 + IOPRIO_CLASS_IDLE = 3 +) + +// I/O priority bitwise encoding constants. +const ( + IOPRIO_CLASS_SHIFT = 13 + IOPRIO_NR_CLASSES = 8 + IOPRIO_CLASS_MASK = IOPRIO_NR_CLASSES - 1 + IOPRIO_PRIO_MASK = (1 << IOPRIO_CLASS_SHIFT) - 1 +) + +// UnwrapIOPrio unwraps the bitmask ioprio into its enclosed ioclass and data fields. +func UnwrapIOPrio(ioprio int) (ioclass int8, iopriodata uint16) { + ioclass = int8((ioprio >> IOPRIO_CLASS_SHIFT) & (IOPRIO_CLASS_MASK)) + iopriodata = uint16(ioprio & IOPRIO_PRIO_MASK) + return +} diff --git a/pkg/abi/linux/seccomp.go b/pkg/abi/linux/seccomp.go index 81c307f..4714f23 100644 --- a/pkg/abi/linux/seccomp.go +++ b/pkg/abi/linux/seccomp.go @@ -14,7 +14,10 @@ package linux -import "fmt" +import ( + "fmt" + "structs" +) // Seccomp constants taken from . const ( @@ -103,6 +106,7 @@ func (a BPFAction) WithReturnCode(code uint16) BPFAction { // SockFprog is sock_fprog taken from . type SockFprog struct { + _ structs.HostLayout Len uint16 pad [6]byte Filter *BPFInstruction @@ -113,6 +117,7 @@ type SockFprog struct { // // +marshal type SeccompData struct { + _ structs.HostLayout // Nr is the system call number. Nr int32 @@ -131,6 +136,7 @@ type SeccompData struct { // // +marshal type SeccompNotifResp struct { + _ structs.HostLayout ID uint64 Val int64 Error int32 @@ -141,6 +147,7 @@ type SeccompNotifResp struct { // // +marshal type SeccompNotifSizes struct { + _ structs.HostLayout Notif uint16 Notif_resp uint16 Data uint16 @@ -150,6 +157,7 @@ type SeccompNotifSizes struct { // // +marshal type SeccompNotif struct { + _ structs.HostLayout ID uint64 Pid int32 Flags uint32 diff --git a/pkg/abi/linux/sem.go b/pkg/abi/linux/sem.go index a9c708d..db48812 100644 --- a/pkg/abi/linux/sem.go +++ b/pkg/abi/linux/sem.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + // semctl Command Definitions. Source: include/uapi/linux/sem.h const ( GETPID = 11 @@ -58,6 +62,7 @@ const ( // // +marshal slice:SembufSlice type Sembuf struct { + _ structs.HostLayout SemNum uint16 SemOp int16 SemFlg int16 @@ -69,6 +74,7 @@ type Sembuf struct { // // +marshal type SemInfo struct { + _ structs.HostLayout SemMap uint32 SemMni uint32 SemMns uint32 diff --git a/pkg/abi/linux/sem_amd64.go b/pkg/abi/linux/sem_amd64.go index cabd2d4..d38af0b 100644 --- a/pkg/abi/linux/sem_amd64.go +++ b/pkg/abi/linux/sem_amd64.go @@ -17,12 +17,17 @@ package linux +import ( + "structs" +) + // SemidDS is equivalent to struct semid64_ds. // // Source: arch/x86/include/uapi/asm/sembuf.h // // +marshal type SemidDS struct { + _ structs.HostLayout SemPerm IPCPerm SemOTime TimeT unused1 uint64 diff --git a/pkg/abi/linux/sem_arm64.go b/pkg/abi/linux/sem_arm64.go index a0c467d..93d80aa 100644 --- a/pkg/abi/linux/sem_arm64.go +++ b/pkg/abi/linux/sem_arm64.go @@ -17,12 +17,17 @@ package linux +import ( + "structs" +) + // SemidDS is equivalent to struct semid64_ds. // // Source: include/uapi/asm-generic/sembuf.h // // +marshal type SemidDS struct { + _ structs.HostLayout SemPerm IPCPerm SemOTime TimeT SemCTime TimeT diff --git a/pkg/abi/linux/shm.go b/pkg/abi/linux/shm.go index 77945bd..5b63646 100644 --- a/pkg/abi/linux/shm.go +++ b/pkg/abi/linux/shm.go @@ -14,7 +14,10 @@ package linux -import "math" +import ( + "math" + "structs" +) // shmat(2) flags. Source: include/uapi/linux/shm.h const ( @@ -54,6 +57,7 @@ const ( // // +marshal type ShmidDS struct { + _ structs.HostLayout ShmPerm IPCPerm ShmSegsz uint64 ShmAtime TimeT @@ -71,6 +75,7 @@ type ShmidDS struct { // // +marshal type ShmParams struct { + _ structs.HostLayout ShmMax uint64 ShmMin uint64 ShmMni uint64 @@ -82,6 +87,7 @@ type ShmParams struct { // // +marshal type ShmInfo struct { + _ structs.HostLayout UsedIDs int32 // Number of currently existing segments. _ [4]byte ShmTot uint64 // Total number of shared memory pages. diff --git a/pkg/abi/linux/signal.go b/pkg/abi/linux/signal.go index 1c2795c..4e81b20 100644 --- a/pkg/abi/linux/signal.go +++ b/pkg/abi/linux/signal.go @@ -15,6 +15,8 @@ package linux import ( + "structs" + "github.com/sagernet/gvisor/pkg/bits" "github.com/sagernet/gvisor/pkg/hostarch" ) @@ -130,12 +132,12 @@ func MakeSignalSet(sigs ...Signal) SignalSet { for i, sig := range sigs { indices[i] = sig.Index() } - return SignalSet(bits.Mask64(indices...)) + return bits.Mask[SignalSet](indices...) } // SignalSetOf returns a SignalSet with a single signal set. func SignalSetOf(sig Signal) SignalSet { - return SignalSet(bits.MaskOf64(sig.Index())) + return bits.MaskOf[SignalSet](sig.Index()) } // ForEachSignal invokes f for each signal set in the given mask. @@ -291,6 +293,7 @@ const ( // // +marshal type Sigevent struct { + _ structs.HostLayout Value uint64 // union sigval {int, void*} Signo int32 Notify int32 @@ -306,6 +309,7 @@ type Sigevent struct { // +marshal // +stateify savable type SigAction struct { + _ structs.HostLayout Handler uint64 Flags uint64 Restorer uint64 @@ -318,6 +322,7 @@ type SigAction struct { // +marshal // +stateify savable type SignalStack struct { + _ structs.HostLayout Addr uint64 Flags uint32 _ uint32 @@ -345,6 +350,7 @@ func (s *SignalStack) IsEnabled() bool { // +marshal // +stateify savable type SignalInfo struct { + _ structs.HostLayout Signo int32 // Signal number Errno int32 // Errno value Code int32 // Signal code diff --git a/pkg/abi/linux/signalfd.go b/pkg/abi/linux/signalfd.go index 8608972..d2cb1ec 100644 --- a/pkg/abi/linux/signalfd.go +++ b/pkg/abi/linux/signalfd.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + const ( // SFD_NONBLOCK is a signalfd(2) flag. SFD_NONBLOCK = 0o0004000 @@ -26,6 +30,7 @@ const ( // // +marshal type SignalfdSiginfo struct { + _ structs.HostLayout Signo uint32 Errno int32 Code int32 diff --git a/pkg/abi/linux/socket.go b/pkg/abi/linux/socket.go index a131995..21ed7ea 100644 --- a/pkg/abi/linux/socket.go +++ b/pkg/abi/linux/socket.go @@ -15,6 +15,8 @@ package linux import ( + "structs" + "github.com/sagernet/gvisor/pkg/marshal" ) @@ -174,6 +176,7 @@ const ( // // +marshal type TpacketReq struct { + _ structs.HostLayout TpBlockSize uint32 TpBlockNr uint32 TpFrameSize uint32 @@ -185,6 +188,7 @@ type TpacketReq struct { // // +marshal type TpacketHdr struct { + _ structs.HostLayout TpStatus uint64 TpLen uint32 TpSnaplen uint32 @@ -200,6 +204,7 @@ type TpacketHdr struct { // // +marshal type Tpacket2Hdr struct { + _ structs.HostLayout TpStatus uint32 TpLen uint32 TpSnaplen uint32 @@ -217,6 +222,7 @@ type Tpacket2Hdr struct { // // +marshal type TpacketStats struct { + _ structs.HostLayout Packets uint32 Dropped uint32 } @@ -373,6 +379,7 @@ var SizeOfInetAddr = uint32((*InetAddr)(nil).SizeBytes()) // // +marshal type SockAddrInet struct { + _ structs.HostLayout Family uint16 Port uint16 Addr InetAddr @@ -383,6 +390,7 @@ type SockAddrInet struct { // // +marshal type Inet6MulticastRequest struct { + _ structs.HostLayout MulticastAddr Inet6Addr InterfaceIndex int32 } @@ -391,6 +399,7 @@ type Inet6MulticastRequest struct { // // +marshal type InetMulticastRequest struct { + _ structs.HostLayout MulticastAddr InetAddr InterfaceAddr InetAddr } @@ -399,6 +408,7 @@ type InetMulticastRequest struct { // // +marshal type InetMulticastRequestWithNIC struct { + _ structs.HostLayout InetMulticastRequest InterfaceIndex int32 } @@ -412,6 +422,7 @@ type Inet6Addr [16]byte // // +marshal type SockAddrInet6 struct { + _ structs.HostLayout Family uint16 Port uint16 Flowinfo uint32 @@ -423,6 +434,7 @@ type SockAddrInet6 struct { // // +marshal type SockAddrLink struct { + _ structs.HostLayout Family uint16 Protocol uint16 InterfaceIndex int32 @@ -441,6 +453,7 @@ const UnixPathMax = 108 // // +marshal type SockAddrUnix struct { + _ structs.HostLayout Family uint16 Path [UnixPathMax]int8 } @@ -466,6 +479,7 @@ func (s *SockAddrNetlink) implementsSockAddr() {} // // +marshal type Linger struct { + _ structs.HostLayout OnOff int32 Linger int32 } @@ -482,6 +496,7 @@ const SizeOfLinger = 8 // // +marshal type TCPInfo struct { + _ structs.HostLayout // State is the state of the connection. State uint8 @@ -640,6 +655,7 @@ const ( // // +marshal type ControlMessageHeader struct { + _ structs.HostLayout Length uint64 Level int32 Type int32 @@ -655,6 +671,7 @@ var SizeOfControlMessageHeader = (*ControlMessageHeader)(nil).SizeBytes() // // +marshal type ControlMessageCredentials struct { + _ structs.HostLayout PID int32 UID uint32 GID uint32 @@ -667,6 +684,7 @@ type ControlMessageCredentials struct { // +marshal // +stateify savable type ControlMessageIPPacketInfo struct { + _ structs.HostLayout NIC int32 LocalAddr InetAddr DestinationAddr InetAddr @@ -677,6 +695,7 @@ type ControlMessageIPPacketInfo struct { // +marshal // +stateify savable type ControlMessageIPv6PacketInfo struct { + _ structs.HostLayout Addr Inet6Addr NIC uint32 } @@ -728,6 +747,7 @@ const SO_ACCEPTCON = 1 << 16 // +marshal // +stateify savable type ICMP6Filter struct { + _ structs.HostLayout Filter [8]uint32 } diff --git a/pkg/abi/linux/time.go b/pkg/abi/linux/time.go index 09407b7..4f6ada9 100644 --- a/pkg/abi/linux/time.go +++ b/pkg/abi/linux/time.go @@ -16,6 +16,7 @@ package linux import ( "math" + "structs" "time" ) @@ -86,7 +87,10 @@ const ( TFD_NONBLOCK = O_NONBLOCK // TFD_TIMER_ABSTIME is a timerfd_settime flag. - TFD_TIMER_ABSTIME = 1 + TFD_TIMER_ABSTIME = 1 << 0 + + // TFD_TIMER_CANCEL_ON_SET is a timerfd_settime flag. + TFD_TIMER_CANCEL_ON_SET = 1 << 1 ) // The safe number of seconds you can represent by int64. @@ -106,6 +110,7 @@ func NsecToTimeT(nsec int64) TimeT { // // +marshal slice:TimespecSlice type Timespec struct { + _ structs.HostLayout Sec int64 Nsec int64 } @@ -162,6 +167,7 @@ const SizeOfTimeval = 16 // // +marshal slice:TimevalSlice type Timeval struct { + _ structs.HostLayout Sec int64 Usec int64 } @@ -201,10 +207,16 @@ func DurationToTimeval(dur time.Duration) Timeval { // // +marshal type Itimerspec struct { + _ structs.HostLayout Interval Timespec Value Timespec } +// Valid returns whether the itimerspec contains valid values. +func (its Itimerspec) Valid() bool { + return its.Interval.Valid() && its.Value.Valid() +} + // ItimerVal mimics the following struct in // // struct itimerval { @@ -214,6 +226,7 @@ type Itimerspec struct { // // +marshal type ItimerVal struct { + _ structs.HostLayout Interval Timeval Value Timeval } @@ -232,6 +245,7 @@ func ClockTFromDuration(d time.Duration) ClockT { // // +marshal type Tms struct { + _ structs.HostLayout UTime ClockT STime ClockT CUTime ClockT @@ -248,6 +262,7 @@ type TimerID int32 // // +marshal type StatxTimestamp struct { + _ structs.HostLayout Sec int64 Nsec uint32 _ int32 @@ -283,6 +298,7 @@ func (sxts StatxTimestamp) ToTime() time.Time { // // +marshal type Utime struct { + _ structs.HostLayout Actime int64 Modtime int64 } diff --git a/pkg/abi/linux/tty.go b/pkg/abi/linux/tty.go index 048d878..b29472d 100644 --- a/pkg/abi/linux/tty.go +++ b/pkg/abi/linux/tty.go @@ -14,6 +14,10 @@ package linux +import ( + "structs" +) + const ( // NumControlCharacters is the number of control characters in Termios. NumControlCharacters = 19 @@ -24,8 +28,10 @@ const ( // Winsize is struct winsize, defined in uapi/asm-generic/termios.h. // +// +stateify savable // +marshal type Winsize struct { + _ structs.HostLayout Row uint16 Col uint16 Xpixel uint16 @@ -36,6 +42,7 @@ type Winsize struct { // // +marshal type Termios struct { + _ structs.HostLayout InputFlags uint32 OutputFlags uint32 ControlFlags uint32 @@ -48,7 +55,9 @@ type Termios struct { // uapi/asm-generic/termbits.h. // // +stateify savable +// +marshal type KernelTermios struct { + _ structs.HostLayout InputFlags uint32 OutputFlags uint32 ControlFlags uint32 @@ -336,14 +345,3 @@ var DefaultReplicaTermios = KernelTermios{ InputSpeed: 38400, OutputSpeed: 38400, } - -// WindowSize corresponds to struct winsize defined in -// include/uapi/asm-generic/termios.h. -// -// +stateify savable -// +marshal -type WindowSize struct { - Rows uint16 - Cols uint16 - _ [4]byte // Padding for 2 unused shorts. -} diff --git a/pkg/abi/linux/utsname.go b/pkg/abi/linux/utsname.go index cb7c954..8012beb 100644 --- a/pkg/abi/linux/utsname.go +++ b/pkg/abi/linux/utsname.go @@ -17,6 +17,7 @@ package linux import ( "bytes" "fmt" + "structs" ) const ( @@ -29,6 +30,7 @@ const ( // // +marshal type UtsName struct { + _ structs.HostLayout Sysname [UTSLen + 1]byte Nodename [UTSLen + 1]byte Release [UTSLen + 1]byte diff --git a/pkg/abi/linux/vfio.go b/pkg/abi/linux/vfio.go index ed821a6..b39c3a4 100644 --- a/pkg/abi/linux/vfio.go +++ b/pkg/abi/linux/vfio.go @@ -16,6 +16,10 @@ package linux +import ( + "structs" +) + // For IOCTLs requests from include/uapi/linux/vfio.h. const ( VFIO_TYPE = ';' @@ -131,15 +135,26 @@ var ( // // +marshal type VFIODeviceInfo struct { + _ structs.HostLayout + VFIODeviceInfoMin + // Offset within info struct of first cap. + CapOffset uint32 + pad uint32 +} + +// VFIODeviceInfoMin is the subset of vfio_device_info (from +// include/uapi/linux/vfio.h) that is copied into +// drivers/vfio/pci/vfio_pci_core.c:vfio_pci_ioctl_get_info(). +// +// +marshal +type VFIODeviceInfoMin struct { + _ structs.HostLayout Argsz uint32 Flags uint32 // The total amount of regions. NumRegions uint32 // The maximum number of IRQ. NumIrqs uint32 - // Offset within info struct of first cap. - CapOffset uint32 - pad uint32 } // VFIORegionInfo is analogous to vfio_region_info @@ -147,11 +162,12 @@ type VFIODeviceInfo struct { // // +marshal type VFIORegionInfo struct { + _ structs.HostLayout Argsz uint32 Flags uint32 Index uint32 // Offset within info struct of first cap. - capOffset uint32 + CapOffset uint32 // Region size in bytes. Size uint64 // Region offset from start of device fd. @@ -163,6 +179,7 @@ type VFIORegionInfo struct { // // +marshal type VFIOIrqInfo struct { + _ structs.HostLayout Argsz uint32 Flags uint32 Index uint32 @@ -176,6 +193,7 @@ type VFIOIrqInfo struct { // // +marshal type VFIOIrqSet struct { + _ structs.HostLayout Argsz uint32 Flags uint32 Index uint32 @@ -188,6 +206,7 @@ type VFIOIrqSet struct { // // +marshal type VFIOIommuType1DmaMap struct { + _ structs.HostLayout Argsz uint32 Flags uint32 // Process virtual address. @@ -203,6 +222,7 @@ type VFIOIommuType1DmaMap struct { // // +marshal type VFIOIommuType1DmaUnmap struct { + _ structs.HostLayout Argsz uint32 Flags uint32 // IO virtual address. diff --git a/pkg/abi/linux/wait.go b/pkg/abi/linux/wait.go index 7107291..131fe44 100644 --- a/pkg/abi/linux/wait.go +++ b/pkg/abi/linux/wait.go @@ -34,9 +34,10 @@ const ( // ID types for waitid(2), from include/uapi/linux/wait.h. const ( - P_ALL = 0x0 - P_PID = 0x1 - P_PGID = 0x2 + P_ALL = 0x0 + P_PID = 0x1 + P_PGID = 0x2 + P_PIDFD = 0x3 ) // WaitStatus represents a thread status, as returned by the wait* family of @@ -84,7 +85,8 @@ func (ws WaitStatus) Exited() bool { // with WIFSIGNALED. func (ws WaitStatus) Signaled() bool { // ws&0x7f != 0 (exited) and ws&0x7f != 0x7f (stopped or continued) - return ((ws&0x7f)+1)>>1 != 0 + bits := ws & 0x7f + return bits != 0 && bits != 0x7f } // CoreDumped returns true if ws indicates that a core dump was produced, diff --git a/pkg/abi/linux/xattr.go b/pkg/abi/linux/xattr.go index 6d6606e..937c57c 100644 --- a/pkg/abi/linux/xattr.go +++ b/pkg/abi/linux/xattr.go @@ -14,6 +14,11 @@ package linux +import ( + "encoding/binary" + "structs" +) + // Constants for extended attributes. const ( XATTR_NAME_MAX = 255 @@ -37,3 +42,112 @@ const ( XATTR_USER_PREFIX = "user." XATTR_USER_PREFIX_LEN = len(XATTR_USER_PREFIX) ) + +// Constants for POSIX ACL extended attributes. +const ( + // Extended attribute names for POSIX ACLs. + XATTR_NAME_POSIX_ACL_ACCESS = XATTR_SYSTEM_PREFIX + "posix_acl_access" + XATTR_NAME_POSIX_ACL_DEFAULT = XATTR_SYSTEM_PREFIX + "posix_acl_default" + + POSIX_ACL_XATTR_VERSION = 2 + + // ACL_UNDEFINED_ID is the ID for entries that do not contain a + // named user or group. + ACL_UNDEFINED_ID = 0xffffffff + + // ACL entry tags. + ACL_USER_OBJ = 0x01 + ACL_USER = 0x02 + ACL_GROUP_OBJ = 0x04 + ACL_GROUP = 0x08 + ACL_MASK = 0x10 + ACL_OTHER = 0x20 + + // ACL entry permission bits. + ACL_READ = 0x04 + ACL_WRITE = 0x02 + ACL_EXECUTE = 0x01 +) + +// PosixACLXattrEntry is a single entry in the userspace representation +// of a POSIX ACL. It corresponds to Linux's struct posix_acl_xattr_entry. +// +// All fields in PosixACLXattrEntry are stored as little-endian. +// +// +marshal dynamic +type PosixACLXattrEntry struct { + _ structs.HostLayout + Tag uint16 + Perm uint16 + ID uint32 +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (a *PosixACLXattrEntry) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (a *PosixACLXattrEntry) MarshalBytes(dst []byte) []byte { + binary.LittleEndian.PutUint16(dst[0:], a.Tag) + binary.LittleEndian.PutUint16(dst[2:], a.Perm) + binary.LittleEndian.PutUint32(dst[4:], a.ID) + + return dst[8:] +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (a *PosixACLXattrEntry) UnmarshalBytes(src []byte) []byte { + a.Tag = binary.LittleEndian.Uint16(src[0:]) + a.Perm = binary.LittleEndian.Uint16(src[2:]) + a.ID = binary.LittleEndian.Uint32(src[4:]) + + return src[8:] +} + +// PosixACLXattr is the userspace representation of a POSIX ACL. +// +// +marshal dynamic +type PosixACLXattr struct { + _ structs.HostLayout + + // Version is the POSIX ACL version, stored as little-endian. + Version uint32 + + // Entries contains the ACL entries. + Entries []PosixACLXattrEntry `hostlayout:"ignore"` +} + +// posixACLXattrHeaderSize is the size in bytes of the header. +const posixACLXattrHeaderSize = 4 + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (a *PosixACLXattr) SizeBytes() int { + return posixACLXattrHeaderSize + len(a.Entries)*(*PosixACLXattrEntry)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (a *PosixACLXattr) MarshalBytes(dst []byte) []byte { + binary.LittleEndian.PutUint32(dst, a.Version) + + dst = dst[posixACLXattrHeaderSize:] + for _, entry := range a.Entries { + dst = entry.MarshalBytes(dst) + } + + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (a *PosixACLXattr) UnmarshalBytes(src []byte) []byte { + a.Version = binary.LittleEndian.Uint32(src) + + src = src[posixACLXattrHeaderSize:] + for len(src) >= (*PosixACLXattrEntry)(nil).SizeBytes() { + var entry PosixACLXattrEntry + src = entry.UnmarshalBytes(src) + a.Entries = append(a.Entries, entry) + } + + return src +} diff --git a/pkg/abi/nvgpu/classes.go b/pkg/abi/nvgpu/classes.go index b186874..5fcb6bc 100644 --- a/pkg/abi/nvgpu/classes.go +++ b/pkg/abi/nvgpu/classes.go @@ -16,6 +16,7 @@ package nvgpu import ( "fmt" + "structs" ) // ClassID is a client class ID, in the sense of @@ -58,7 +59,10 @@ const ( NV01_DEVICE_0 = 0x00000080 NV_SEMAPHORE_SURFACE = 0x000000da RM_USER_SHARED_DATA = 0x000000de + NV_MEMORY_EXPORT = 0x000000e0 + NV_IMEX_SESSION = 0x000000f1 NV_MEMORY_FABRIC = 0x000000f8 + NV_MEMORY_FABRIC_IMPORTED_REF = 0x000000fb NV_MEMORY_MULTICAST_FABRIC = 0x000000fd NV_MEMORY_MAPPER = 0x000000fe NV20_SUBDEVICE_0 = 0x00002080 @@ -67,56 +71,72 @@ const ( NV50_THIRD_PARTY_P2P = 0x0000503c NV50_MEMORY_VIRTUAL = 0x000050a0 GT200_DEBUGGER = 0x000083de + MPS_COMPUTE = 0x0000900e FERMI_TWOD_A = 0x0000902d FERMI_CONTEXT_SHARE_A = 0x00009067 GF100_DISP_SW = 0x00009072 GF100_ZBC_CLEAR = 0x00009096 + GF100_SUBDEVICE_INFOROM = 0x000090e7 GF100_PROFILER = 0x000090cc + MAXWELL_PROFILER_DEVICE = 0x0000b2cc + NV_COUNTER_COLLECTION_UNIT = 0x0000cbca GF100_SUBDEVICE_MASTER = 0x000090e6 FERMI_VASPACE_A = 0x000090f1 KEPLER_CHANNEL_GROUP_A = 0x0000a06c NVENC_SW_SESSION = 0x0000a0bc KEPLER_INLINE_TO_MEMORY_B = 0x0000a140 + NVB8B0_VIDEO_DECODER = 0x0000b8b0 + NVB8D1_VIDEO_NVJPG = 0x0000b8d1 + NVB8FA_VIDEO_OFA = 0x0000b8fa VOLTA_USERMODE_A = 0x0000c361 TURING_USERMODE_A = 0x0000c461 TURING_CHANNEL_GPFIFO_A = 0x0000c46f - NVB8B0_VIDEO_DECODER = 0x0000b8b0 // Hopper - NVC4B0_VIDEO_DECODER = 0x0000c4b0 // Turing - NVC6B0_VIDEO_DECODER = 0x0000c6b0 // Ampere - NVC7B0_VIDEO_DECODER = 0x0000c7b0 // Ampere - NVC9B0_VIDEO_DECODER = 0x0000c9b0 // Ada - NVCDB0_VIDEO_DECODER = 0x0000cdb0 // Blackwell + NVC4B0_VIDEO_DECODER = 0x0000c4b0 NVC4B7_VIDEO_ENCODER = 0x0000c4b7 - NVC7B7_VIDEO_ENCODER = 0x0000c7b7 - NVC9B7_VIDEO_ENCODER = 0x0000c9b7 + NVC4D1_VIDEO_NVJPG = 0x0000c4d1 AMPERE_CHANNEL_GPFIFO_A = 0x0000c56f TURING_A = 0x0000c597 TURING_DMA_COPY_A = 0x0000c5b5 TURING_COMPUTE_A = 0x0000c5c0 HOPPER_USERMODE_A = 0x0000c661 - BLACKWELL_USERMODE_A = 0x0000c761 AMPERE_A = 0x0000c697 + NVC6B0_VIDEO_DECODER = 0x0000c6b0 AMPERE_DMA_COPY_A = 0x0000c6b5 AMPERE_COMPUTE_A = 0x0000c6c0 + NVC6FA_VIDEO_OFA = 0x0000c6fa + BLACKWELL_USERMODE_A = 0x0000c761 + NVC7B0_VIDEO_DECODER = 0x0000c7b0 AMPERE_DMA_COPY_B = 0x0000c7b5 + NVC7B7_VIDEO_ENCODER = 0x0000c7b7 AMPERE_COMPUTE_B = 0x0000c7c0 + NVC7FA_VIDEO_OFA = 0x0000c7fa HOPPER_CHANNEL_GPFIFO_A = 0x0000c86f HOPPER_DMA_COPY_A = 0x0000c8b5 BLACKWELL_CHANNEL_GPFIFO_A = 0x0000c96f + ADA_A = 0x0000c997 + NVC9B0_VIDEO_DECODER = 0x0000c9b0 BLACKWELL_DMA_COPY_A = 0x0000c9b5 + NVC9B7_VIDEO_ENCODER = 0x0000c9b7 + ADA_COMPUTE_A = 0x0000c9c0 + NVC9D1_VIDEO_NVJPG = 0x0000c9d1 + NVC9FA_VIDEO_OFA = 0x0000c9fa BLACKWELL_CHANNEL_GPFIFO_B = 0x0000ca6f BLACKWELL_DMA_COPY_B = 0x0000cab5 - ADA_A = 0x0000c997 - ADA_COMPUTE_A = 0x0000c9c0 NV_CONFIDENTIAL_COMPUTE = 0x0000cb33 HOPPER_A = 0x0000cb97 HOPPER_SEC2_WORK_LAUNCH_A = 0x0000cba2 HOPPER_COMPUTE_A = 0x0000cbc0 BLACKWELL_INLINE_TO_MEMORY_A = 0x0000cd40 BLACKWELL_A = 0x0000cd97 + NVCDB0_VIDEO_DECODER = 0x0000cdb0 BLACKWELL_COMPUTE_A = 0x0000cdc0 + NVCDD1_VIDEO_NVJPG = 0x0000cdd1 + NVCDFA_VIDEO_OFA = 0x0000cdfa BLACKWELL_B = 0x0000ce97 + NVCEB7_VIDEO_ENCODER = 0x0000ceb7 BLACKWELL_COMPUTE_B = 0x0000cec0 + NVCFB7_VIDEO_ENCODER = 0x0000cfb7 + NVD1B7_VIDEO_ENCODER = 0x0000d1b7 ) // From src/common/sdk/nvidia/inc/class/cl0000.h: @@ -129,6 +149,7 @@ const ( // // +marshal type NV2081_ALLOC_PARAMETERS struct { + _ structs.HostLayout Reserved uint32 } @@ -137,6 +158,7 @@ type NV2081_ALLOC_PARAMETERS struct { // // +marshal type NV0005_ALLOC_PARAMETERS struct { + _ structs.HostLayout HParentClient Handle HSrcResource Handle HClass ClassID @@ -159,6 +181,7 @@ const ( // // +marshal type NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS struct { + _ structs.HostLayout Offset uint64 Limit uint64 HVASpace Handle @@ -175,6 +198,7 @@ const ( // // +marshal type NV0080_ALLOC_PARAMETERS struct { + _ structs.HostLayout DeviceID uint32 HClientShare Handle HTargetClient Handle @@ -193,6 +217,7 @@ type NV0080_ALLOC_PARAMETERS struct { // // +marshal type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS struct { + _ structs.HostLayout HSemaphoreMem Handle HMaxSubmittedMem Handle flags uint64 @@ -203,6 +228,7 @@ type NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS struct { // // +marshal type NV2080_ALLOC_PARAMETERS struct { + _ structs.HostLayout SubDeviceID uint32 } @@ -211,6 +237,7 @@ type NV2080_ALLOC_PARAMETERS struct { // // +marshal type NV_CONTEXT_DMA_ALLOCATION_PARAMS struct { + _ structs.HostLayout HSubDevice Handle Flags uint32 HMemory Handle @@ -224,6 +251,7 @@ type NV_CONTEXT_DMA_ALLOCATION_PARAMS struct { // // +marshal type NV_MEMORY_ALLOCATION_PARAMS struct { + _ structs.HostLayout Owner uint32 Type uint32 Flags uint32 @@ -254,6 +282,7 @@ type NV_MEMORY_ALLOCATION_PARAMS struct { // // +marshal type NV_MEMORY_ALLOCATION_PARAMS_V545 struct { + _ structs.HostLayout NV_MEMORY_ALLOCATION_PARAMS NumaNode int32 _ uint32 @@ -263,6 +292,7 @@ type NV_MEMORY_ALLOCATION_PARAMS_V545 struct { // // +marshal type NV503B_BAR1_P2P_DMA_INFO struct { + _ structs.HostLayout DmaAddress uint64 DmaSize uint64 } @@ -272,6 +302,7 @@ type NV503B_BAR1_P2P_DMA_INFO struct { // // +marshal type NV503B_ALLOC_PARAMETERS struct { + _ structs.HostLayout HSubDevice Handle HPeerSubDevice Handle SubDevicePeerIDMask uint32 @@ -285,11 +316,31 @@ type NV503B_ALLOC_PARAMETERS struct { P2lBar1P2PDmaInfo NV503B_BAR1_P2P_DMA_INFO } +// NV503B_FABRIC_P2P_DMA_INFO from src/common/sdk/nvidia/inc/class/cl503b.h. +// +// +marshal +type NV503B_FABRIC_P2P_DMA_INFO struct { + _ structs.HostLayout + Gpa uint64 +} + +// NV503B_ALLOC_PARAMETERS_V590 is the updated version of +// NV503B_ALLOC_PARAMETERS since 590.44.01. +// +// +marshal +type NV503B_ALLOC_PARAMETERS_V590 struct { + _ structs.HostLayout + NV503B_ALLOC_PARAMETERS + L2pFabricP2PInfo NV503B_FABRIC_P2P_DMA_INFO + P2lFabricP2PInfo NV503B_FABRIC_P2P_DMA_INFO +} + // NV503C_ALLOC_PARAMETERS is the alloc params type for NV50_THIRD_PARTY_P2P, // from src/common/sdk/nvidia/inc/class/cl503c.h. // // +marshal type NV503C_ALLOC_PARAMETERS struct { + _ structs.HostLayout Flags uint32 } @@ -298,6 +349,7 @@ type NV503C_ALLOC_PARAMETERS struct { // // +marshal type NV83DE_ALLOC_PARAMETERS struct { + _ structs.HostLayout HDebuggerClient_Obsolete Handle HAppClient Handle HClass3DObject Handle @@ -308,6 +360,7 @@ type NV83DE_ALLOC_PARAMETERS struct { // // +marshal type NV_CTXSHARE_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout HVASpace Handle Flags uint32 SubctxID uint32 @@ -318,6 +371,7 @@ type NV_CTXSHARE_ALLOCATION_PARAMETERS struct { // // +marshal type NV_VASPACE_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Index uint32 Flags uint32 VASize uint64 @@ -333,6 +387,7 @@ type NV_VASPACE_ALLOCATION_PARAMETERS struct { // // +marshal type NV_VASPACE_ALLOCATION_PARAMETERS_V580 struct { + _ structs.HostLayout NV_VASPACE_ALLOCATION_PARAMETERS Pasid uint32 Pad1 [4]byte @@ -343,6 +398,7 @@ type NV_VASPACE_ALLOCATION_PARAMETERS_V580 struct { // // +marshal type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout HObjectError Handle HObjectECCError Handle HVASpace Handle @@ -356,6 +412,7 @@ type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS struct { // // +marshal type NV_MEMORY_DESC_PARAMS struct { + _ structs.HostLayout Base uint64 Size uint64 AddressSpace uint32 @@ -365,8 +422,12 @@ type NV_MEMORY_DESC_PARAMS struct { // NV_BSP_ALLOCATION_PARAMETERS is the alloc params type for // NV*VIDEO_DECODER, from src/common/sdk/nvidia/inc/nvos.h. // +// It is also used to represent the following driver structs: +// - NV_NVDEC_ALLOCATION_PARAMETERS +// // +marshal type NV_BSP_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Size uint32 ProhibitMultipleInstances uint32 EngineInstance uint32 @@ -375,8 +436,12 @@ type NV_BSP_ALLOCATION_PARAMETERS struct { // NV_MSENC_ALLOCATION_PARAMETERS is the alloc params type for // NV*_VIDEO_ENCODER, from src/common/sdk/nvidia/inc/nvos.h. // +// It is also used to represent the following driver structs: +// - NV_NVENC_ALLOCATION_PARAMETERS +// // +marshal type NV_MSENC_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Size uint32 ProhibitMultipleInstances uint32 EngineInstance uint32 @@ -388,6 +453,7 @@ type NV_MSENC_ALLOCATION_PARAMETERS struct { // // +marshal type NV_CHANNEL_ALLOC_PARAMS struct { + _ structs.HostLayout HObjectError Handle HObjectBuffer Handle GPFIFOOffset uint64 @@ -421,17 +487,57 @@ type NV_CHANNEL_ALLOC_PARAMS struct { // // +marshal type NV_CHANNEL_ALLOC_PARAMS_V570 struct { + _ structs.HostLayout NV_CHANNEL_ALLOC_PARAMS TPCConfigID uint32 _ uint32 } +// NV_CHANNEL_ALLOC_PARAMS_V610 is the updated version of +// NV_CHANNEL_ALLOC_PARAMS since 610.43.02. +// +// +marshal +type NV_CHANNEL_ALLOC_PARAMS_V610 struct { + _ structs.HostLayout + HObjectError Handle + HObjectBuffer Handle + GPFIFOOffset uint64 + GPFIFOEntries uint32 + Flags uint32 + HContextShare Handle + HVASpace Handle + HHandleVASpace Handle + HUserdMemory [NV_MAX_SUBDEVICES]Handle + _ uint32 + UserdOffset [NV_MAX_SUBDEVICES]uint64 + EngineType uint32 + CID uint32 + SubDeviceID uint32 + HObjectECCError Handle + InstanceMem NV_MEMORY_DESC_PARAMS + UserdMem NV_MEMORY_DESC_PARAMS + RamfcMem NV_MEMORY_DESC_PARAMS + MthdbufMem NV_MEMORY_DESC_PARAMS + HPhysChannelGroup Handle + InternalFlags uint32 + ErrorNotifierMem NV_MEMORY_DESC_PARAMS + ECCErrorNotifierMem NV_MEMORY_DESC_PARAMS + ProcessID uint32 + SubProcessID uint32 + EncryptIv [CC_CHAN_ALLOC_IV_SIZE_DWORD]uint32 + DecryptIv [CC_CHAN_ALLOC_IV_SIZE_DWORD]uint32 + HmacNonce [CC_CHAN_ALLOC_NONCE_SIZE_DWORD]uint32 + TPCConfigID uint32 + _ uint32 +} + // NVB0B5_ALLOCATION_PARAMETERS is the alloc param type for TURING_DMA_COPY_A, // AMPERE_DMA_COPY_A, and AMPERE_DMA_COPY_B from // src/common/sdk/nvidia/inc/class/clb0b5sw.h. // // +marshal type NVB0B5_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Version uint32 EngineType uint32 } @@ -441,6 +547,7 @@ type NVB0B5_ALLOCATION_PARAMETERS struct { // // +marshal type NV_GR_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Version uint32 Flags uint32 Size uint32 @@ -452,6 +559,7 @@ type NV_GR_ALLOCATION_PARAMETERS struct { // // +marshal type NV_HOPPER_USERMODE_A_PARAMS struct { + _ structs.HostLayout Bar1Mapping uint8 Priv uint8 } @@ -461,6 +569,7 @@ type NV_HOPPER_USERMODE_A_PARAMS struct { // // +marshal type NV9072_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout LogicalHeadID uint32 DisplayMask uint32 Caps uint32 @@ -471,6 +580,7 @@ type NV9072_ALLOCATION_PARAMETERS struct { // // +marshal type NV00DE_ALLOC_PARAMETERS struct { + _ structs.HostLayout Reserved uint32 } @@ -479,21 +589,76 @@ type NV00DE_ALLOC_PARAMETERS struct { // // +marshal type NV00DE_ALLOC_PARAMETERS_V545 struct { + _ structs.HostLayout PolledDataMask uint64 } // +marshal type nv00f8Map struct { + _ structs.HostLayout offset uint64 hVidMem Handle flags uint32 } +// From src/common/sdk/nvidia/inc/class/cl00e0.h: +const ( + NV_MEM_EXPORT_UUID_LEN = 16 + NV_MEM_EXPORT_METADATA_LEN = 64 +) + +// NV_EXPORT_MEM_PACKET is from +// src/common/sdk/nvidia/inc/class/cl00e0.h +// +// +marshal +type NV_EXPORT_MEM_PACKET struct { + _ structs.HostLayout + UUID [NV_MEM_EXPORT_UUID_LEN]uint8 + Opaque [16]uint8 +} + +// NV00E0_ALLOCATION_PARAMETERS is the alloc params type for NV_MEMORY_EXPORT, +// from src/common/sdk/nvidia/inc/class/cl00e0.h. +// +// +marshal +type NV00E0_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout + IMEXChannel uint32 + Packet NV_EXPORT_MEM_PACKET + NumMaxHandles uint16 + Pad0 [2]byte + Flags uint32 + Metadata [NV_MEM_EXPORT_METADATA_LEN]uint8 + DeviceInstanceMask uint32 + GIIDMasks [NV_MAX_DEVICES]uint32 + NumCurHandles uint16 + Pad1 [2]byte +} + +// NV00F1_ALLOCATION_PARAMETERS is the alloc params type for NV_IMEX_SESSION, +// from src/common/sdk/nvidia/inc/class/cl00f1.h. +// +// +marshal +type NV00F1_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout + CapDescriptor uint64 + Flags uint32 + Pad0 [4]byte + // Despite being P64-typed by the driver, pOsEvent is a file descriptor + // whose numerical value is used by the driver. See + // src/nvidia/src/kernel/compute/imex_session_api.c:imexsessionapiConstruct_IMPL() + // => src/nvidia/arch/nvalloc/unix/src/os.c:osUserHandleToKernelPtr(). + POsEvent P64 + NodeID uint16 + Pad1 [6]byte +} + // NV00F8_ALLOCATION_PARAMETERS is the alloc param type for NV_MEMORY_FABRIC, // from src/common/sdk/nvidia/inc/class/cl00f8.h. // // +marshal type NV00F8_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Alignment uint64 AllocSize uint64 PageSize uint64 @@ -502,18 +667,32 @@ type NV00F8_ALLOCATION_PARAMETERS struct { Map nv00f8Map } -// From src/common/sdk/nvidia/inc/class/cl00e0.h -const ( - NV_MEM_EXPORT_UUID_LEN = 16 -) - -// NV_EXPORT_MEM_PACKET is from -// src/common/sdk/nvidia/inc/class/cl00e0.h +// NV00FB_ALLOCATION_PARAMETERS is the alloc param type for +// NV_MEMORY_FABRIC_IMPORTED_REF, from +// src/common/sdk/nvidia/inc/class/cl00fb.h. // // +marshal -type NV_EXPORT_MEM_PACKET struct { - UUID [NV_MEM_EXPORT_UUID_LEN]uint8 - Opaque [16]uint8 +type NV00FB_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout + ExportUUID [NV_MEM_EXPORT_UUID_LEN]uint8 + Index uint16 + Pad0 [2]byte + Flags uint32 + ID uint64 +} + +// HasPOsEvent is a type constraint for allocation parameter structs containing +// an optional pOsEvent field. Despite being P64-typed by the driver, pOsEvent +// is a file descriptor whose numerical value is resolved by the driver. So it +// must be translated from a sandbox FD to a host FD. See +// src/nvidia/src/kernel/mem_mgr/mem_multicast_fabric.c:_memMulticastFabricDescriptorEnqueueWait() +// => src/nvidia/arch/nvalloc/unix/src/os.c:osUserHandleToKernelPtr(). +// +// This is necessary because, as of this writing (Go 1.20), there is no way to +// enable field access using a Go type constraint. +type HasPOsEvent interface { + GetPOsEvent() P64 + SetPOsEvent(P64) } // NV00FD_ALLOCATION_PARAMETERS is the alloc param type for NV_MEMORY_MULTICAST_FABRIC @@ -521,13 +700,25 @@ type NV_EXPORT_MEM_PACKET struct { // // +marshal type NV00FD_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout Alignment uint64 AllocSize uint64 PageSize uint32 AllocFlags uint32 NumGPUs uint32 _ uint32 - POsEvent P64 + // POsEvent is an optional file descriptor; see HasPOsEvent. + POsEvent P64 +} + +// GetPOsEvent implements HasPOsEvent.GetPOsEvent. +func (p *NV00FD_ALLOCATION_PARAMETERS) GetPOsEvent() P64 { + return p.POsEvent +} + +// SetPOsEvent implements HasPOsEvent.SetPOsEvent. +func (p *NV00FD_ALLOCATION_PARAMETERS) SetPOsEvent(posEvent P64) { + p.POsEvent = posEvent } // NV00FD_ALLOCATION_PARAMETERS_V545 is the updated version of @@ -535,17 +726,47 @@ type NV00FD_ALLOCATION_PARAMETERS struct { // // +marshal type NV00FD_ALLOCATION_PARAMETERS_V545 struct { + _ structs.HostLayout ExpPacket NV_EXPORT_MEM_PACKET Index uint16 _ [6]byte NV00FD_ALLOCATION_PARAMETERS } +// NV00FD_ALLOCATION_PARAMETERS_V590 is the updated version of +// NV00FD_ALLOCATION_PARAMETERS since 590.44.01. +// +// +marshal +type NV00FD_ALLOCATION_PARAMETERS_V590 struct { + _ structs.HostLayout + ExpPacket NV_EXPORT_MEM_PACKET + Index uint16 + _ [6]byte + Alignment uint64 + AllocSize uint64 + PageSize uint64 + AllocFlags uint32 + NumGPUs uint32 + // POsEvent is an optional file descriptor; see HasPOsEvent. + POsEvent P64 +} + +// GetPOsEvent implements HasPOsEvent.GetPOsEvent. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) GetPOsEvent() P64 { + return p.POsEvent +} + +// SetPOsEvent implements HasPOsEvent.SetPOsEvent. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) SetPOsEvent(posEvent P64) { + p.POsEvent = posEvent +} + // NV_MEMORY_MAPPER_ALLOCATION_PARAMS is the alloc param type for // NV_MEMORY_MAPPER, from src/common/sdk/nvidia/inc/class/cl00fe.h. // // +marshal type NV_MEMORY_MAPPER_ALLOCATION_PARAMS struct { + _ structs.HostLayout unused uint8 } @@ -554,6 +775,7 @@ type NV_MEMORY_MAPPER_ALLOCATION_PARAMS struct { // // +marshal type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 struct { + _ structs.HostLayout HSemaphoreSurface Handle MaxQueueSize uint32 } @@ -563,6 +785,7 @@ type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 struct { // // +marshal type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 struct { + _ structs.HostLayout NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V550 HNotificationMemory Handle _ uint32 @@ -574,6 +797,7 @@ type NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555 struct { // // +marshal type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS struct { + _ structs.HostLayout Handle Handle } @@ -582,9 +806,51 @@ type NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS struct { // // +marshal type NVA0BC_ALLOC_PARAMETERS struct { + _ structs.HostLayout CodecType uint32 HResolution uint32 VResolution uint32 Version uint32 HMem Handle } + +// NV_NVJPG_ALLOCATION_PARAMETERS is the alloc params type for +// NVxxD1_VIDEO_NVJPG classes, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_NVJPG_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout + Size uint32 + ProhibitMultipleInstances uint32 + EngineInstance uint32 +} + +// NV_OFA_ALLOCATION_PARAMETERS is the alloc params type for +// NVxxFA_VIDEO_OFA classes, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_OFA_ALLOCATION_PARAMETERS struct { + _ structs.HostLayout + Size uint32 + ProhibitMultipleInstances uint32 +} + +// NV_OFA_ALLOCATION_PARAMETERS_V545 is the updated version of +// NV_OFA_ALLOCATION_PARAMETERS since 545.23.06. +// +// +marshal +type NV_OFA_ALLOCATION_PARAMETERS_V545 struct { + _ structs.HostLayout + NV_OFA_ALLOCATION_PARAMETERS + EngineInstance uint32 +} + +// NVB2CC_ALLOC_PARAMETERS is the alloc params type for MAXWELL_PROFILER_DEVICE, +// from src/common/sdk/nvidia/inc/class/clb2cc.h. +// +// +marshal +type NVB2CC_ALLOC_PARAMETERS struct { + _ structs.HostLayout + HClientTarget Handle + HContextTarget Handle +} diff --git a/pkg/abi/nvgpu/ctrl.go b/pkg/abi/nvgpu/ctrl.go index 4d3db96..bcc06ae 100644 --- a/pkg/abi/nvgpu/ctrl.go +++ b/pkg/abi/nvgpu/ctrl.go @@ -14,6 +14,10 @@ package nvgpu +import ( + "structs" +) + // From src/nvidia/interface/deprecated/rmapi_deprecated.h: const ( RM_GSS_LEGACY_MASK = 0x00008000 @@ -38,6 +42,7 @@ const ( // // +marshal type NVXXXX_CTRL_XXX_INFO struct { + _ structs.HostLayout Index uint32 Data uint32 } @@ -75,11 +80,15 @@ const ( NV0000_CTRL_CMD_GPU_GET_ACTIVE_DEVICE_IDS = 0x288 NV0000_CTRL_CMD_GPU_ASYNC_ATTACH_ID = 0x289 NV0000_CTRL_CMD_GPU_WAIT_ATTACH_ID = 0x290 + + NV0000_CTRL_GPU_INVALID_ID = 0xffffffff + NV0000_CTRL_GPU_MAX_PROBED_GPUS = NV_MAX_DEVICES + NV0000_GPU_MAX_GID_LENGTH = 0x100 ) -// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gsync.h: +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gpuacct.h: const ( - NV0000_CTRL_CMD_GSYNC_GET_ATTACHED_IDS = 0x301 + NV0000_CTRL_CMD_GPUACCT_GET_ACCOUNTING_STATE = 0xb02 ) // NV0000_CTRL_GPU_GET_ID_INFO_PARAMS is the param type for NV0000_CTRL_CMD_GPU_GET_ID_INFO, @@ -87,6 +96,7 @@ const ( // // +marshal type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS struct { + _ structs.HostLayout GpuID uint32 GpuFlags uint32 DeviceInstance uint32 @@ -98,6 +108,27 @@ type NV0000_CTRL_GPU_GET_ID_INFO_PARAMS struct { NumaID int32 } +// +marshal +type NV0000_CTRL_GPU_ATTACH_IDS_PARAMS struct { + _ structs.HostLayout + GPUIDs [NV0000_CTRL_GPU_MAX_PROBED_GPUS]uint32 + FailedID uint32 +} + +// +marshal +type NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS struct { + _ structs.HostLayout + GPUID uint32 + Flags uint32 + GPUUUID [NV0000_GPU_MAX_GID_LENGTH]byte + UUIDStrLen uint32 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gsync.h: +const ( + NV0000_CTRL_CMD_GSYNC_GET_ATTACHED_IDS = 0x301 +) + // From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000syncgpuboost.h: const ( NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_INFO = 0xa04 @@ -105,15 +136,16 @@ const ( // From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000system.h: const ( - NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION = 0x101 - NV0000_CTRL_CMD_SYSTEM_GET_CPU_INFO = 0x102 - NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS = 0x127 - NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_V2 = 0x12b - NV0000_CTRL_CMD_SYSTEM_GET_FABRIC_STATUS = 0x136 - NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_MATRIX = 0x13a - NV0000_CTRL_CMD_SYSTEM_GET_FEATURES = 0x1f0 - NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS = 32 - NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE = 9 + NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION = 0x101 + NV0000_CTRL_CMD_SYSTEM_GET_CPU_INFO = 0x102 + NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS = 0x127 + NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_V2 = 0x12b + NV0000_CTRL_CMD_SYSTEM_GET_FABRIC_STATUS = 0x136 + NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_MATRIX = 0x13a + NV0000_CTRL_CMD_SYSTEM_GET_FEATURES = 0x1f0 + NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_MAX_STRING_SIZE = 256 + NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS = 32 + NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE = 9 ) // NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS is the param type for NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS, @@ -121,6 +153,7 @@ const ( // // +marshal type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS struct { + _ structs.HostLayout GpuIDs [NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS]uint32 GpuCount uint32 P2PCaps uint32 @@ -136,6 +169,7 @@ type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS struct { // // +marshal type NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_V550 struct { + _ structs.HostLayout NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS BusEgmPeerIDs P64 } @@ -154,6 +188,7 @@ const ( // +marshal type NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS struct { + _ structs.HostLayout FD int32 DeviceInstance uint32 MaxObjects uint16 @@ -173,6 +208,7 @@ func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS) SetFrontendFD(fd int // +marshal type NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545 struct { + _ structs.HostLayout FD int32 DeviceInstance uint32 GpuInstanceID uint32 @@ -193,12 +229,14 @@ func (p *NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_V545) SetFrontendFD(f // +marshal type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT struct { + _ structs.HostLayout Type uint32 // enum NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE Data [12]byte // union } // +marshal type NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS struct { + _ structs.HostLayout Object NV0000_CTRL_OS_UNIX_EXPORT_OBJECT FD int32 Flags uint32 @@ -216,6 +254,7 @@ func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS) SetFrontendFD(fd int32) // +marshal type NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS struct { + _ structs.HostLayout FD int32 Object NV0000_CTRL_OS_UNIX_EXPORT_OBJECT } @@ -232,6 +271,7 @@ func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS) SetFrontendFD(fd int3 // +marshal type NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS struct { + _ structs.HostLayout FD int32 HDevice Handle MaxObjects uint16 @@ -254,6 +294,7 @@ func (p *NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS) SetFrontendFD(fd int32 // +marshal type NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS struct { + _ structs.HostLayout FD int32 HParent Handle Objects [NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS]Handle @@ -274,6 +315,7 @@ func (p *NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS) SetFrontendFD(fd int // +marshal type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS struct { + _ structs.HostLayout SizeOfStrings uint32 Pad [4]byte PDriverVersionBuffer P64 @@ -310,6 +352,7 @@ const ( // +marshal type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS struct { + _ structs.HostLayout NumChannels uint32 Pad [4]byte PChannelHandleList P64 @@ -331,6 +374,7 @@ const ( // // +marshal type RmapiParamNvU32List struct { + _ structs.HostLayout NumElems uint32 Pad [4]byte List P64 @@ -338,9 +382,10 @@ type RmapiParamNvU32List struct { // From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gr.h: const ( - NV0080_CTRL_CMD_GR_GET_CAPS = 0x801102 - NV0080_CTRL_CMD_GR_GET_INFO = 0x801104 - NV0080_CTRL_CMD_GR_GET_CAPS_V2 = 0x801109 + NV0080_CTRL_CMD_GR_GET_CAPS = 0x801102 + NV0080_CTRL_CMD_GR_GET_INFO = 0x801104 + NV0080_CTRL_CMD_GR_SET_TPC_PARTITION_MODE = 0x801108 + NV0080_CTRL_CMD_GR_GET_CAPS_V2 = 0x801109 ) // NV0080_CTRL_GET_CAPS_PARAMS is used to represent the following: @@ -348,9 +393,11 @@ const ( // - NV0080_CTRL_GR_GET_CAPS_PARAMS // - NV0080_CTRL_FIFO_GET_CAPS_PARAMS // - NV0080_CTRL_MSENC_GET_CAPS_PARAMS +// - NV0080_CTRL_NVENC_GET_CAPS_PARAMS // // +marshal type NV0080_CTRL_GET_CAPS_PARAMS struct { + _ structs.HostLayout CapsTblSize uint32 Pad [4]byte CapsTbl P64 @@ -358,6 +405,7 @@ type NV0080_CTRL_GET_CAPS_PARAMS struct { // +marshal type NV0080_CTRL_GR_ROUTE_INFO struct { + _ structs.HostLayout Flags uint32 Pad [4]byte Route uint64 @@ -373,6 +421,7 @@ const ( // // +marshal type NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS struct { + _ structs.HostLayout Result uint32 Checksum uint32 } @@ -386,6 +435,7 @@ type NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS struct { // // +marshal type NvxxxCtrlXxxGetInfoParams struct { + _ structs.HostLayout InfoListSize uint32 Pad [4]byte InfoList P64 @@ -442,21 +492,42 @@ const ( NV00DE_CTRL_CMD_REQUEST_DATA_POLL = 0xde0001 ) +// From src/common/sdk/nvidia/inc/ctrl/ctrl00e0.h: +const ( + NV00E0_CTRL_CMD_IMPORT_MEM = 0xe00102 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl00f1.h: +const ( + NV00F1_CTRL_CMD_GET_FABRIC_EVENTS = 0xf10001 + NV00F1_CTRL_CMD_FINISH_MEM_UNIMPORT = 0xf10002 + NV00F1_CTRL_CMD_DISABLE_IMPORTERS = 0xf10003 +) + // From src/common/sdk/nvidia/inc/ctrl/ctrl00f8.h: const ( + NV00F8_CTRL_CMD_DESCRIBE = 0xf80102 NV00F8_CTRL_CMD_ATTACH_MEM = 0xf80103 ) +// From src/common/sdk/nvidia/inc/ctrl/ctrl00fb.h: +const ( + NV00FB_CTRL_CMD_VALIDATE = 0xfb0101 +) + // From src/common/sdk/nvidia/inc/ctrl/ctrl00fd.h: const ( - NV00FD_CTRL_CMD_GET_INFO = 0xfd0101 - NV00FD_CTRL_CMD_ATTACH_MEM = 0xfd0102 - NV00FD_CTRL_CMD_ATTACH_GPU = 0xfd0104 - NV00FD_CTRL_CMD_DETACH_MEM = 0xfd0105 + NV00FD_CTRL_CMD_GET_INFO = 0xfd0101 + NV00FD_CTRL_CMD_ATTACH_MEM = 0xfd0102 + NV00FD_CTRL_CMD_ATTACH_GPU = 0xfd0104 + NV00FD_CTRL_CMD_DETACH_MEM = 0xfd0105 + NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU = 0xfd0106 + NV00FD_CTRL_CMD_SET_FAILURE = 0xfd0107 ) // +marshal type NV00FD_CTRL_ATTACH_GPU_PARAMS struct { + _ structs.HostLayout HSubDevice Handle Flags uint32 DevDescriptor uint64 @@ -475,6 +546,10 @@ const ( NV2080_CTRL_CMD_BUS_GET_INFO_V2 = 0x20801823 NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS = 0x2080182a NV2080_CTRL_CMD_BUS_GET_C2C_INFO = 0x2080182b + NV2080_CTRL_CMD_BUS_GET_PEX_COUNTERS = 0x20801813 + NV2080_CTRL_CMD_BUS_GET_PEX_UTIL_COUNTERS = 0x20801819 + NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS = 0x20801829 + NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS = 0x20801830 ) // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ce.h: @@ -489,17 +564,26 @@ const ( NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION = 0x20800301 ) +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ecc.h: +const ( + NV2080_CTRL_CMD_ECC_GET_REPAIR_STATUS = 0x20803404 +) + // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h: const ( - NV2080_CTRL_CMD_FB_GET_INFO = 0x20801301 - NV2080_CTRL_CMD_FB_GET_INFO_V2 = 0x20801303 - NV2080_CTRL_CMD_FB_GET_GPU_CACHE_INFO = 0x20801315 - NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO = 0x20801320 - NV2080_CTRL_CMD_FB_GET_SEMAPHORE_SURFACE_LAYOUT = 0x20801352 - NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT = 0x20801358 - NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS = 0x20801359 - NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT_V575 = NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT - 1 - NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS_V575 = NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS - 1 + NV2080_CTRL_CMD_FB_GET_INFO = 0x20801301 + NV2080_CTRL_CMD_FB_GET_INFO_V2 = 0x20801303 + NV2080_CTRL_CMD_FB_GET_GPU_CACHE_INFO = 0x20801315 + NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO = 0x20801320 + NV2080_CTRL_CMD_FB_GET_SEMAPHORE_SURFACE_LAYOUT = 0x20801352 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT = 0x20801358 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS = 0x20801359 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT_V575 = NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT - 1 + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS_V575 = NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS - 1 + NV2080_CTRL_CMD_FB_GET_OFFLINED_PAGES = 0x20801322 + NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE = 0x2080130e + NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION = 0x20801355 + NV2080_CTRL_CMD_FB_GET_FS_INFO = 0x20801346 ) // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fifo.h: @@ -516,6 +600,7 @@ const ( // +marshal type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS struct { + _ structs.HostLayout BDisable uint8 Pad1 [3]byte NumChannels uint32 @@ -553,6 +638,12 @@ const ( NV2080_CTRL_CMD_GPU_GET_PID_INFO = 0x2080018e NV2080_CTRL_CMD_GPU_GET_COMPUTE_POLICY_CONFIG = 0x20800195 NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO = 0x208001a3 + NV2080_CTRL_CMD_GPU_GET_NVENC_SW_SESSION_STATS = 0x2080016d + NV2080_CTRL_CMD_GPU_GET_NVFBC_SW_SESSION_STATS = 0x2080017b + NV2080_CTRL_CMD_GPU_GET_SKYLINE_INFO = 0x2080019f + NV2080_CTRL_CMD_GPU_GET_CHIP_DETAILS = 0x208001a4 + NV2080_CTRL_CMD_GPU_GET_RECOVERY_ACTION = 0x208001b2 + NV2080_CTRL_CMD_GPU_EXEC_REG_OPS = 0x20800122 ) // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gr.h: @@ -560,6 +651,7 @@ const ( NV2080_CTRL_CMD_GR_GET_INFO = 0x20801201 NV2080_CTRL_CMD_GR_GET_ZCULL_INFO = 0x20801206 NV2080_CTRL_CMD_GR_CTXSW_ZCULL_BIND = 0x20801208 + NV2080_CTRL_CMD_GR_GET_SM_TO_GPC_TPC_MAPPINGS = 0x2080120f NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE = 0x20801210 NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_SIZE = 0x20801218 NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER = 0x2080121b @@ -581,6 +673,7 @@ const ( // +marshal type NV2080_CTRL_GR_GET_INFO_PARAMS struct { + _ structs.HostLayout NvxxxCtrlXxxGetInfoParams GRRouteInfo NV0080_CTRL_GR_ROUTE_INFO } @@ -600,6 +693,44 @@ func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) CtrlInfoList() P64 { return p.InfoList } +// NV2080_CTRL_GPU_REG_OP is the type of an individual register operation, +// from src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h. +// +// +marshal +type NV2080_CTRL_GPU_REG_OP struct { + _ structs.HostLayout + RegOp uint8 + RegType uint8 + RegStatus uint8 + RegQuad uint8 + RegGroupMask uint32 + RegSubGroupMask uint32 + RegOffset uint32 + RegValueHi uint32 + RegValueLo uint32 + RegAndNMaskHi uint32 + RegAndNMaskLo uint32 +} + +// CtrlGpuRegOpSize is sizeof(NV2080_CTRL_GPU_REG_OP). +var CtrlGpuRegOpSize = uint32((*NV2080_CTRL_GPU_REG_OP)(nil).SizeBytes()) + +// NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS is the params type for +// NV2080_CTRL_CMD_GPU_EXEC_REG_OPS, from +// src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h. +// +// +marshal +type NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS struct { + _ structs.HostLayout + HClientTarget Handle + HChannelTarget Handle + BNonTransactional uint32 + Reserved00 [2]uint32 + RegOpCount uint32 + RegOps P64 + GRRouteInfo NV0080_CTRL_GR_ROUTE_INFO +} + // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080mc.h: const ( NV2080_CTRL_CMD_MC_GET_ARCH_INFO = 0x20801701 @@ -608,8 +739,13 @@ const ( // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080nvlink.h: const ( - NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS = 0x20803001 - NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS = 0x20803002 + NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS = 0x20803001 + NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS = 0x20803002 + NV2080_CTRL_CMD_NVLINK_GET_PLATFORM_INFO = 0x20803083 + NV2080_CTRL_CMD_NVLINK_LOCK_REMAP_TABLE_AND_MSE = 0x2080309d + NV2080_CTRL_CMD_NVLINK_SETUP_NVLE_ENCRYPTION_KEY = 0x208030a0 + NV2080_CTRL_CMD_NVLINK_GET_REMAP_TABLE_INFO_V2 = 0x208030a1 + NV2080_CTRL_CMD_NVLINK_GET_UPDATE_NVLE_LIDS_V2 = 0x208030a2 ) // From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080perf.h: @@ -651,6 +787,7 @@ const ( // +marshal type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS struct { + _ structs.HostLayout HVASpace Handle Pad [4]byte VASpaceToken uint64 @@ -685,6 +822,7 @@ const ( const ( NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE = 0x90960106 NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_ENTRY = 0x90960107 + NV9096_CTRL_CMD_SET_ZBC_COLOR_CLEAR = 0x90960101 ) // From src/common/sdk/nvidia/inc/ctrl/ctrl90e6.h: @@ -696,6 +834,7 @@ const ( const ( NVA06C_CTRL_CMD_GPFIFO_SCHEDULE = 0xa06c0101 NVA06C_CTRL_CMD_SET_TIMESLICE = 0xa06c0103 + NVA06C_CTRL_CMD_GET_TIMESLICE = 0xa06c0104 NVA06C_CTRL_CMD_PREEMPT = 0xa06c0105 ) @@ -705,6 +844,33 @@ const ( NVA06F_CTRL_CMD_BIND = 0xa06f0104 ) +// From src/common/sdk/nvidia/inc/ctrl/ctrl90cc.h: +const ( + NV90CC_CTRL_CMD_POWER_REQUEST_FEATURES = 0x90cc0301 + NV90CC_CTRL_CMD_POWER_RELEASE_FEATURES = 0x90cc0302 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlb0cc.h: +const ( + NVB0CC_CTRL_CMD_RESERVE_HWPM_LEGACY = 0xb0cc0101 + NVB0CC_CTRL_CMD_RELEASE_HWPM_LEGACY = 0xb0cc0102 + NVB0CC_CTRL_CMD_RESERVE_PM_AREA_SMPC = 0xb0cc0103 + NVB0CC_CTRL_CMD_RELEASE_PM_AREA_SMPC = 0xb0cc0104 + NVB0CC_CTRL_CMD_ALLOC_PMA_STREAM = 0xb0cc0105 + NVB0CC_CTRL_CMD_FREE_PMA_STREAM = 0xb0cc0106 + NVB0CC_CTRL_CMD_BIND_PM_RESOURCES = 0xb0cc0107 + NVB0CC_CTRL_CMD_UNBIND_PM_RESOURCES = 0xb0cc0108 + NVB0CC_CTRL_CMD_PMA_STREAM_UPDATE_GET_PUT = 0xb0cc0109 + NVB0CC_CTRL_CMD_EXEC_REG_OPS = 0xb0cc010a + NVB0CC_CTRL_CMD_RESERVE_PM_AREA_PC_SAMPLER = 0xb0cc010b + NVB0CC_CTRL_CMD_RELEASE_PM_AREA_PC_SAMPLER = 0xb0cc010c + NVB0CC_CTRL_CMD_GET_TOTAL_HS_CREDITS = 0xb0cc010d + NVB0CC_CTRL_CMD_SET_HS_CREDITS = 0xb0cc010e + NVB0CC_CTRL_CMD_RESERVE_CCU_PROF = 0xb0cc0119 + NVB0CC_CTRL_CMD_POWER_REQUEST_FEATURES = 0xb0cc0301 + NVB0CC_CTRL_CMD_POWER_RELEASE_FEATURES = 0xb0cc0302 +) + // From src/common/sdk/nvidia/inc/ctrl/ctrlcb33.h: const ( NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES = 0xcb330101 diff --git a/pkg/abi/nvgpu/frontend.go b/pkg/abi/nvgpu/frontend.go index f857392..d8ada7a 100644 --- a/pkg/abi/nvgpu/frontend.go +++ b/pkg/abi/nvgpu/frontend.go @@ -15,6 +15,8 @@ package nvgpu import ( + "structs" + "github.com/sagernet/gvisor/pkg/marshal" ) @@ -26,15 +28,16 @@ const NV_IOCTL_MAGIC = uint32('F') // Note that these are only the IOC_NR part of the ioctl command. const ( // From kernel-open/common/inc/nv-ioctl-numbers.h: - NV_IOCTL_BASE = 200 - NV_ESC_CARD_INFO = NV_IOCTL_BASE + 0 - NV_ESC_REGISTER_FD = NV_IOCTL_BASE + 1 - NV_ESC_ALLOC_OS_EVENT = NV_IOCTL_BASE + 6 - NV_ESC_FREE_OS_EVENT = NV_IOCTL_BASE + 7 - NV_ESC_CHECK_VERSION_STR = NV_IOCTL_BASE + 10 - NV_ESC_ATTACH_GPUS_TO_FD = NV_IOCTL_BASE + 12 - NV_ESC_SYS_PARAMS = NV_IOCTL_BASE + 14 - NV_ESC_WAIT_OPEN_COMPLETE = NV_IOCTL_BASE + 18 + NV_IOCTL_BASE = 200 + NV_ESC_CARD_INFO = NV_IOCTL_BASE + 0 + NV_ESC_REGISTER_FD = NV_IOCTL_BASE + 1 + NV_ESC_ALLOC_OS_EVENT = NV_IOCTL_BASE + 6 + NV_ESC_FREE_OS_EVENT = NV_IOCTL_BASE + 7 + NV_ESC_CHECK_VERSION_STR = NV_IOCTL_BASE + 10 + NV_ESC_ATTACH_GPUS_TO_FD = NV_IOCTL_BASE + 12 + NV_ESC_SYS_PARAMS = NV_IOCTL_BASE + 14 + NV_ESC_EXPORT_TO_DMABUF_FD = NV_IOCTL_BASE + 17 + NV_ESC_WAIT_OPEN_COMPLETE = NV_IOCTL_BASE + 18 // From kernel-open/common/inc/nv-ioctl-numa.h: NV_ESC_NUMA_INFO = NV_IOCTL_BASE + 15 @@ -59,10 +62,46 @@ const ( // Frontend ioctl parameter structs, from src/common/sdk/nvidia/inc/nvos.h or // kernel-open/common/inc/nv-ioctl.h. +// IoctlCardInfo is nv_ioctl_card_info_t. NV_ESC_CARD_INFO takes an array of +// IoctlCardInfo as parameter. +// +// +marshal +type IoctlCardInfo struct { + _ structs.HostLayout + Valid uint8 + Pad0 [3]byte + PCIInfo PCIInfo + GPUID uint32 + InterruptLine uint16 + Pad1 [2]byte + RegAddress uint64 + RegSize uint64 + FBAddress uint64 + FBSize uint64 + MinorNumber uint32 + DevName [10]byte + Pad2 [2]byte +} + +// PCIInfo is nv_pci_info_t. +// +// +marshal +type PCIInfo struct { + _ structs.HostLayout + Domain uint32 + Bus uint8 + Slot uint8 + Function uint8 + Pad0 uint8 + VendorID uint16 + DeviceID uint16 +} + // IoctlRegisterFD is the parameter type for NV_ESC_REGISTER_FD. // // +marshal type IoctlRegisterFD struct { + _ structs.HostLayout CtlFD int32 } @@ -70,6 +109,7 @@ type IoctlRegisterFD struct { // // +marshal type IoctlAllocOSEvent struct { + _ structs.HostLayout HClient Handle HDevice Handle FD uint32 @@ -100,6 +140,7 @@ func (p *IoctlAllocOSEvent) SetStatus(status uint32) { // // +marshal type IoctlFreeOSEvent struct { + _ structs.HostLayout HClient Handle HDevice Handle FD uint32 @@ -130,6 +171,7 @@ func (p *IoctlFreeOSEvent) SetStatus(status uint32) { // // +marshal type RMAPIVersion struct { + _ structs.HostLayout Cmd uint32 Reply uint32 VersionString [64]byte @@ -139,6 +181,7 @@ type RMAPIVersion struct { // // +marshal type IoctlSysParams struct { + _ structs.HostLayout MemblockSize uint64 } @@ -146,6 +189,7 @@ type IoctlSysParams struct { // // +marshal type IoctlWaitOpenComplete struct { + _ structs.HostLayout Rc int32 AdapterStatus uint32 } @@ -160,10 +204,119 @@ func (p *IoctlWaitOpenComplete) SetStatus(status uint32) { p.AdapterStatus = status } +// NV_DMABUF_EXPORT_MAX_HANDLES is the fixed size of the handle/offset/size +// arrays in nv_ioctl_export_to_dma_buf_fd_t. +// From kernel-open/common/inc/nv-ioctl.h. +const NV_DMABUF_EXPORT_MAX_HANDLES = 128 + +// IoctlExportToDMABufFD is nv_ioctl_export_to_dma_buf_fd_t, the parameter type +// for NV_ESC_EXPORT_TO_DMABUF_FD (kernel-open/common/inc/nv-ioctl.h). +// +// +marshal +type IoctlExportToDMABufFD struct { + _ structs.HostLayout + FD int32 + HClient Handle + TotalObjects uint32 + NumObjects uint32 + Index uint32 + Pad0 uint32 + TotalSize uint64 + Handles [NV_DMABUF_EXPORT_MAX_HANDLES]Handle + Offsets [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Sizes [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Status uint32 + Pad1 uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlExportToDMABufFD) GetFrontendFD() int32 { return p.FD } + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlExportToDMABufFD) SetFrontendFD(fd int32) { p.FD = fd } + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlExportToDMABufFD) GetStatus() uint32 { return p.Status } + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlExportToDMABufFD) SetStatus(status uint32) { p.Status = status } + +// IoctlExportToDMABufFD_V570 is the updated version of +// nv_ioctl_export_to_dma_buf_fd_t since 570.86.15. +// +// +marshal +type IoctlExportToDMABufFD_V570 struct { + _ structs.HostLayout + FD int32 + HClient Handle + TotalObjects uint32 + NumObjects uint32 + Index uint32 + Pad0 uint32 + TotalSize uint64 + MappingType uint8 + Pad1 [3]byte + Handles [NV_DMABUF_EXPORT_MAX_HANDLES]Handle + Pad2 uint32 + Offsets [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Sizes [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Status uint32 + Pad3 uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlExportToDMABufFD_V570) GetFrontendFD() int32 { return p.FD } + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlExportToDMABufFD_V570) SetFrontendFD(fd int32) { p.FD = fd } + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlExportToDMABufFD_V570) GetStatus() uint32 { return p.Status } + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlExportToDMABufFD_V570) SetStatus(status uint32) { p.Status = status } + +// IoctlExportToDMABufFD_V580 is the updated version of +// nv_ioctl_export_to_dma_buf_fd_t since 580.65.06. +// +// +marshal +type IoctlExportToDMABufFD_V580 struct { + _ structs.HostLayout + FD int32 + HClient Handle + TotalObjects uint32 + NumObjects uint32 + Index uint32 + Pad0 uint32 + TotalSize uint64 + MappingType uint8 + AllowMmap uint8 + Pad1 [2]byte + Handles [NV_DMABUF_EXPORT_MAX_HANDLES]Handle + Pad2 uint32 + Offsets [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Sizes [NV_DMABUF_EXPORT_MAX_HANDLES]uint64 + Status uint32 + Pad3 uint32 +} + +// GetFrontendFD implements HasFrontendFD.GetFrontendFD. +func (p *IoctlExportToDMABufFD_V580) GetFrontendFD() int32 { return p.FD } + +// SetFrontendFD implements HasFrontendFD.SetFrontendFD. +func (p *IoctlExportToDMABufFD_V580) SetFrontendFD(fd int32) { p.FD = fd } + +// GetStatus implements HasStatus.GetStatus. +func (p *IoctlExportToDMABufFD_V580) GetStatus() uint32 { return p.Status } + +// SetStatus implements HasStatus.SetStatus. +func (p *IoctlExportToDMABufFD_V580) SetStatus(status uint32) { p.Status = status } + // IoctlNVOS02ParametersWithFD is the parameter type for NV_ESC_RM_ALLOC_MEMORY. // // +marshal type IoctlNVOS02ParametersWithFD struct { + _ structs.HostLayout Params NVOS02_PARAMETERS FD int32 Pad0 [4]byte @@ -181,6 +334,7 @@ func (p *IoctlNVOS02ParametersWithFD) SetStatus(status uint32) { // +marshal type NVOS02_PARAMETERS struct { + _ structs.HostLayout HRoot Handle HObjectParent Handle HObjectNew Handle @@ -208,6 +362,7 @@ const ( // // +marshal type NVOS00_PARAMETERS struct { + _ structs.HostLayout HRoot Handle HObjectParent Handle HObjectOld Handle @@ -252,6 +407,7 @@ func GetRmAllocParamObj(isNVOS64 bool) RmAllocParamType { // // +marshal type NVOS21_PARAMETERS struct { + _ structs.HostLayout HRoot Handle HObjectParent Handle HObjectNew Handle @@ -322,6 +478,7 @@ func (n *NVOS21_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS55_PARAMETERS struct { + _ structs.HostLayout HClient Handle HParent Handle HObject Handle @@ -345,6 +502,7 @@ func (n *NVOS55_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS57_PARAMETERS struct { + _ structs.HostLayout HClient Handle HObject Handle SharePolicy RS_SHARE_POLICY @@ -365,6 +523,7 @@ func (n *NVOS57_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS30_PARAMETERS struct { + _ structs.HostLayout Client Handle Device Handle Channel Handle @@ -394,6 +553,7 @@ func (n *NVOS30_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS32_PARAMETERS struct { + _ structs.HostLayout HRoot Handle HObjectParent Handle Function uint32 @@ -424,6 +584,7 @@ const ( // NVOS32AllocSize is the type of NVOS32Parameters.Data for // NVOS32_FUNCTION_ALLOC_SIZE. type NVOS32AllocSize struct { + _ structs.HostLayout Owner uint32 HMemory Handle Type uint32 @@ -472,6 +633,7 @@ const ( // // +marshal type IoctlNVOS33ParametersWithFD struct { + _ structs.HostLayout Params NVOS33_PARAMETERS FD int32 Pad0 [4]byte @@ -489,6 +651,7 @@ func (p *IoctlNVOS33ParametersWithFD) SetStatus(status uint32) { // +marshal type NVOS33_PARAMETERS struct { + _ structs.HostLayout HClient Handle HDevice Handle HMemory Handle @@ -516,6 +679,7 @@ const ( // // +marshal type NVOS34_PARAMETERS struct { + _ structs.HostLayout HClient Handle HDevice Handle HMemory Handle @@ -539,6 +703,7 @@ func (n *NVOS34_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS39_PARAMETERS struct { + _ structs.HostLayout HObjectParent Handle HSubDevice Handle HObjectNew Handle @@ -567,6 +732,7 @@ func (n *NVOS39_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS46_PARAMETERS struct { + _ structs.HostLayout Client Handle Device Handle Dma Handle @@ -595,6 +761,7 @@ func (n *NVOS46_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS46_PARAMETERS_V580 struct { + _ structs.HostLayout Client Handle Device Handle Dma Handle @@ -624,6 +791,7 @@ func (n *NVOS46_PARAMETERS_V580) SetStatus(status uint32) { // // +marshal type NVOS47_PARAMETERS struct { + _ structs.HostLayout Client Handle Device Handle Dma Handle @@ -650,6 +818,7 @@ func (n *NVOS47_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS47_PARAMETERS_V550 struct { + _ structs.HostLayout Client Handle Device Handle Dma Handle @@ -676,6 +845,7 @@ func (n *NVOS47_PARAMETERS_V550) SetStatus(status uint32) { // // +marshal type NVOS54_PARAMETERS struct { + _ structs.HostLayout HClient Handle HObject Handle Cmd uint32 @@ -699,6 +869,7 @@ func (n *NVOS54_PARAMETERS) SetStatus(status uint32) { // // +marshal type NVOS56_PARAMETERS struct { + _ structs.HostLayout HClient Handle HDevice Handle HMemory Handle @@ -724,6 +895,7 @@ func (n *NVOS56_PARAMETERS) SetStatus(status uint32) { // +marshal // +stateify savable type NVOS64_PARAMETERS struct { + _ structs.HostLayout HRoot Handle HObjectParent Handle HObjectNew Handle diff --git a/pkg/abi/nvgpu/nvgpu.go b/pkg/abi/nvgpu/nvgpu.go index 5d4df11..27bf4dc 100644 --- a/pkg/abi/nvgpu/nvgpu.go +++ b/pkg/abi/nvgpu/nvgpu.go @@ -18,13 +18,21 @@ package nvgpu import ( "fmt" + "structs" ) // Device numbers. const ( - NV_MAJOR_DEVICE_NUMBER = 195 // from kernel-open/common/inc/nv.h - NV_CONTROL_DEVICE_MINOR = 255 // from kernel-open/common/inc/nv-linux.h - NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0 // from kernel-open/nvidia-uvm/uvm_common.h + // From kernel-open/common/inc/nv-chardev-numbers.h: + NV_MAJOR_DEVICE_NUMBER = 195 + NV_MINOR_DEVICE_NUMBER_REGULAR_MAX = 247 + NV_MINOR_DEVICE_NUMBER_CONTROL_DEVICE = 255 + + // From kernel-open/nvidia-uvm/uvm_common.h: + NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0 + + // From kernel-open/nvidia/nv-caps.c: + NV_CAP_DRV_MINOR_COUNT = 8192 ) // Handle is NvHandle, from src/common/sdk/nvidia/inc/nvtypes.h. @@ -32,6 +40,7 @@ const ( // +marshal // +stateify savable type Handle struct { + _ structs.HostLayout Val uint32 } @@ -63,6 +72,7 @@ const ( // +marshal // +stateify savable type RS_ACCESS_MASK struct { + _ structs.HostLayout Limbs [SDK_RS_ACCESS_MAX_LIMBS]uint32 // RsAccessLimb } @@ -73,6 +83,7 @@ const SDK_RS_ACCESS_MAX_LIMBS = 1 // // +marshal type RS_SHARE_POLICY struct { + _ structs.HostLayout Target uint32 AccessMask RS_ACCESS_MASK Type uint16 diff --git a/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go b/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go index f3f9923..d429be9 100644 --- a/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go +++ b/pkg/abi/nvgpu/nvgpu_abi_autogen_unsafe.go @@ -18,13 +18,19 @@ var ( _ marshal.Marshallable = (*ClassID)(nil) _ marshal.Marshallable = (*Handle)(nil) _ marshal.Marshallable = (*IoctlAllocOSEvent)(nil) + _ marshal.Marshallable = (*IoctlCardInfo)(nil) + _ marshal.Marshallable = (*IoctlExportToDMABufFD)(nil) + _ marshal.Marshallable = (*IoctlExportToDMABufFD_V570)(nil) + _ marshal.Marshallable = (*IoctlExportToDMABufFD_V580)(nil) _ marshal.Marshallable = (*IoctlFreeOSEvent)(nil) _ marshal.Marshallable = (*IoctlNVOS02ParametersWithFD)(nil) _ marshal.Marshallable = (*IoctlNVOS33ParametersWithFD)(nil) _ marshal.Marshallable = (*IoctlRegisterFD)(nil) _ marshal.Marshallable = (*IoctlSysParams)(nil) _ marshal.Marshallable = (*IoctlWaitOpenComplete)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_GPU_ATTACH_IDS_PARAMS)(nil) _ marshal.Marshallable = (*NV0000_CTRL_GPU_GET_ID_INFO_PARAMS)(nil) + _ marshal.Marshallable = (*NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS)(nil) _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECT)(nil) _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS)(nil) _ marshal.Marshallable = (*NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS)(nil) @@ -42,23 +48,32 @@ var ( _ marshal.Marshallable = (*NV0080_CTRL_GR_ROUTE_INFO)(nil) _ marshal.Marshallable = (*NV00DE_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NV00DE_ALLOC_PARAMETERS_V545)(nil) + _ marshal.Marshallable = (*NV00E0_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV00F1_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NV00F8_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV00FB_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NV00FD_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NV00FD_ALLOCATION_PARAMETERS_V545)(nil) + _ marshal.Marshallable = (*NV00FD_ALLOCATION_PARAMETERS_V590)(nil) _ marshal.Marshallable = (*NV00FD_CTRL_ATTACH_GPU_PARAMS)(nil) _ marshal.Marshallable = (*NV2080_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS)(nil) + _ marshal.Marshallable = (*NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS)(nil) + _ marshal.Marshallable = (*NV2080_CTRL_GPU_REG_OP)(nil) _ marshal.Marshallable = (*NV2080_CTRL_GR_GET_INFO_PARAMS)(nil) _ marshal.Marshallable = (*NV2081_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NV208F_CTRL_GPU_VERIFY_INFOROM_PARAMS)(nil) _ marshal.Marshallable = (*NV503B_ALLOC_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV503B_ALLOC_PARAMETERS_V590)(nil) _ marshal.Marshallable = (*NV503B_BAR1_P2P_DMA_INFO)(nil) + _ marshal.Marshallable = (*NV503B_FABRIC_P2P_DMA_INFO)(nil) _ marshal.Marshallable = (*NV503C_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NV503C_CTRL_REGISTER_VA_SPACE_PARAMS)(nil) _ marshal.Marshallable = (*NV83DE_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NV9072_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NVA0BC_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NVB0B5_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NVB2CC_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NVOS00_PARAMETERS)(nil) _ marshal.Marshallable = (*NVOS02_PARAMETERS)(nil) _ marshal.Marshallable = (*NVOS21_PARAMETERS)(nil) @@ -80,6 +95,7 @@ var ( _ marshal.Marshallable = (*NV_BSP_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NV_CHANNEL_ALLOC_PARAMS)(nil) _ marshal.Marshallable = (*NV_CHANNEL_ALLOC_PARAMS_V570)(nil) + _ marshal.Marshallable = (*NV_CHANNEL_ALLOC_PARAMS_V610)(nil) _ marshal.Marshallable = (*NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NV_CONFIDENTIAL_COMPUTE_ALLOC_PARAMS)(nil) _ marshal.Marshallable = (*NV_CONTEXT_DMA_ALLOCATION_PARAMS)(nil) @@ -95,12 +111,16 @@ var ( _ marshal.Marshallable = (*NV_MEMORY_MAPPER_ALLOCATION_PARAMS_V555)(nil) _ marshal.Marshallable = (*NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS)(nil) _ marshal.Marshallable = (*NV_MSENC_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_NVJPG_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_OFA_ALLOCATION_PARAMETERS)(nil) + _ marshal.Marshallable = (*NV_OFA_ALLOCATION_PARAMETERS_V545)(nil) _ marshal.Marshallable = (*NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS)(nil) _ marshal.Marshallable = (*NV_VASPACE_ALLOCATION_PARAMETERS)(nil) _ marshal.Marshallable = (*NV_VASPACE_ALLOCATION_PARAMETERS_V580)(nil) _ marshal.Marshallable = (*NvUUID)(nil) _ marshal.Marshallable = (*NvxxxCtrlXxxGetInfoParams)(nil) _ marshal.Marshallable = (*P64)(nil) + _ marshal.Marshallable = (*PCIInfo)(nil) _ marshal.Marshallable = (*RMAPIVersion)(nil) _ marshal.Marshallable = (*RS_ACCESS_MASK)(nil) _ marshal.Marshallable = (*RS_SHARE_POLICY)(nil) @@ -113,7 +133,9 @@ var ( _ marshal.Marshallable = (*UVM_DISABLE_PEER_ACCESS_PARAMS)(nil) _ marshal.Marshallable = (*UVM_DISABLE_READ_DUPLICATION_PARAMS)(nil) _ marshal.Marshallable = (*UVM_ENABLE_PEER_ACCESS_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_ENABLE_READ_DUPLICATION_PARAMS)(nil) _ marshal.Marshallable = (*UVM_FREE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_FREE_PARAMS_V590)(nil) _ marshal.Marshallable = (*UVM_INITIALIZE_PARAMS)(nil) _ marshal.Marshallable = (*UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS)(nil) _ marshal.Marshallable = (*UVM_MAP_EXTERNAL_ALLOCATION_PARAMS)(nil) @@ -127,6 +149,7 @@ var ( _ marshal.Marshallable = (*UVM_REGISTER_CHANNEL_PARAMS)(nil) _ marshal.Marshallable = (*UVM_REGISTER_GPU_PARAMS)(nil) _ marshal.Marshallable = (*UVM_REGISTER_GPU_VASPACE_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_SET_ACCESSED_BY_PARAMS)(nil) _ marshal.Marshallable = (*UVM_SET_PREFERRED_LOCATION_PARAMS)(nil) _ marshal.Marshallable = (*UVM_SET_PREFERRED_LOCATION_PARAMS_V550)(nil) _ marshal.Marshallable = (*UVM_SET_RANGE_GROUP_PARAMS)(nil) @@ -134,6 +157,7 @@ var ( _ marshal.Marshallable = (*UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS)(nil) _ marshal.Marshallable = (*UVM_UNMAP_EXTERNAL_PARAMS)(nil) _ marshal.Marshallable = (*UVM_UNREGISTER_CHANNEL_PARAMS)(nil) + _ marshal.Marshallable = (*UVM_UNREGISTER_CHANNEL_PARAMS_V590)(nil) _ marshal.Marshallable = (*UVM_UNREGISTER_GPU_PARAMS)(nil) _ marshal.Marshallable = (*UVM_UNREGISTER_GPU_VASPACE_PARAMS)(nil) _ marshal.Marshallable = (*UVM_UNSET_ACCESSED_BY_PARAMS)(nil) @@ -759,6 +783,352 @@ func (n *NV00DE_ALLOC_PARAMETERS_V545) WriteTo(writer io.Writer) (int64, error) return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00E0_ALLOCATION_PARAMETERS) SizeBytes() int { + return 16 + + (*NV_EXPORT_MEM_PACKET)(nil).SizeBytes() + + 1*2 + + 1*NV_MEM_EXPORT_METADATA_LEN + + 4*NV_MAX_DEVICES + + 1*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00E0_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.IMEXChannel)) + dst = dst[4:] + dst = n.Packet.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.NumMaxHandles)) + dst = dst[2:] + for idx := 0; idx < 2; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < NV_MEM_EXPORT_METADATA_LEN; idx++ { + dst[0] = byte(n.Metadata[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.DeviceInstanceMask)) + dst = dst[4:] + for idx := 0; idx < NV_MAX_DEVICES; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GIIDMasks[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.NumCurHandles)) + dst = dst[2:] + for idx := 0; idx < 2; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00E0_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.IMEXChannel = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.Packet.UnmarshalUnsafe(src) + n.NumMaxHandles = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < 2; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NV_MEM_EXPORT_METADATA_LEN; idx++ { + n.Metadata[idx] = uint8(src[0]) + src = src[1:] + } + n.DeviceInstanceMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NV_MAX_DEVICES; idx++ { + n.GIIDMasks[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + n.NumCurHandles = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < 2; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00E0_ALLOCATION_PARAMETERS) Packed() bool { + return n.Packet.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00E0_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.Packet.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV00E0_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00E0_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.Packet.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00E0_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00E0_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Packet.Packed() { + // Type NV00E0_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00E0_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00E0_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.Packet.Packed() { + // Type NV00E0_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00E0_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00E0_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.Packet.Packed() { + // Type NV00E0_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV00F1_ALLOCATION_PARAMETERS) SizeBytes() int { + return 14 + + 1*4 + + (*P64)(nil).SizeBytes() + + 1*6 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV00F1_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.CapDescriptor)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + dst = n.POsEvent.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.NodeID)) + dst = dst[2:] + for idx := 0; idx < 6; idx++ { + dst[0] = byte(n.Pad1[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV00F1_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.CapDescriptor = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + src = n.POsEvent.UnmarshalUnsafe(src) + n.NodeID = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < 6; idx++ { + n.Pad1[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV00F1_ALLOCATION_PARAMETERS) Packed() bool { + return n.POsEvent.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV00F1_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.POsEvent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV00F1_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV00F1_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.POsEvent.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00F1_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV00F1_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.POsEvent.Packed() { + // Type NV00F1_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV00F1_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV00F1_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.POsEvent.Packed() { + // Type NV00F1_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV00F1_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV00F1_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.POsEvent.Packed() { + // Type NV00F1_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV00F8_ALLOCATION_PARAMETERS) SizeBytes() int { return 32 + @@ -910,85 +1280,73 @@ func (n *NV00F8_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) } // SizeBytes implements marshal.Marshallable.SizeBytes. -func (n *NV00FD_ALLOCATION_PARAMETERS) SizeBytes() int { - return 32 + - (*P64)(nil).SizeBytes() +func (n *NV00FB_ALLOCATION_PARAMETERS) SizeBytes() int { + return 14 + + 1*NV_MEM_EXPORT_UUID_LEN + + 1*2 } // MarshalBytes implements marshal.Marshallable.MarshalBytes. -func (n *NV00FD_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { - hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Alignment)) +func (n *NV00FB_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < NV_MEM_EXPORT_UUID_LEN; idx++ { + dst[0] = byte(n.ExportUUID[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint16(dst[:2], uint16(n.Index)) + dst = dst[2:] + for idx := 0; idx < 2; idx++ { + dst[0] = byte(n.Pad0[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.ID)) dst = dst[8:] - hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.AllocSize)) - dst = dst[8:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.PageSize)) - dst = dst[4:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.AllocFlags)) - dst = dst[4:] - hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.NumGPUs)) - dst = dst[4:] - // Padding: dst[:sizeof(uint32)] ~= uint32(0) - dst = dst[4:] - dst = n.POsEvent.MarshalUnsafe(dst) return dst } // UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. -func (n *NV00FD_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { - n.Alignment = uint64(hostarch.ByteOrder.Uint64(src[:8])) +func (n *NV00FB_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < NV_MEM_EXPORT_UUID_LEN; idx++ { + n.ExportUUID[idx] = uint8(src[0]) + src = src[1:] + } + n.Index = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < 2; idx++ { + n.Pad0[idx] = src[0] + src = src[1:] + } + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ID = uint64(hostarch.ByteOrder.Uint64(src[:8])) src = src[8:] - n.AllocSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) - src = src[8:] - n.PageSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] - n.AllocFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] - n.NumGPUs = uint32(hostarch.ByteOrder.Uint32(src[:4])) - src = src[4:] - // Padding: var _ uint32 ~= src[:sizeof(uint32)] - src = src[4:] - src = n.POsEvent.UnmarshalUnsafe(src) return src } // Packed implements marshal.Marshallable.Packed. // //go:nosplit -func (n *NV00FD_ALLOCATION_PARAMETERS) Packed() bool { - return n.POsEvent.Packed() +func (n *NV00FB_ALLOCATION_PARAMETERS) Packed() bool { + return true } // MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. -func (n *NV00FD_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { - if n.POsEvent.Packed() { - size := n.SizeBytes() - gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) - return dst[size:] - } - // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. - return n.MarshalBytes(dst) +func (n *NV00FB_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] } // UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. -func (n *NV00FD_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { - if n.POsEvent.Packed() { - size := n.SizeBytes() - gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) - return src[size:] - } - // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. - return n.UnmarshalBytes(src) +func (n *NV00FB_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] } // CopyOutN implements marshal.Marshallable.CopyOutN. -func (n *NV00FD_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { - if !n.POsEvent.Packed() { - // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. - buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. - n.MarshalBytes(buf) // escapes: fallback. - return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. - } - +func (n *NV00FB_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) @@ -1004,22 +1362,12 @@ func (n *NV00FD_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hos } // CopyOut implements marshal.Marshallable.CopyOut. -func (n *NV00FD_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { +func (n *NV00FB_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { return n.CopyOutN(cc, addr, n.SizeBytes()) } // CopyInN implements marshal.Marshallable.CopyInN. -func (n *NV00FD_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { - if !n.POsEvent.Packed() { - // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. - buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. - length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. - // Unmarshal unconditionally. If we had a short copy-in, this results in a - // partially unmarshalled struct. - n.UnmarshalBytes(buf) // escapes: fallback. - return length, err - } - +func (n *NV00FB_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) @@ -1035,20 +1383,12 @@ func (n *NV00FD_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr host } // CopyIn implements marshal.Marshallable.CopyIn. -func (n *NV00FD_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { +func (n *NV00FB_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { return n.CopyInN(cc, addr, n.SizeBytes()) } // WriteTo implements io.WriterTo.WriteTo. -func (n *NV00FD_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { - if !n.POsEvent.Packed() { - // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. - buf := make([]byte, n.SizeBytes()) - n.MarshalBytes(buf) - length, err := writer.Write(buf) - return int64(length), err - } - +func (n *NV00FB_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { // Construct a slice backed by dst's underlying memory. var buf []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) @@ -1063,6 +1403,160 @@ func (n *NV00FD_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV00FD_ALLOCATION_PARAMETERS) SizeBytes() int { + return 32 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV00FD_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Alignment)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.AllocSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.PageSize)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.AllocFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.NumGPUs)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + dst = p.POsEvent.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV00FD_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + p.Alignment = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.AllocSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.PageSize = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.AllocFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.NumGPUs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + src = p.POsEvent.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV00FD_ALLOCATION_PARAMETERS) Packed() bool { + return p.POsEvent.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV00FD_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if p.POsEvent.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV00FD_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if p.POsEvent.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV00FD_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV00FD_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV00FD_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV00FD_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV00FD_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !p.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV00FD_ALLOCATION_PARAMETERS_V545) SizeBytes() int { return 2 + @@ -1205,6 +1699,168 @@ func (n *NV00FD_ALLOCATION_PARAMETERS_V545) WriteTo(writer io.Writer) (int64, er return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) SizeBytes() int { + return 34 + + (*NV_EXPORT_MEM_PACKET)(nil).SizeBytes() + + 1*6 + + (*P64)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) MarshalBytes(dst []byte) []byte { + dst = p.ExpPacket.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.Index)) + dst = dst[2:] + // Padding: dst[:sizeof(byte)*6] ~= [6]byte{0} + dst = dst[1*(6):] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Alignment)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.AllocSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.PageSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.AllocFlags)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.NumGPUs)) + dst = dst[4:] + dst = p.POsEvent.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) UnmarshalBytes(src []byte) []byte { + src = p.ExpPacket.UnmarshalUnsafe(src) + p.Index = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + // Padding: ~ copy([6]byte(p._), src[:sizeof(byte)*6]) + src = src[1*(6):] + p.Alignment = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.AllocSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.PageSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.AllocFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.NumGPUs = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.POsEvent.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) Packed() bool { + return p.ExpPacket.Packed() && p.POsEvent.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) MarshalUnsafe(dst []byte) []byte { + if p.ExpPacket.Packed() && p.POsEvent.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS_V590 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) UnmarshalUnsafe(src []byte) []byte { + if p.ExpPacket.Packed() && p.POsEvent.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV00FD_ALLOCATION_PARAMETERS_V590 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.ExpPacket.Packed() && p.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS_V590 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.ExpPacket.Packed() && p.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS_V590 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *NV00FD_ALLOCATION_PARAMETERS_V590) WriteTo(writer io.Writer) (int64, error) { + if !p.ExpPacket.Packed() && p.POsEvent.Packed() { + // Type NV00FD_ALLOCATION_PARAMETERS_V590 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV2080_ALLOC_PARAMETERS) SizeBytes() int { return 4 @@ -1568,6 +2224,142 @@ func (n *NV503B_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV503B_ALLOC_PARAMETERS_V590) SizeBytes() int { + return 0 + + (*NV503B_ALLOC_PARAMETERS)(nil).SizeBytes() + + (*NV503B_FABRIC_P2P_DMA_INFO)(nil).SizeBytes() + + (*NV503B_FABRIC_P2P_DMA_INFO)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV503B_ALLOC_PARAMETERS_V590) MarshalBytes(dst []byte) []byte { + dst = n.NV503B_ALLOC_PARAMETERS.MarshalUnsafe(dst) + dst = n.L2pFabricP2PInfo.MarshalUnsafe(dst) + dst = n.P2lFabricP2PInfo.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV503B_ALLOC_PARAMETERS_V590) UnmarshalBytes(src []byte) []byte { + src = n.NV503B_ALLOC_PARAMETERS.UnmarshalUnsafe(src) + src = n.L2pFabricP2PInfo.UnmarshalUnsafe(src) + src = n.P2lFabricP2PInfo.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV503B_ALLOC_PARAMETERS_V590) Packed() bool { + return n.L2pFabricP2PInfo.Packed() && n.NV503B_ALLOC_PARAMETERS.Packed() && n.P2lFabricP2PInfo.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV503B_ALLOC_PARAMETERS_V590) MarshalUnsafe(dst []byte) []byte { + if n.L2pFabricP2PInfo.Packed() && n.NV503B_ALLOC_PARAMETERS.Packed() && n.P2lFabricP2PInfo.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV503B_ALLOC_PARAMETERS_V590 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV503B_ALLOC_PARAMETERS_V590) UnmarshalUnsafe(src []byte) []byte { + if n.L2pFabricP2PInfo.Packed() && n.NV503B_ALLOC_PARAMETERS.Packed() && n.P2lFabricP2PInfo.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV503B_ALLOC_PARAMETERS_V590 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV503B_ALLOC_PARAMETERS_V590) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.L2pFabricP2PInfo.Packed() && n.NV503B_ALLOC_PARAMETERS.Packed() && n.P2lFabricP2PInfo.Packed() { + // Type NV503B_ALLOC_PARAMETERS_V590 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV503B_ALLOC_PARAMETERS_V590) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV503B_ALLOC_PARAMETERS_V590) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.L2pFabricP2PInfo.Packed() && n.NV503B_ALLOC_PARAMETERS.Packed() && n.P2lFabricP2PInfo.Packed() { + // Type NV503B_ALLOC_PARAMETERS_V590 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV503B_ALLOC_PARAMETERS_V590) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV503B_ALLOC_PARAMETERS_V590) WriteTo(writer io.Writer) (int64, error) { + if !n.L2pFabricP2PInfo.Packed() && n.NV503B_ALLOC_PARAMETERS.Packed() && n.P2lFabricP2PInfo.Packed() { + // Type NV503B_ALLOC_PARAMETERS_V590 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV503B_BAR1_P2P_DMA_INFO) SizeBytes() int { return 16 @@ -1670,6 +2462,104 @@ func (n *NV503B_BAR1_P2P_DMA_INFO) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV503B_FABRIC_P2P_DMA_INFO) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV503B_FABRIC_P2P_DMA_INFO) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.Gpa)) + dst = dst[8:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV503B_FABRIC_P2P_DMA_INFO) UnmarshalBytes(src []byte) []byte { + n.Gpa = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV503B_FABRIC_P2P_DMA_INFO) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV503B_FABRIC_P2P_DMA_INFO) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV503B_FABRIC_P2P_DMA_INFO) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV503B_FABRIC_P2P_DMA_INFO) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV503B_FABRIC_P2P_DMA_INFO) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV503B_FABRIC_P2P_DMA_INFO) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV503B_FABRIC_P2P_DMA_INFO) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV503B_FABRIC_P2P_DMA_INFO) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV503C_ALLOC_PARAMETERS) SizeBytes() int { return 4 @@ -2258,6 +3148,139 @@ func (n *NVB0B5_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NVB2CC_ALLOC_PARAMETERS) SizeBytes() int { + return 0 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NVB2CC_ALLOC_PARAMETERS) MarshalBytes(dst []byte) []byte { + dst = n.HClientTarget.MarshalUnsafe(dst) + dst = n.HContextTarget.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NVB2CC_ALLOC_PARAMETERS) UnmarshalBytes(src []byte) []byte { + src = n.HClientTarget.UnmarshalUnsafe(src) + src = n.HContextTarget.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NVB2CC_ALLOC_PARAMETERS) Packed() bool { + return n.HClientTarget.Packed() && n.HContextTarget.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NVB2CC_ALLOC_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + if n.HClientTarget.Packed() && n.HContextTarget.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NVB2CC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NVB2CC_ALLOC_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + if n.HClientTarget.Packed() && n.HContextTarget.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NVB2CC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NVB2CC_ALLOC_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClientTarget.Packed() && n.HContextTarget.Packed() { + // Type NVB2CC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NVB2CC_ALLOC_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NVB2CC_ALLOC_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.HClientTarget.Packed() && n.HContextTarget.Packed() { + // Type NVB2CC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NVB2CC_ALLOC_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NVB2CC_ALLOC_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + if !n.HClientTarget.Packed() && n.HContextTarget.Packed() { + // Type NVB2CC_ALLOC_PARAMETERS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV_BSP_ALLOCATION_PARAMETERS) SizeBytes() int { return 12 @@ -2744,6 +3767,263 @@ func (n *NV_CHANNEL_ALLOC_PARAMS_V570) WriteTo(writer io.Writer) (int64, error) return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) SizeBytes() int { + return 52 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes()*NV_MAX_SUBDEVICES + + 8*NV_MAX_SUBDEVICES + + (*Handle)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + (*NV_MEMORY_DESC_PARAMS)(nil).SizeBytes() + + 4*CC_CHAN_ALLOC_IV_SIZE_DWORD + + 4*CC_CHAN_ALLOC_IV_SIZE_DWORD + + 4*CC_CHAN_ALLOC_NONCE_SIZE_DWORD +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) MarshalBytes(dst []byte) []byte { + dst = n.HObjectError.MarshalUnsafe(dst) + dst = n.HObjectBuffer.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.GPFIFOOffset)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GPFIFOEntries)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + dst = n.HContextShare.MarshalUnsafe(dst) + dst = n.HVASpace.MarshalUnsafe(dst) + dst = n.HHandleVASpace.MarshalUnsafe(dst) + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + dst = n.HUserdMemory[idx].MarshalUnsafe(dst) + } + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(n.UserdOffset[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineType)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.CID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubDeviceID)) + dst = dst[4:] + dst = n.HObjectECCError.MarshalUnsafe(dst) + dst = n.InstanceMem.MarshalUnsafe(dst) + dst = n.UserdMem.MarshalUnsafe(dst) + dst = n.RamfcMem.MarshalUnsafe(dst) + dst = n.MthdbufMem.MarshalUnsafe(dst) + dst = n.HPhysChannelGroup.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.InternalFlags)) + dst = dst[4:] + dst = n.ErrorNotifierMem.MarshalUnsafe(dst) + dst = n.ECCErrorNotifierMem.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ProcessID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.SubProcessID)) + dst = dst[4:] + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EncryptIv[idx])) + dst = dst[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.DecryptIv[idx])) + dst = dst[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_NONCE_SIZE_DWORD; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.HmacNonce[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.TPCConfigID)) + dst = dst[4:] + // Padding: dst[:sizeof(uint32)] ~= uint32(0) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) UnmarshalBytes(src []byte) []byte { + src = n.HObjectError.UnmarshalUnsafe(src) + src = n.HObjectBuffer.UnmarshalUnsafe(src) + n.GPFIFOOffset = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + n.GPFIFOEntries = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HContextShare.UnmarshalUnsafe(src) + src = n.HVASpace.UnmarshalUnsafe(src) + src = n.HHandleVASpace.UnmarshalUnsafe(src) + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + src = n.HUserdMemory[idx].UnmarshalUnsafe(src) + } + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + for idx := 0; idx < NV_MAX_SUBDEVICES; idx++ { + n.UserdOffset[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + n.EngineType = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.CID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubDeviceID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.HObjectECCError.UnmarshalUnsafe(src) + src = n.InstanceMem.UnmarshalUnsafe(src) + src = n.UserdMem.UnmarshalUnsafe(src) + src = n.RamfcMem.UnmarshalUnsafe(src) + src = n.MthdbufMem.UnmarshalUnsafe(src) + src = n.HPhysChannelGroup.UnmarshalUnsafe(src) + n.InternalFlags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.ErrorNotifierMem.UnmarshalUnsafe(src) + src = n.ECCErrorNotifierMem.UnmarshalUnsafe(src) + n.ProcessID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.SubProcessID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + n.EncryptIv[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_IV_SIZE_DWORD; idx++ { + n.DecryptIv[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + for idx := 0; idx < CC_CHAN_ALLOC_NONCE_SIZE_DWORD; idx++ { + n.HmacNonce[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + n.TPCConfigID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + // Padding: var _ uint32 ~= src[:sizeof(uint32)] + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) Packed() bool { + return n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HHandleVASpace.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) MarshalUnsafe(dst []byte) []byte { + if n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HHandleVASpace.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_CHANNEL_ALLOC_PARAMS_V610 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) UnmarshalUnsafe(src []byte) []byte { + if n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HHandleVASpace.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_CHANNEL_ALLOC_PARAMS_V610 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HHandleVASpace.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS_V610 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HHandleVASpace.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS_V610 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_CHANNEL_ALLOC_PARAMS_V610) WriteTo(writer io.Writer) (int64, error) { + if !n.ECCErrorNotifierMem.Packed() && n.ErrorNotifierMem.Packed() && n.HContextShare.Packed() && n.HHandleVASpace.Packed() && n.HObjectBuffer.Packed() && n.HObjectECCError.Packed() && n.HObjectError.Packed() && n.HPhysChannelGroup.Packed() && n.HUserdMemory[0].Packed() && n.HVASpace.Packed() && n.InstanceMem.Packed() && n.MthdbufMem.Packed() && n.RamfcMem.Packed() && n.UserdMem.Packed() { + // Type NV_CHANNEL_ALLOC_PARAMS_V610 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS) SizeBytes() int { return 5 + @@ -4729,6 +6009,348 @@ func (n *NV_MSENC_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ProhibitMultipleInstances)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineInstance)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ProhibitMultipleInstances = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.EngineInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_NVJPG_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_OFA_ALLOCATION_PARAMETERS) SizeBytes() int { + return 8 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_OFA_ALLOCATION_PARAMETERS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Size)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.ProhibitMultipleInstances)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_OFA_ALLOCATION_PARAMETERS) UnmarshalBytes(src []byte) []byte { + n.Size = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.ProhibitMultipleInstances = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_OFA_ALLOCATION_PARAMETERS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_OFA_ALLOCATION_PARAMETERS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_OFA_ALLOCATION_PARAMETERS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_OFA_ALLOCATION_PARAMETERS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_OFA_ALLOCATION_PARAMETERS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_OFA_ALLOCATION_PARAMETERS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_OFA_ALLOCATION_PARAMETERS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_OFA_ALLOCATION_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) SizeBytes() int { + return 4 + + (*NV_OFA_ALLOCATION_PARAMETERS)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) MarshalBytes(dst []byte) []byte { + dst = n.NV_OFA_ALLOCATION_PARAMETERS.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.EngineInstance)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) UnmarshalBytes(src []byte) []byte { + src = n.NV_OFA_ALLOCATION_PARAMETERS.UnmarshalUnsafe(src) + n.EngineInstance = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) Packed() bool { + return n.NV_OFA_ALLOCATION_PARAMETERS.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) MarshalUnsafe(dst []byte) []byte { + if n.NV_OFA_ALLOCATION_PARAMETERS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV_OFA_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) UnmarshalUnsafe(src []byte) []byte { + if n.NV_OFA_ALLOCATION_PARAMETERS.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV_OFA_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_OFA_ALLOCATION_PARAMETERS.Packed() { + // Type NV_OFA_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.NV_OFA_ALLOCATION_PARAMETERS.Packed() { + // Type NV_OFA_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV_OFA_ALLOCATION_PARAMETERS_V545) WriteTo(writer io.Writer) (int64, error) { + if !n.NV_OFA_ALLOCATION_PARAMETERS.Packed() { + // Type NV_OFA_ALLOCATION_PARAMETERS_V545 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV_SEMAPHORE_SURFACE_ALLOC_PARAMETERS) SizeBytes() int { return 8 + @@ -5278,6 +6900,113 @@ func (n *nv00f8Map) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) SizeBytes() int { + return 4 + + 4*NV0000_CTRL_GPU_MAX_PROBED_GPUS +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) MarshalBytes(dst []byte) []byte { + for idx := 0; idx < NV0000_CTRL_GPU_MAX_PROBED_GPUS; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GPUIDs[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.FailedID)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) UnmarshalBytes(src []byte) []byte { + for idx := 0; idx < NV0000_CTRL_GPU_MAX_PROBED_GPUS; idx++ { + n.GPUIDs[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + n.FailedID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_GPU_ATTACH_IDS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) SizeBytes() int { return 32 + @@ -5440,6 +7169,121 @@ func (n *NV0000_CTRL_GPU_GET_ID_INFO_PARAMS) WriteTo(writer io.Writer) (int64, e return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) SizeBytes() int { + return 12 + + 1*NV0000_GPU_MAX_GID_LENGTH +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.GPUID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Flags)) + dst = dst[4:] + for idx := 0; idx < NV0000_GPU_MAX_GID_LENGTH; idx++ { + dst[0] = byte(n.GPUUUID[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.UUIDStrLen)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) UnmarshalBytes(src []byte) []byte { + n.GPUID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.Flags = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NV0000_GPU_MAX_GID_LENGTH; idx++ { + n.GPUUUID[idx] = src[0] + src = src[1:] + } + n.UUIDStrLen = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (n *NV0000_CTRL_OS_UNIX_EXPORT_OBJECT) SizeBytes() int { return 4 + @@ -7575,6 +9419,300 @@ func (n *NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS) WriteTo(writer io.Writer) (in return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) SizeBytes() int { + return 8 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() + + 4*2 + + (*P64)(nil).SizeBytes() + + (*NV0080_CTRL_GR_ROUTE_INFO)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) MarshalBytes(dst []byte) []byte { + dst = n.HClientTarget.MarshalUnsafe(dst) + dst = n.HChannelTarget.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.BNonTransactional)) + dst = dst[4:] + for idx := 0; idx < 2; idx++ { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.Reserved00[idx])) + dst = dst[4:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegOpCount)) + dst = dst[4:] + dst = n.RegOps.MarshalUnsafe(dst) + dst = n.GRRouteInfo.MarshalUnsafe(dst) + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) UnmarshalBytes(src []byte) []byte { + src = n.HClientTarget.UnmarshalUnsafe(src) + src = n.HChannelTarget.UnmarshalUnsafe(src) + n.BNonTransactional = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 2; idx++ { + n.Reserved00[idx] = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + } + n.RegOpCount = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = n.RegOps.UnmarshalUnsafe(src) + src = n.GRRouteInfo.UnmarshalUnsafe(src) + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) Packed() bool { + return n.GRRouteInfo.Packed() && n.HChannelTarget.Packed() && n.HClientTarget.Packed() && n.RegOps.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) MarshalUnsafe(dst []byte) []byte { + if n.GRRouteInfo.Packed() && n.HChannelTarget.Packed() && n.HClientTarget.Packed() && n.RegOps.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] + } + // Type NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return n.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if n.GRRouteInfo.Packed() && n.HChannelTarget.Packed() && n.HClientTarget.Packed() && n.RegOps.Packed() { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return n.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.GRRouteInfo.Packed() && n.HChannelTarget.Packed() && n.HClientTarget.Packed() && n.RegOps.Packed() { + // Type NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + n.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !n.GRRouteInfo.Packed() && n.HChannelTarget.Packed() && n.HClientTarget.Packed() && n.RegOps.Packed() { + // Type NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(n.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + n.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !n.GRRouteInfo.Packed() && n.HChannelTarget.Packed() && n.HClientTarget.Packed() && n.RegOps.Packed() { + // Type NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, n.SizeBytes()) + n.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (n *NV2080_CTRL_GPU_REG_OP) SizeBytes() int { + return 32 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (n *NV2080_CTRL_GPU_REG_OP) MarshalBytes(dst []byte) []byte { + dst[0] = byte(n.RegOp) + dst = dst[1:] + dst[0] = byte(n.RegType) + dst = dst[1:] + dst[0] = byte(n.RegStatus) + dst = dst[1:] + dst[0] = byte(n.RegQuad) + dst = dst[1:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegGroupMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegSubGroupMask)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegOffset)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegValueHi)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegValueLo)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegAndNMaskHi)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(n.RegAndNMaskLo)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (n *NV2080_CTRL_GPU_REG_OP) UnmarshalBytes(src []byte) []byte { + n.RegOp = uint8(src[0]) + src = src[1:] + n.RegType = uint8(src[0]) + src = src[1:] + n.RegStatus = uint8(src[0]) + src = src[1:] + n.RegQuad = uint8(src[0]) + src = src[1:] + n.RegGroupMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.RegSubGroupMask = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.RegOffset = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.RegValueHi = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.RegValueLo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.RegAndNMaskHi = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + n.RegAndNMaskLo = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (n *NV2080_CTRL_GPU_REG_OP) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (n *NV2080_CTRL_GPU_REG_OP) MarshalUnsafe(dst []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(n), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (n *NV2080_CTRL_GPU_REG_OP) UnmarshalUnsafe(src []byte) []byte { + size := n.SizeBytes() + gohacks.Memmove(unsafe.Pointer(n), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (n *NV2080_CTRL_GPU_REG_OP) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (n *NV2080_CTRL_GPU_REG_OP) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyOutN(cc, addr, n.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (n *NV2080_CTRL_GPU_REG_OP) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (n *NV2080_CTRL_GPU_REG_OP) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return n.CopyInN(cc, addr, n.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (n *NV2080_CTRL_GPU_REG_OP) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(n))) + hdr.Len = n.SizeBytes() + hdr.Cap = n.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that n + // must live until the use above. + runtime.KeepAlive(n) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (p *NV2080_CTRL_GR_GET_INFO_PARAMS) SizeBytes() int { return 0 + @@ -8482,6 +10620,803 @@ func (p *IoctlAllocOSEvent) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (i *IoctlCardInfo) SizeBytes() int { + return 43 + + 1*3 + + (*PCIInfo)(nil).SizeBytes() + + 1*2 + + 1*10 + + 1*2 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (i *IoctlCardInfo) MarshalBytes(dst []byte) []byte { + dst[0] = byte(i.Valid) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(i.Pad0[idx]) + dst = dst[1:] + } + dst = i.PCIInfo.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.GPUID)) + dst = dst[4:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(i.InterruptLine)) + dst = dst[2:] + for idx := 0; idx < 2; idx++ { + dst[0] = byte(i.Pad1[idx]) + dst = dst[1:] + } + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.RegAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.RegSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.FBAddress)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(i.FBSize)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(i.MinorNumber)) + dst = dst[4:] + for idx := 0; idx < 10; idx++ { + dst[0] = byte(i.DevName[idx]) + dst = dst[1:] + } + for idx := 0; idx < 2; idx++ { + dst[0] = byte(i.Pad2[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (i *IoctlCardInfo) UnmarshalBytes(src []byte) []byte { + i.Valid = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + i.Pad0[idx] = src[0] + src = src[1:] + } + src = i.PCIInfo.UnmarshalUnsafe(src) + i.GPUID = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + i.InterruptLine = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + for idx := 0; idx < 2; idx++ { + i.Pad1[idx] = src[0] + src = src[1:] + } + i.RegAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.RegSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.FBAddress = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.FBSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + i.MinorNumber = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 10; idx++ { + i.DevName[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < 2; idx++ { + i.Pad2[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (i *IoctlCardInfo) Packed() bool { + return i.PCIInfo.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (i *IoctlCardInfo) MarshalUnsafe(dst []byte) []byte { + if i.PCIInfo.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(i), uintptr(size)) + return dst[size:] + } + // Type IoctlCardInfo doesn't have a packed layout in memory, fallback to MarshalBytes. + return i.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (i *IoctlCardInfo) UnmarshalUnsafe(src []byte) []byte { + if i.PCIInfo.Packed() { + size := i.SizeBytes() + gohacks.Memmove(unsafe.Pointer(i), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlCardInfo doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return i.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (i *IoctlCardInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.PCIInfo.Packed() { + // Type IoctlCardInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + i.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (i *IoctlCardInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyOutN(cc, addr, i.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (i *IoctlCardInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !i.PCIInfo.Packed() { + // Type IoctlCardInfo doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(i.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + i.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (i *IoctlCardInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return i.CopyInN(cc, addr, i.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (i *IoctlCardInfo) WriteTo(writer io.Writer) (int64, error) { + if !i.PCIInfo.Packed() { + // Type IoctlCardInfo doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, i.SizeBytes()) + i.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(i))) + hdr.Len = i.SizeBytes() + hdr.Cap = i.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that i + // must live until the use above. + runtime.KeepAlive(i) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlExportToDMABufFD) SizeBytes() int { + return 36 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes()*NV_DMABUF_EXPORT_MAX_HANDLES + + 8*NV_DMABUF_EXPORT_MAX_HANDLES + + 8*NV_DMABUF_EXPORT_MAX_HANDLES +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlExportToDMABufFD) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + dst = p.HClient.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.TotalObjects)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.NumObjects)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad0)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.TotalSize)) + dst = dst[8:] + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + dst = p.Handles[idx].MarshalUnsafe(dst) + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Offsets[idx])) + dst = dst[8:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Sizes[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad1)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlExportToDMABufFD) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HClient.UnmarshalUnsafe(src) + p.TotalObjects = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.NumObjects = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Pad0 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.TotalSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + src = p.Handles[idx].UnmarshalUnsafe(src) + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + p.Offsets[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + p.Sizes[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + p.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Pad1 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlExportToDMABufFD) Packed() bool { + return p.HClient.Packed() && p.Handles[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlExportToDMABufFD) MarshalUnsafe(dst []byte) []byte { + if p.HClient.Packed() && p.Handles[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlExportToDMABufFD doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlExportToDMABufFD) UnmarshalUnsafe(src []byte) []byte { + if p.HClient.Packed() && p.Handles[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlExportToDMABufFD doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlExportToDMABufFD) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlExportToDMABufFD) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlExportToDMABufFD) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlExportToDMABufFD) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlExportToDMABufFD) WriteTo(writer io.Writer) (int64, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlExportToDMABufFD_V570) SizeBytes() int { + return 41 + + (*Handle)(nil).SizeBytes() + + 1*3 + + (*Handle)(nil).SizeBytes()*NV_DMABUF_EXPORT_MAX_HANDLES + + 8*NV_DMABUF_EXPORT_MAX_HANDLES + + 8*NV_DMABUF_EXPORT_MAX_HANDLES +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlExportToDMABufFD_V570) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + dst = p.HClient.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.TotalObjects)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.NumObjects)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad0)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.TotalSize)) + dst = dst[8:] + dst[0] = byte(p.MappingType) + dst = dst[1:] + for idx := 0; idx < 3; idx++ { + dst[0] = byte(p.Pad1[idx]) + dst = dst[1:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + dst = p.Handles[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad2)) + dst = dst[4:] + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Offsets[idx])) + dst = dst[8:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Sizes[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad3)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlExportToDMABufFD_V570) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HClient.UnmarshalUnsafe(src) + p.TotalObjects = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.NumObjects = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Pad0 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.TotalSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.MappingType = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 3; idx++ { + p.Pad1[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + src = p.Handles[idx].UnmarshalUnsafe(src) + } + p.Pad2 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + p.Offsets[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + p.Sizes[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + p.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Pad3 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlExportToDMABufFD_V570) Packed() bool { + return p.HClient.Packed() && p.Handles[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlExportToDMABufFD_V570) MarshalUnsafe(dst []byte) []byte { + if p.HClient.Packed() && p.Handles[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlExportToDMABufFD_V570 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlExportToDMABufFD_V570) UnmarshalUnsafe(src []byte) []byte { + if p.HClient.Packed() && p.Handles[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlExportToDMABufFD_V570 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlExportToDMABufFD_V570) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD_V570 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlExportToDMABufFD_V570) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlExportToDMABufFD_V570) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD_V570 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlExportToDMABufFD_V570) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlExportToDMABufFD_V570) WriteTo(writer io.Writer) (int64, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD_V570 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *IoctlExportToDMABufFD_V580) SizeBytes() int { + return 42 + + (*Handle)(nil).SizeBytes() + + 1*2 + + (*Handle)(nil).SizeBytes()*NV_DMABUF_EXPORT_MAX_HANDLES + + 8*NV_DMABUF_EXPORT_MAX_HANDLES + + 8*NV_DMABUF_EXPORT_MAX_HANDLES +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *IoctlExportToDMABufFD_V580) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.FD)) + dst = dst[4:] + dst = p.HClient.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.TotalObjects)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.NumObjects)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Index)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad0)) + dst = dst[4:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.TotalSize)) + dst = dst[8:] + dst[0] = byte(p.MappingType) + dst = dst[1:] + dst[0] = byte(p.AllowMmap) + dst = dst[1:] + for idx := 0; idx < 2; idx++ { + dst[0] = byte(p.Pad1[idx]) + dst = dst[1:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + dst = p.Handles[idx].MarshalUnsafe(dst) + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad2)) + dst = dst[4:] + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Offsets[idx])) + dst = dst[8:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Sizes[idx])) + dst = dst[8:] + } + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Status)) + dst = dst[4:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Pad3)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *IoctlExportToDMABufFD_V580) UnmarshalBytes(src []byte) []byte { + p.FD = int32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + src = p.HClient.UnmarshalUnsafe(src) + p.TotalObjects = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.NumObjects = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Index = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Pad0 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.TotalSize = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.MappingType = uint8(src[0]) + src = src[1:] + p.AllowMmap = uint8(src[0]) + src = src[1:] + for idx := 0; idx < 2; idx++ { + p.Pad1[idx] = src[0] + src = src[1:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + src = p.Handles[idx].UnmarshalUnsafe(src) + } + p.Pad2 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + p.Offsets[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + for idx := 0; idx < NV_DMABUF_EXPORT_MAX_HANDLES; idx++ { + p.Sizes[idx] = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + } + p.Status = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Pad3 = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *IoctlExportToDMABufFD_V580) Packed() bool { + return p.HClient.Packed() && p.Handles[0].Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *IoctlExportToDMABufFD_V580) MarshalUnsafe(dst []byte) []byte { + if p.HClient.Packed() && p.Handles[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type IoctlExportToDMABufFD_V580 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *IoctlExportToDMABufFD_V580) UnmarshalUnsafe(src []byte) []byte { + if p.HClient.Packed() && p.Handles[0].Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type IoctlExportToDMABufFD_V580 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *IoctlExportToDMABufFD_V580) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD_V580 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *IoctlExportToDMABufFD_V580) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *IoctlExportToDMABufFD_V580) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD_V580 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *IoctlExportToDMABufFD_V580) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *IoctlExportToDMABufFD_V580) WriteTo(writer io.Writer) (int64, error) { + if !p.HClient.Packed() && p.Handles[0].Packed() { + // Type IoctlExportToDMABufFD_V580 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (p *IoctlFreeOSEvent) SizeBytes() int { return 8 + @@ -11984,6 +14919,128 @@ func (n *NVOS64_PARAMETERS) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *PCIInfo) SizeBytes() int { + return 12 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *PCIInfo) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.Domain)) + dst = dst[4:] + dst[0] = byte(p.Bus) + dst = dst[1:] + dst[0] = byte(p.Slot) + dst = dst[1:] + dst[0] = byte(p.Function) + dst = dst[1:] + dst[0] = byte(p.Pad0) + dst = dst[1:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.VendorID)) + dst = dst[2:] + hostarch.ByteOrder.PutUint16(dst[:2], uint16(p.DeviceID)) + dst = dst[2:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *PCIInfo) UnmarshalBytes(src []byte) []byte { + p.Domain = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + p.Bus = uint8(src[0]) + src = src[1:] + p.Slot = uint8(src[0]) + src = src[1:] + p.Function = uint8(src[0]) + src = src[1:] + p.Pad0 = uint8(src[0]) + src = src[1:] + p.VendorID = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + p.DeviceID = uint16(hostarch.ByteOrder.Uint16(src[:2])) + src = src[2:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *PCIInfo) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *PCIInfo) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *PCIInfo) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *PCIInfo) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *PCIInfo) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *PCIInfo) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *PCIInfo) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *PCIInfo) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (r *RMAPIVersion) SizeBytes() int { return 8 + @@ -13695,6 +16752,121 @@ func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) WriteTo(writer io.Writer) (int64, error) return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) SizeBytes() int { + return 20 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (p *UVM_FREE_PARAMS) SizeBytes() int { return 20 + @@ -13810,6 +16982,117 @@ func (p *UVM_FREE_PARAMS) WriteTo(writer io.Writer) (int64, error) { return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_FREE_PARAMS_V590) SizeBytes() int { + return 12 + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_FREE_PARAMS_V590) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Base)) + dst = dst[8:] + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_FREE_PARAMS_V590) UnmarshalBytes(src []byte) []byte { + p.Base = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_FREE_PARAMS_V590) Packed() bool { + return true +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_FREE_PARAMS_V590) MarshalUnsafe(dst []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_FREE_PARAMS_V590) UnmarshalUnsafe(src []byte) []byte { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_FREE_PARAMS_V590) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_FREE_PARAMS_V590) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_FREE_PARAMS_V590) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_FREE_PARAMS_V590) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_FREE_PARAMS_V590) WriteTo(writer io.Writer) (int64, error) { + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (p *UVM_INITIALIZE_PARAMS) SizeBytes() int { return 12 + @@ -15738,6 +19021,157 @@ func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) WriteTo(writer io.Writer) (int64, erro return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_SET_ACCESSED_BY_PARAMS) SizeBytes() int { + return 20 + + (*NvUUID)(nil).SizeBytes() + + 1*4 +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_SET_ACCESSED_BY_PARAMS) MarshalBytes(dst []byte) []byte { + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.RequestedBase)) + dst = dst[8:] + hostarch.ByteOrder.PutUint64(dst[:8], uint64(p.Length)) + dst = dst[8:] + dst = p.AccessedByUUID.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + for idx := 0; idx < 4; idx++ { + dst[0] = byte(p.Pad0[idx]) + dst = dst[1:] + } + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_SET_ACCESSED_BY_PARAMS) UnmarshalBytes(src []byte) []byte { + p.RequestedBase = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + p.Length = uint64(hostarch.ByteOrder.Uint64(src[:8])) + src = src[8:] + src = p.AccessedByUUID.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + for idx := 0; idx < 4; idx++ { + p.Pad0[idx] = src[0] + src = src[1:] + } + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_SET_ACCESSED_BY_PARAMS) Packed() bool { + return p.AccessedByUUID.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_SET_ACCESSED_BY_PARAMS) MarshalUnsafe(dst []byte) []byte { + if p.AccessedByUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_SET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_SET_ACCESSED_BY_PARAMS) UnmarshalUnsafe(src []byte) []byte { + if p.AccessedByUUID.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_SET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_SET_ACCESSED_BY_PARAMS) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.AccessedByUUID.Packed() { + // Type UVM_SET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_SET_ACCESSED_BY_PARAMS) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_SET_ACCESSED_BY_PARAMS) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.AccessedByUUID.Packed() { + // Type UVM_SET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_SET_ACCESSED_BY_PARAMS) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_SET_ACCESSED_BY_PARAMS) WriteTo(writer io.Writer) (int64, error) { + if !p.AccessedByUUID.Packed() { + // Type UVM_SET_ACCESSED_BY_PARAMS doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) SizeBytes() int { return 20 + @@ -16691,6 +20125,143 @@ func (p *UVM_UNREGISTER_CHANNEL_PARAMS) WriteTo(writer io.Writer) (int64, error) return int64(length), err } +// SizeBytes implements marshal.Marshallable.SizeBytes. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) SizeBytes() int { + return 4 + + (*Handle)(nil).SizeBytes() + + (*Handle)(nil).SizeBytes() +} + +// MarshalBytes implements marshal.Marshallable.MarshalBytes. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) MarshalBytes(dst []byte) []byte { + dst = p.HClient.MarshalUnsafe(dst) + dst = p.HChannel.MarshalUnsafe(dst) + hostarch.ByteOrder.PutUint32(dst[:4], uint32(p.RMStatus)) + dst = dst[4:] + return dst +} + +// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) UnmarshalBytes(src []byte) []byte { + src = p.HClient.UnmarshalUnsafe(src) + src = p.HChannel.UnmarshalUnsafe(src) + p.RMStatus = uint32(hostarch.ByteOrder.Uint32(src[:4])) + src = src[4:] + return src +} + +// Packed implements marshal.Marshallable.Packed. +// +//go:nosplit +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) Packed() bool { + return p.HChannel.Packed() && p.HClient.Packed() +} + +// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) MarshalUnsafe(dst []byte) []byte { + if p.HChannel.Packed() && p.HClient.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(p), uintptr(size)) + return dst[size:] + } + // Type UVM_UNREGISTER_CHANNEL_PARAMS_V590 doesn't have a packed layout in memory, fallback to MarshalBytes. + return p.MarshalBytes(dst) +} + +// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) UnmarshalUnsafe(src []byte) []byte { + if p.HChannel.Packed() && p.HClient.Packed() { + size := p.SizeBytes() + gohacks.Memmove(unsafe.Pointer(p), unsafe.Pointer(&src[0]), uintptr(size)) + return src[size:] + } + // Type UVM_UNREGISTER_CHANNEL_PARAMS_V590 doesn't have a packed layout in memory, fallback to UnmarshalBytes. + return p.UnmarshalBytes(src) +} + +// CopyOutN implements marshal.Marshallable.CopyOutN. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_UNREGISTER_CHANNEL_PARAMS_V590 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + p.MarshalBytes(buf) // escapes: fallback. + return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyOut implements marshal.Marshallable.CopyOut. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyOutN(cc, addr, p.SizeBytes()) +} + +// CopyInN implements marshal.Marshallable.CopyInN. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) CopyInN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) { + if !p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_UNREGISTER_CHANNEL_PARAMS_V590 doesn't have a packed layout in memory, fall back to UnmarshalBytes. + buf := cc.CopyScratchBuffer(p.SizeBytes()) // escapes: okay. + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Unmarshal unconditionally. If we had a short copy-in, this results in a + // partially unmarshalled struct. + p.UnmarshalBytes(buf) // escapes: fallback. + return length, err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := cc.CopyInBytes(addr, buf[:limit]) // escapes: okay. + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return length, err +} + +// CopyIn implements marshal.Marshallable.CopyIn. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) { + return p.CopyInN(cc, addr, p.SizeBytes()) +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) WriteTo(writer io.Writer) (int64, error) { + if !p.HChannel.Packed() && p.HClient.Packed() { + // Type UVM_UNREGISTER_CHANNEL_PARAMS_V590 doesn't have a packed layout in memory, fall back to MarshalBytes. + buf := make([]byte, p.SizeBytes()) + p.MarshalBytes(buf) + length, err := writer.Write(buf) + return int64(length), err + } + + // Construct a slice backed by dst's underlying memory. + var buf []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(p))) + hdr.Len = p.SizeBytes() + hdr.Cap = p.SizeBytes() + + length, err := writer.Write(buf) + // Since we bypassed the compiler's escape analysis, indicate that p + // must live until the use above. + runtime.KeepAlive(p) // escapes: replaced by intrinsic. + return int64(length), err +} + // SizeBytes implements marshal.Marshallable.SizeBytes. func (p *UVM_UNREGISTER_GPU_PARAMS) SizeBytes() int { return 4 + diff --git a/pkg/abi/nvgpu/uvm.go b/pkg/abi/nvgpu/uvm.go index 0226a4e..a7d1dd1 100644 --- a/pkg/abi/nvgpu/uvm.go +++ b/pkg/abi/nvgpu/uvm.go @@ -14,6 +14,10 @@ package nvgpu +import ( + "structs" +) + // UVM ioctl commands. const ( // From kernel-open/nvidia-uvm/uvm_linux_ioctl.h: @@ -37,7 +41,9 @@ const ( UVM_PAGEABLE_MEM_ACCESS = 39 UVM_SET_PREFERRED_LOCATION = 42 UVM_UNSET_PREFERRED_LOCATION = 43 + UVM_ENABLE_READ_DUPLICATION = 44 UVM_DISABLE_READ_DUPLICATION = 45 + UVM_SET_ACCESSED_BY = 46 UVM_UNSET_ACCESSED_BY = 47 UVM_MIGRATE = 51 UVM_MIGRATE_RANGE_GROUP = 53 @@ -54,6 +60,7 @@ const ( // +marshal type UVM_INITIALIZE_PARAMS struct { + _ structs.HostLayout Flags uint64 RMStatus uint32 Pad0 [4]byte @@ -76,6 +83,7 @@ const ( // +marshal type UVM_CREATE_RANGE_GROUP_PARAMS struct { + _ structs.HostLayout RangeGroupID uint64 RMStatus uint32 Pad0 [4]byte @@ -93,6 +101,7 @@ func (p *UVM_CREATE_RANGE_GROUP_PARAMS) SetStatus(status uint32) { // +marshal type UVM_DESTROY_RANGE_GROUP_PARAMS struct { + _ structs.HostLayout RangeGroupID uint64 RMStatus uint32 Pad0 [4]byte @@ -110,6 +119,7 @@ func (p *UVM_DESTROY_RANGE_GROUP_PARAMS) SetStatus(status uint32) { // +marshal type UVM_REGISTER_GPU_VASPACE_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID RMCtrlFD int32 HClient Handle @@ -139,6 +149,7 @@ func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) SetStatus(status uint32) { // +marshal type UVM_UNREGISTER_GPU_VASPACE_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID RMStatus uint32 } @@ -155,6 +166,7 @@ func (p *UVM_UNREGISTER_GPU_VASPACE_PARAMS) SetStatus(status uint32) { // +marshal type UVM_REGISTER_CHANNEL_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID RMCtrlFD int32 HClient Handle @@ -188,6 +200,7 @@ func (p *UVM_REGISTER_CHANNEL_PARAMS) SetStatus(status uint32) { // +marshal type UVM_UNREGISTER_CHANNEL_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID HClient Handle HChannel Handle @@ -204,8 +217,30 @@ func (p *UVM_UNREGISTER_CHANNEL_PARAMS) SetStatus(status uint32) { p.RMStatus = status } +// UVM_UNREGISTER_CHANNEL_PARAMS_V590 is the updated version of +// UVM_UNREGISTER_CHANNEL_PARAMS since 590.44.01. +// +// +marshal +type UVM_UNREGISTER_CHANNEL_PARAMS_V590 struct { + _ structs.HostLayout + HClient Handle + HChannel Handle + RMStatus uint32 +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_UNREGISTER_CHANNEL_PARAMS_V590) SetStatus(status uint32) { + p.RMStatus = status +} + // +marshal type UVM_ENABLE_PEER_ACCESS_PARAMS struct { + _ structs.HostLayout GPUUUIDA NvUUID GPUUUIDB NvUUID RMStatus uint32 @@ -223,6 +258,7 @@ func (p *UVM_ENABLE_PEER_ACCESS_PARAMS) SetStatus(status uint32) { // +marshal type UVM_DISABLE_PEER_ACCESS_PARAMS struct { + _ structs.HostLayout GPUUUIDA NvUUID GPUUUIDB NvUUID RMStatus uint32 @@ -240,6 +276,7 @@ func (p *UVM_DISABLE_PEER_ACCESS_PARAMS) SetStatus(status uint32) { // +marshal type UVM_SET_RANGE_GROUP_PARAMS struct { + _ structs.HostLayout RangeGroupID uint64 RequestedBase uint64 Length uint64 @@ -259,6 +296,7 @@ func (p *UVM_SET_RANGE_GROUP_PARAMS) SetStatus(status uint32) { // +marshal type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 Offset uint64 @@ -292,6 +330,7 @@ func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) SetStatus(status uint32) { // +marshal type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550 struct { + _ structs.HostLayout Base uint64 Length uint64 Offset uint64 @@ -325,6 +364,7 @@ func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS_V550) SetStatus(status uint32) { // +marshal type UVM_FREE_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 RMStatus uint32 @@ -341,8 +381,30 @@ func (p *UVM_FREE_PARAMS) SetStatus(status uint32) { p.RMStatus = status } +// UVM_FREE_PARAMS_V590 is the updated version of +// UVM_FREE_PARAMS since 590.44.01. +// +// +marshal +type UVM_FREE_PARAMS_V590 struct { + _ structs.HostLayout + Base uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_FREE_PARAMS_V590) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_FREE_PARAMS_V590) SetStatus(status uint32) { + p.RMStatus = status +} + // +marshal type UVM_REGISTER_GPU_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID NumaEnabled uint8 Pad [3]byte @@ -375,6 +437,7 @@ func (p *UVM_REGISTER_GPU_PARAMS) SetStatus(status uint32) { // +marshal type UVM_UNREGISTER_GPU_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID RMStatus uint32 } @@ -391,6 +454,7 @@ func (p *UVM_UNREGISTER_GPU_PARAMS) SetStatus(status uint32) { // +marshal type UVM_PAGEABLE_MEM_ACCESS_PARAMS struct { + _ structs.HostLayout PageableMemAccess uint8 Pad [3]byte RMStatus uint32 @@ -408,6 +472,7 @@ func (p *UVM_PAGEABLE_MEM_ACCESS_PARAMS) SetStatus(status uint32) { // +marshal type UVM_SET_PREFERRED_LOCATION_PARAMS struct { + _ structs.HostLayout RequestedBase uint64 Length uint64 PreferredLocation NvUUID @@ -427,6 +492,7 @@ func (p *UVM_SET_PREFERRED_LOCATION_PARAMS) SetStatus(status uint32) { // +marshal type UVM_SET_PREFERRED_LOCATION_PARAMS_V550 struct { + _ structs.HostLayout RequestedBase uint64 Length uint64 PreferredLocation NvUUID @@ -446,6 +512,7 @@ func (p *UVM_SET_PREFERRED_LOCATION_PARAMS_V550) SetStatus(status uint32) { // +marshal type UVM_UNSET_PREFERRED_LOCATION_PARAMS struct { + _ structs.HostLayout RequestedBase uint64 Length uint64 RMStatus uint32 @@ -462,8 +529,28 @@ func (p *UVM_UNSET_PREFERRED_LOCATION_PARAMS) SetStatus(status uint32) { p.RMStatus = status } +// +marshal +type UVM_ENABLE_READ_DUPLICATION_PARAMS struct { + _ structs.HostLayout + RequestedBase uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_ENABLE_READ_DUPLICATION_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + // +marshal type UVM_DISABLE_READ_DUPLICATION_PARAMS struct { + _ structs.HostLayout RequestedBase uint64 Length uint64 RMStatus uint32 @@ -480,8 +567,29 @@ func (p *UVM_DISABLE_READ_DUPLICATION_PARAMS) SetStatus(status uint32) { p.RMStatus = status } +// +marshal +type UVM_SET_ACCESSED_BY_PARAMS struct { + _ structs.HostLayout + RequestedBase uint64 + Length uint64 + AccessedByUUID NvUUID + RMStatus uint32 + Pad0 [4]byte +} + +// GetStatus implements HasStatus.GetStatus. +func (p *UVM_SET_ACCESSED_BY_PARAMS) GetStatus() uint32 { + return p.RMStatus +} + +// SetStatus implements HasStatus.SetStatus. +func (p *UVM_SET_ACCESSED_BY_PARAMS) SetStatus(status uint32) { + p.RMStatus = status +} + // +marshal type UVM_UNSET_ACCESSED_BY_PARAMS struct { + _ structs.HostLayout RequestedBase uint64 Length uint64 AccessedByUUID NvUUID @@ -501,6 +609,7 @@ func (p *UVM_UNSET_ACCESSED_BY_PARAMS) SetStatus(status uint32) { // +marshal type UVM_MIGRATE_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 DestinationUUID NvUUID @@ -530,6 +639,7 @@ func (p *UVM_MIGRATE_PARAMS) SetStatus(status uint32) { // // +marshal type UVM_MIGRATE_PARAMS_V550 struct { + _ structs.HostLayout Base uint64 Length uint64 DestinationUUID NvUUID @@ -556,6 +666,7 @@ func (p *UVM_MIGRATE_PARAMS_V550) SetStatus(status uint32) { // +marshal type UVM_MIGRATE_RANGE_GROUP_PARAMS struct { + _ structs.HostLayout RangeGroupID uint64 DestinationUUID NvUUID RMStatus uint32 @@ -574,6 +685,7 @@ func (p *UVM_MIGRATE_RANGE_GROUP_PARAMS) SetStatus(status uint32) { // +marshal type UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS struct { + _ structs.HostLayout Buffer uint64 Size uint64 TargetVA uint64 @@ -594,6 +706,7 @@ func (p *UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS) SetStatus(status uint32) { // +marshal type UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS struct { + _ structs.HostLayout Buffer uint64 Size uint64 TargetVA uint64 @@ -614,6 +727,7 @@ func (p *UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS) SetStatus(status uint32) { // +marshal type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 GPUUUID NvUUID @@ -633,6 +747,7 @@ func (p *UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS) SetStatus(status uint32) { // +marshal type UVM_UNMAP_EXTERNAL_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 GPUUUID NvUUID @@ -652,6 +767,7 @@ func (p *UVM_UNMAP_EXTERNAL_PARAMS) SetStatus(status uint32) { // +marshal type UVM_ALLOC_SEMAPHORE_POOL_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 PerGPUAttributes [UVM_MAX_GPUS]UvmGpuMappingAttributes @@ -672,6 +788,7 @@ func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS) SetStatus(status uint32) { // +marshal type UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550 struct { + _ structs.HostLayout Base uint64 Length uint64 PerGPUAttributes [UVM_MAX_GPUS_V2]UvmGpuMappingAttributes @@ -692,6 +809,7 @@ func (p *UVM_ALLOC_SEMAPHORE_POOL_PARAMS_V550) SetStatus(status uint32) { // +marshal type UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS struct { + _ structs.HostLayout GPUUUID NvUUID PageableMemAccess uint8 Pad [3]byte @@ -710,6 +828,7 @@ func (p *UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS) SetStatus(status uint32) { // +marshal type UVM_VALIDATE_VA_RANGE_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 RMStatus uint32 @@ -728,6 +847,7 @@ func (p *UVM_VALIDATE_VA_RANGE_PARAMS) SetStatus(status uint32) { // +marshal type UVM_CREATE_EXTERNAL_RANGE_PARAMS struct { + _ structs.HostLayout Base uint64 Length uint64 RMStatus uint32 @@ -746,6 +866,7 @@ func (p *UVM_CREATE_EXTERNAL_RANGE_PARAMS) SetStatus(status uint32) { // +marshal type UVM_MM_INITIALIZE_PARAMS struct { + _ structs.HostLayout UvmFD int32 RMStatus uint32 } @@ -769,6 +890,7 @@ const ( // +marshal type UvmGpuMappingAttributes struct { + _ structs.HostLayout GPUUUID NvUUID GPUMappingType uint32 GPUCachingType uint32 diff --git a/pkg/abi/tpu/tpu.go b/pkg/abi/tpu/tpu.go index e31c647..bc96a6c 100644 --- a/pkg/abi/tpu/tpu.go +++ b/pkg/abi/tpu/tpu.go @@ -51,11 +51,11 @@ const ( // TPUV6ePFDeviceID is PCI device ID of TPU V6e hardware PF. TPUV6ePFDeviceID = 0x006e - // TPUV6pDeviceID is the PCI device ID of TPU V6p hardware. - TPUV6pDeviceID = 0x0075 + // TPU7xDeviceID is the PCI device ID of TPU 7x hardware. + TPU7xDeviceID = 0x0075 - // TPUV6pPFDeviceID is the PCI device ID of TPU V6p hardware PF. - TPUV6pPFDeviceID = 0x0076 + // TPU7xPFDeviceID is the PCI device ID of TPU 7x hardware PF. + TPU7xPFDeviceID = 0x0076 ) // TPUV4InterruptsMap maps BAR indices to valid register offsets. diff --git a/pkg/bits/bits32.go b/pkg/bits/bits32.go deleted file mode 100644 index 28134a9..0000000 --- a/pkg/bits/bits32.go +++ /dev/null @@ -1,33 +0,0 @@ -package bits - -// IsOn returns true if *all* bits set in 'bits' are set in 'mask'. -func IsOn32(mask, bits uint32) bool { - return mask&bits == bits -} - -// IsAnyOn returns true if *any* bit set in 'bits' is set in 'mask'. -func IsAnyOn32(mask, bits uint32) bool { - return mask&bits != 0 -} - -// Mask returns a T with all of the given bits set. -func Mask32(is ...int) uint32 { - ret := uint32(0) - for _, i := range is { - ret |= MaskOf32(i) - } - return ret -} - -// MaskOf is like Mask, but sets only a single bit (more efficiently). -func MaskOf32(i int) uint32 { - return uint32(1) << uint32(i) -} - -// IsPowerOfTwo returns true if v is power of 2. -func IsPowerOfTwo32(v uint32) bool { - if v == 0 { - return false - } - return v&(v-1) == 0 -} diff --git a/pkg/bits/bits64.go b/pkg/bits/bits64.go deleted file mode 100644 index 73117b1..0000000 --- a/pkg/bits/bits64.go +++ /dev/null @@ -1,33 +0,0 @@ -package bits - -// IsOn returns true if *all* bits set in 'bits' are set in 'mask'. -func IsOn64(mask, bits uint64) bool { - return mask&bits == bits -} - -// IsAnyOn returns true if *any* bit set in 'bits' is set in 'mask'. -func IsAnyOn64(mask, bits uint64) bool { - return mask&bits != 0 -} - -// Mask returns a T with all of the given bits set. -func Mask64(is ...int) uint64 { - ret := uint64(0) - for _, i := range is { - ret |= MaskOf64(i) - } - return ret -} - -// MaskOf is like Mask, but sets only a single bit (more efficiently). -func MaskOf64(i int) uint64 { - return uint64(1) << uint64(i) -} - -// IsPowerOfTwo returns true if v is power of 2. -func IsPowerOfTwo64(v uint64) bool { - if v == 0 { - return false - } - return v&(v-1) == 0 -} diff --git a/pkg/bits/bits_template.go b/pkg/bits/bits_template.go new file mode 100644 index 0000000..0d65312 --- /dev/null +++ b/pkg/bits/bits_template.go @@ -0,0 +1,51 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bits + +import "golang.org/x/exp/constraints" + +// Non-atomic bit operations on integral types. + +// IsOn returns true if *all* bits set in 'bits' are set in 'mask'. +func IsOn[T constraints.Integer](mask, bits T) bool { + return mask&bits == bits +} + +// IsAnyOn returns true if *any* bit set in 'bits' is set in 'mask'. +func IsAnyOn[T constraints.Integer](mask, bits T) bool { + return mask&bits != 0 +} + +// Mask returns a T with all of the given bits set. +func Mask[T constraints.Integer](is ...int) T { + ret := T(0) + for _, i := range is { + ret |= MaskOf[T](i) + } + return ret +} + +// MaskOf is like Mask, but sets only a single bit (more efficiently). +func MaskOf[T constraints.Integer](i int) T { + return T(1) << T(i) +} + +// IsPowerOfTwo returns true if v is power of 2. +func IsPowerOfTwo[T constraints.Integer](v T) bool { + if v == 0 { + return false + } + return v&(v-1) == 0 +} diff --git a/pkg/bits/uint64_arch.go b/pkg/bits/uint64_arch.go index fc56341..c31dc60 100644 --- a/pkg/bits/uint64_arch.go +++ b/pkg/bits/uint64_arch.go @@ -32,6 +32,6 @@ func ForEachSetBit64(x uint64, f func(i int)) { for x != 0 { i := TrailingZeros64(x) f(i) - x &^= MaskOf64(i) + x &^= MaskOf[uint64](i) } } diff --git a/pkg/buffer/buffer_state_autogen.go b/pkg/buffer/buffer_state_autogen.go index 1cf2bd0..30c35c7 100644 --- a/pkg/buffer/buffer_state_autogen.go +++ b/pkg/buffer/buffer_state_autogen.go @@ -24,8 +24,8 @@ func (b *Buffer) beforeSave() {} // +checklocksignore func (b *Buffer) StateSave(stateSinkObject state.Sink) { b.beforeSave() - var dataValue []byte - dataValue = b.saveData() + dataValue := b.saveData() + _ = ([]byte)(dataValue) stateSinkObject.SaveValue(0, dataValue) stateSinkObject.Save(1, &b.size) } diff --git a/pkg/buffer/view_unsafe.go b/pkg/buffer/view_unsafe.go index cef7e7e..b03fec9 100644 --- a/pkg/buffer/view_unsafe.go +++ b/pkg/buffer/view_unsafe.go @@ -15,12 +15,10 @@ package buffer import ( - "reflect" "unsafe" ) // BasePtr returns a pointer to the view's chunk. func (v *View) BasePtr() *byte { - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&v.chunk.data)) - return (*byte)(unsafe.Pointer(hdr.Data)) + return unsafe.SliceData(v.chunk.data) } diff --git a/pkg/compressio/compressio.go b/pkg/compressio/compressio.go index 6f10475..b965fd6 100644 --- a/pkg/compressio/compressio.go +++ b/pkg/compressio/compressio.go @@ -280,7 +280,7 @@ type pool struct { // // This should only be called once. func (p *pool) init(key []byte, workers int, compress bool, level int) { - if key != nil { + if len(key) > 0 { p.hashPool = &hashPool{key: key} } p.workers = make([]worker, workers) @@ -402,6 +402,9 @@ func NewReader(in io.ReadCloser, key []byte) (*Reader, error) { r.hashPool.putHash(h) sum := make([]byte, len(r.lastSum)) if _, err := io.ReadFull(r.in, sum); err != nil { + if err == io.EOF { + return nil, io.ErrUnexpectedEOF + } return nil, err } if !hmac.Equal(r.lastSum, sum) { diff --git a/pkg/compressio/nocompressio.go b/pkg/compressio/nocompressio.go index d257b8a..4ba7533 100644 --- a/pkg/compressio/nocompressio.go +++ b/pkg/compressio/nocompressio.go @@ -85,7 +85,7 @@ func NewSimpleReader(in io.ReadCloser, key []byte) *SimpleReader { source: in, bin: bin, } - if key != nil { + if len(key) > 0 { r.h = hmac.New(sha256.New, key) } return r @@ -124,10 +124,7 @@ func (r *SimpleReader) Read(p []byte) (int, error) { n, err := r.bin.Read(p[:toRead]) if err != nil { if err == io.EOF { - // this only can happen if storage or data size is corrupted, - // but we have no other means to detect it earlier as we store - // hash after the data block. - return n, ErrHashMismatch + return n, io.ErrUnexpectedEOF } return n, err } @@ -209,7 +206,7 @@ var ( // comments for details. chunkSize is the buffer size used for buffering. Large // writes are not buffered and written out directly as a single chunk. func NewSimpleWriter(out io.Writer, key []byte, chunkSize uint32) *SimpleWriter { - if key == nil { + if len(key) == 0 { // Since there is no key, this image doesn't use the data integrity stream // format mentioned in package comments. We can just use a bufio writer. return &SimpleWriter{ diff --git a/pkg/context/context.go b/pkg/context/context.go index dc0e1ef..5ce7491 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -58,25 +58,32 @@ type Blocker interface { // is interrupted. Block(C <-chan struct{}) error + // BlockWithTimeout blocks until an event is received from C, the timeout + // has elapsed (only if haveTimeout is true), or some external interrupt. + // + // It returns: + // - The remaining timeout, which is guaranteed to be 0 if the timeout + // expired, and is unspecified if haveTimeout is false. + // - An error which if the timeout expired or if interrupted. + BlockWithTimeout(C chan struct{}, haveTimeout bool, timeout time.Duration) (time.Duration, error) + // BlockWithTimeoutOn blocks until either the conditions of Block are // satisfied, or the timeout is hit. Note that deadlines are not supported // since the notion of "with respect to what clock" is not resolved. // - // The return value is per BlockOn. + // It returns: + // - The remaining timeout, which is guaranteed to be 0 if the timeout + // expired. + // - Boolean as per BlockOn return value. BlockWithTimeoutOn(waiter.Waitable, waiter.EventMask, time.Duration) (time.Duration, bool) // UninterruptibleSleepStart indicates the beginning of an uninterruptible - // sleep state (equivalent to Linux's TASK_UNINTERRUPTIBLE). If deactivate - // is true and the Context represents a Task, the Task's AddressSpace is - // deactivated. - UninterruptibleSleepStart(deactivate bool) + // sleep state (equivalent to Linux's TASK_UNINTERRUPTIBLE). + UninterruptibleSleepStart() // UninterruptibleSleepFinish indicates the end of an uninterruptible sleep - // state that was begun by a previous call to UninterruptibleSleepStart. If - // activate is true and the Context represents a Task, the Task's - // AddressSpace is activated. Normally activate is the same value as the - // deactivate parameter passed to UninterruptibleSleepStart. - UninterruptibleSleepFinish(activate bool) + // state that was begun by a previous call to UninterruptibleSleepStart. + UninterruptibleSleepFinish() } // NoTask is an implementation of Blocker that does not block. @@ -94,7 +101,7 @@ func (nt *NoTask) Interrupt() { // Interrupted implements Blocker.Interrupted. func (nt *NoTask) Interrupted() bool { - return nt.cancel != nil && len(nt.cancel) > 0 + return len(nt.cancel) > 0 } // Killed implements Blocker.Killed. @@ -131,34 +138,47 @@ func (nt *NoTask) BlockOn(w waiter.Waitable, mask waiter.EventMask) bool { } } -// BlockWithTimeoutOn implements Blocker.BlockWithTimeoutOn. -func (nt *NoTask) BlockWithTimeoutOn(w waiter.Waitable, mask waiter.EventMask, duration time.Duration) (time.Duration, bool) { +// BlockWithTimeout implements Blocker.BlockWithTimeout. +func (nt *NoTask) BlockWithTimeout(C chan struct{}, haveTimeout bool, timeout time.Duration) (time.Duration, error) { + if !haveTimeout { + return timeout, nt.Block(C) + } + if nt.cancel == nil { nt.cancel = make(chan struct{}, 1) } - e, ch := waiter.NewChannelEntry(mask) - w.EventRegister(&e) - defer w.EventUnregister(&e) start := time.Now() // In system time. - t := time.AfterFunc(duration, func() { ch <- struct{}{} }) + remainingTimeout := func() time.Duration { + rt := timeout - time.Since(start) + if rt < 0 { + rt = 0 + } + return rt + } select { case <-nt.cancel: - return time.Since(start), false // Interrupted. - case _, ok := <-ch: - if ok && t.Stop() { - // Timer never fired. - return time.Since(start), ok - } - // Timer fired, remain is zero. - return time.Duration(0), ok + return remainingTimeout(), errors.New("interrupted system call") // Interrupted. + case <-C: + return remainingTimeout(), nil + case <-time.After(timeout): + return 0, errors.New("timeout expired") } } +// BlockWithTimeoutOn implements Blocker.BlockWithTimeoutOn. +func (nt *NoTask) BlockWithTimeoutOn(w waiter.Waitable, mask waiter.EventMask, timeout time.Duration) (time.Duration, bool) { + e, ch := waiter.NewChannelEntry(mask) + w.EventRegister(&e) + defer w.EventUnregister(&e) + left, err := nt.BlockWithTimeout(ch, true, timeout) + return left, err == nil +} + // UninterruptibleSleepStart implmenents Blocker.UninterruptedSleepStart. -func (*NoTask) UninterruptibleSleepStart(bool) {} +func (*NoTask) UninterruptibleSleepStart() {} // UninterruptibleSleepFinish implmenents Blocker.UninterruptibleSleepFinish. -func (*NoTask) UninterruptibleSleepFinish(bool) {} +func (*NoTask) UninterruptibleSleepFinish() {} // Context represents a thread of execution (hereafter "goroutine" to reflect // Go idiosyncrasy). It carries state associated with the goroutine across API @@ -236,3 +256,28 @@ func (ctx *withValue) Value(key any) any { } return ctx.Context.Value(key) } + +// WithValues returns a copy of parent in which the values associated with keys +// are the corresponding values in the map. +func WithValues(parent Context, values map[any]any) Context { + if len(values) == 0 { + return parent + } + return &withValues{ + Context: parent, + values: values, + } +} + +type withValues struct { + Context + values map[any]any +} + +// Value implements Context.Value. +func (ctx *withValues) Value(key any) any { + if val, ok := ctx.values[key]; ok { + return val + } + return ctx.Context.Value(key) +} diff --git a/pkg/cpuid/cpuid.go b/pkg/cpuid/cpuid.go index 129d3b9..18fa1f8 100644 --- a/pkg/cpuid/cpuid.go +++ b/pkg/cpuid/cpuid.go @@ -233,9 +233,10 @@ func readHWCap(auxvFilepath string) (hwCap, error) { for i := 0; i < l; i++ { tag := binary.LittleEndian.Uint64(auxv[i*16:]) val := binary.LittleEndian.Uint64(auxv[i*16+8:]) - if tag == _AT_HWCAP { + switch tag { + case _AT_HWCAP: c.hwCap1 = val - } else if tag == _AT_HWCAP2 { + case _AT_HWCAP2: c.hwCap2 = val } @@ -249,7 +250,7 @@ func readHWCap(auxvFilepath string) (hwCap, error) { func initHWCap() { c, err := readHWCap("/proc/self/auxv") if err != nil { - log.Warningf("cpuid HWCap not initialized: %w", err) + log.Warningf("cpuid HWCap not initialized: %v", err) } else { hostFeatureSet.hwCap = c } diff --git a/pkg/cpuid/cpuid_amd64.go b/pkg/cpuid/cpuid_amd64.go index 0f7e1d0..11072e7 100644 --- a/pkg/cpuid/cpuid_amd64.go +++ b/pkg/cpuid/cpuid_amd64.go @@ -192,7 +192,24 @@ func (fs FeatureSet) VirtualAddressBits() uint32 { //go:nosplit func (fs FeatureSet) PhysicalAddressBits() uint32 { ax, _, _, _ := fs.query(addressSizes) - return ax & 0xff + physBits := ax & 0xff + if !fs.AMD() { + return physBits + } + + maxExtended, _, _, _ := fs.query(extendedFunctionInfo) + if maxExtended < uint32(amdMemoryEncryptionInfo) { + return physBits + } + + memEncAX, memEncBX, _, _ := fs.query(amdMemoryEncryptionInfo) + if memEncAX&amdMemoryEncryptionFeatureMask == 0 { + return physBits + } + // AMD memory encryption reduces usable physical address width by the + // CPUID-reported amount. Match Linux's + // arch/x86/kernel/cpu/amd.c:early_detect_mem_encrypt(). + return physBits - ((memEncBX >> amdPhysAddrReductionShift) & amdPhysAddrReductionMask) } // CacheType describes the type of a cache, as returned in eax[4:0] for eax=4. @@ -382,15 +399,16 @@ var ( ) const ( - // XCR0AMXMask are the bits that enable xsave to operate on AMX TILECFG - // and TILEDATA. - // - // Note: TILECFG and TILEDATA are always either both enabled or both - // disabled. + // XCR0AmxCfgMask is the bits that enable xsave to operate on + // AMX TILECFG. // // See Intel® 64 and IA-32 Architectures Software Developer’s Manual Vol.1 // section 13.3 for details. - XCR0AMXMask = uint64((1 << 17) | (1 << 18)) + XCR0AmxCfgMask = uint64(1 << 17) + + // XCR0AmxDataMask is the bits that enable xsave to operate on + // AMX TILEDATA. + XCR0AmxDataMask = uint64(1 << 18) ) // ExtendedStateSize returns the number of bytes needed to save the "extended @@ -415,13 +433,15 @@ func (fs FeatureSet) ExtendedStateSize() (size, align uint) { // AMXExtendedStateSize returns the number of bytes within the "extended state" // area that is used for AMX. func (fs FeatureSet) AMXExtendedStateSize() uint { + total := uint(0) if fs.UseXsave() { xcr0 := xgetbv(0) - if (xcr0 & XCR0AMXMask) != 0 { - return uint(amxTileCfgSize + amxTileDataSize) + // TILECFG is not part of AMX extended state, only TILEDATA. + if (xcr0 & XCR0AmxDataMask) != 0 { + total += uint(amxTileDataSize) } } - return 0 + return total } // ValidXCR0Mask returns the valid bits in control register XCR0. @@ -435,7 +455,7 @@ func (fs FeatureSet) ValidXCR0Mask() uint64 { return 0 } ax, _, _, dx := fs.query(xSaveInfo) - return (uint64(dx)<<32 | uint64(ax)) &^ XCR0AMXMask + return (uint64(dx)<<32 | uint64(ax)) &^ (XCR0AmxCfgMask | XCR0AmxDataMask) } // UseXsave returns the choice of fp state saving instruction. diff --git a/pkg/cpuid/cpuid_amd64_state_autogen.go b/pkg/cpuid/cpuid_amd64_state_autogen.go index a487bbc..1f3d61c 100644 --- a/pkg/cpuid/cpuid_amd64_state_autogen.go +++ b/pkg/cpuid/cpuid_amd64_state_autogen.go @@ -27,8 +27,8 @@ func (fs *FeatureSet) beforeSave() {} // +checklocksignore func (fs *FeatureSet) StateSave(stateSinkObject state.Sink) { fs.beforeSave() - var FunctionValue Static - FunctionValue = fs.saveFunction() + FunctionValue := fs.saveFunction() + _ = (Static)(FunctionValue) stateSinkObject.SaveValue(0, FunctionValue) stateSinkObject.Save(1, &fs.hwCap) } diff --git a/pkg/cpuid/native_amd64.go b/pkg/cpuid/native_amd64.go index fb2bc5c..dbfceb1 100644 --- a/pkg/cpuid/native_amd64.go +++ b/pkg/cpuid/native_amd64.go @@ -63,15 +63,25 @@ const xSaveInfoNumLeaves = 64 // Maximum number of xSaveInfo leaves. // The "extended" functions. const ( - extendedStart cpuidFunction = 0x80000000 - extendedFunctionInfo cpuidFunction = extendedStart + 0 // Returns highest available extended function in eax. - extendedFeatures = extendedStart + 1 // Returns some extended feature bits in edx and ecx. - processorBrandString2 = extendedStart + 2 // Processor Name String Identifier. - processorBrandString3 = extendedStart + 3 // Processor Name String Identifier. - processorBrandString4 = extendedStart + 4 // Processor Name String Identifier. - l1CacheAndTLBInfo = extendedStart + 5 // Returns L2 cache information. - l2CacheInfo = extendedStart + 6 // Returns L2 cache information. - addressSizes = extendedStart + 8 // Physical and virtual address sizes. + extendedStart cpuidFunction = 0x80000000 + extendedFunctionInfo cpuidFunction = extendedStart + 0 // Returns highest available extended function in eax. + extendedFeatures = extendedStart + 1 // Returns some extended feature bits in edx and ecx. + processorBrandString2 = extendedStart + 2 // Processor Name String Identifier. + processorBrandString3 = extendedStart + 3 // Processor Name String Identifier. + processorBrandString4 = extendedStart + 4 // Processor Name String Identifier. + l1CacheAndTLBInfo = extendedStart + 5 // Returns L2 cache information. + l2CacheInfo = extendedStart + 6 // Returns L2 cache information. + addressSizes = extendedStart + 8 // Physical and virtual address sizes. + amdMemoryEncryptionInfo = extendedStart + 31 // AMD memory encryption information. +) + +// AMD-defined memory encryption feature bits and fields. +const ( + amdSecureMemoryEncryption = 1 << 0 + amdSecureEncryptedVirtualization = 1 << 1 + amdMemoryEncryptionFeatureMask = amdSecureMemoryEncryption | amdSecureEncryptedVirtualization + amdPhysAddrReductionShift = 6 + amdPhysAddrReductionMask = 0x3f ) var allowedBasicFunctions = [...]bool{ @@ -84,14 +94,15 @@ var allowedBasicFunctions = [...]bool{ } var allowedExtendedFunctions = [...]bool{ - extendedFunctionInfo - extendedStart: true, - extendedFeatures - extendedStart: true, - addressSizes - extendedStart: true, - processorBrandString2 - extendedStart: true, - processorBrandString3 - extendedStart: true, - processorBrandString4 - extendedStart: true, - l1CacheAndTLBInfo - extendedStart: true, - l2CacheInfo - extendedStart: true, + extendedFunctionInfo - extendedStart: true, + extendedFeatures - extendedStart: true, + addressSizes - extendedStart: true, + processorBrandString2 - extendedStart: true, + processorBrandString3 - extendedStart: true, + processorBrandString4 - extendedStart: true, + l1CacheAndTLBInfo - extendedStart: true, + l2CacheInfo - extendedStart: true, + amdMemoryEncryptionInfo - extendedStart: true, } // Function executes a CPUID function. @@ -119,7 +130,7 @@ func (i *In) normalize() { switch cpuidFunction(i.Eax) { case vendorID, featureInfo, intelCacheDescriptors, extendedFunctionInfo, extendedFeatures: i.Ecx = 0 // Ignore. - case processorBrandString2, processorBrandString3, processorBrandString4, l1CacheAndTLBInfo, l2CacheInfo: + case processorBrandString2, processorBrandString3, processorBrandString4, l1CacheAndTLBInfo, l2CacheInfo, amdMemoryEncryptionInfo: i.Ecx = 0 // Ignore. case intelDeterministicCacheParams, extendedFeatureInfo: // Preserve i.Ecx. diff --git a/pkg/ebpf/ebpf.go b/pkg/ebpf/ebpf.go new file mode 100644 index 0000000..22dae0a --- /dev/null +++ b/pkg/ebpf/ebpf.go @@ -0,0 +1,82 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ebpf provides tools for working with extended Berkely Packet Filter (eBPF) programs. +// +// gVisor currently does not support running eBPF programs. +package ebpf + +import ( + "github.com/sagernet/gvisor/pkg/abi/linux" +) + +// BPFID is a sequential, globally-unique (though unloaded +// programs' IDs are reused) ID for an eBPF program. +type BPFID uint32 + +// UnverifiedProgram represents an eBPF program provided by userspace that has not +// been validated. +// +// +stateify savable +type UnverifiedProgram struct { + // instructions is a list of eBPF instructions. + // + // Immutable. + instructions []linux.EBPFInstruction +} + +// NewUnverifiedProgram creates an unverified eBPF program from a set of instructions. +func NewUnverifiedProgram(instructions []linux.EBPFInstruction) UnverifiedProgram { + return UnverifiedProgram{ + instructions: instructions, + } +} + +// Program represents an eBPF program that has been validated. +// +// All fields of Program are immutable. +// +// +stateify savable +type Program struct { + // instructions is a list of eBPF instructions. + instructions []linux.EBPFInstruction + + // id is the program's ID. + id BPFID + + // progType is the program's type. + progType linux.BPFProgramType +} + +// ID returns the unique identifier for the eBPF program. +func (p *Program) ID() BPFID { + return p.id +} + +// ProgType returns the type of the eBPF program. +func (p *Program) ProgType() linux.BPFProgramType { + return p.progType +} + +// Validate validates an unverified eBPF program. +// +// Currently, no validation is performed, so the resulting program MUST not be run. +func (uprog *UnverifiedProgram) Validate(id BPFID, progType linux.BPFProgramType) (Program, error) { + prog := Program{ + instructions: uprog.instructions, + id: id, + progType: progType, + } + return prog, nil +} diff --git a/pkg/ebpf/ebpf_state_autogen.go b/pkg/ebpf/ebpf_state_autogen.go new file mode 100644 index 0000000..558e9df --- /dev/null +++ b/pkg/ebpf/ebpf_state_autogen.go @@ -0,0 +1,70 @@ +// automatically generated by stateify. + +package ebpf + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (uprog *UnverifiedProgram) StateTypeName() string { + return "pkg/ebpf.UnverifiedProgram" +} + +func (uprog *UnverifiedProgram) StateFields() []string { + return []string{ + "instructions", + } +} + +func (uprog *UnverifiedProgram) beforeSave() {} + +// +checklocksignore +func (uprog *UnverifiedProgram) StateSave(stateSinkObject state.Sink) { + uprog.beforeSave() + stateSinkObject.Save(0, &uprog.instructions) +} + +func (uprog *UnverifiedProgram) afterLoad(context.Context) {} + +// +checklocksignore +func (uprog *UnverifiedProgram) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &uprog.instructions) +} + +func (p *Program) StateTypeName() string { + return "pkg/ebpf.Program" +} + +func (p *Program) StateFields() []string { + return []string{ + "instructions", + "id", + "progType", + } +} + +func (p *Program) beforeSave() {} + +// +checklocksignore +func (p *Program) StateSave(stateSinkObject state.Sink) { + p.beforeSave() + stateSinkObject.Save(0, &p.instructions) + stateSinkObject.Save(1, &p.id) + stateSinkObject.Save(2, &p.progType) +} + +func (p *Program) afterLoad(context.Context) {} + +// +checklocksignore +func (p *Program) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &p.instructions) + stateSourceObject.Load(1, &p.id) + stateSourceObject.Load(2, &p.progType) +} + +func init() { + state.Register((*UnverifiedProgram)(nil)) + state.Register((*Program)(nil)) +} diff --git a/pkg/ebpf/types.go b/pkg/ebpf/types.go new file mode 100644 index 0000000..16dd5ac --- /dev/null +++ b/pkg/ebpf/types.go @@ -0,0 +1,155 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebpf + +import ( + "github.com/sagernet/gvisor/pkg/abi/linux" +) + +// CgroupAttachType is an attachment type that is valid for a cgroup eBPF program. +type CgroupAttachType uint + +// Subset of attachment types that are valid for cgroup eBPF programs. +const ( + CGROUP_INET_INGRESS CgroupAttachType = iota + CGROUP_INET_EGRESS + CGROUP_INET_SOCK_CREATE + CGROUP_SOCK_OPS + CGROUP_DEVICE + CGROUP_INET4_BIND + CGROUP_INET6_BIND + CGROUP_INET4_CONNECT + CGROUP_INET6_CONNECT + CGROUP_UNIX_CONNECT + CGROUP_INET4_POST_BIND + CGROUP_INET6_POST_BIND + CGROUP_UDP4_SENDMSG + CGROUP_UDP6_SENDMSG + CGROUP_UNIX_SENDMSG + CGROUP_SYSCTL + CGROUP_UDP4_RECVMSG + CGROUP_UDP6_RECVMSG + CGROUP_UNIX_RECVMSG + CGROUP_GETSOCKOPT + CGROUP_SETSOCKOPT + CGROUP_INET4_GETPEERNAME + CGROUP_INET6_GETPEERNAME + CGROUP_UNIX_GETPEERNAME + CGROUP_INET4_GETSOCKNAME + CGROUP_INET6_GETSOCKNAME + CGROUP_UNIX_GETSOCKNAME + CGROUP_INET_SOCK_RELEASE + MAX_CGROUP_BPF_ATTACH_TYPE uint = iota +) + +// AttachType is an interface implemented by each subsystem's subset of valid +// attachment types. +type AttachType interface { + // MatchingProgType returns the program type that expects to be attached + // at this attachment type. + // + // Analogous to kernel/bpf/syscall.c:attach_type_to_prog_type() in Linux. + MatchingProgType() linux.BPFProgramType +} + +// MatchingProgType implements AttachType.MatchingProgType. +func (c CgroupAttachType) MatchingProgType() linux.BPFProgramType { + switch c { + case CGROUP_INET_INGRESS, CGROUP_INET_EGRESS: + return linux.BPF_PROG_TYPE_CGROUP_SKB + case CGROUP_INET_SOCK_CREATE, CGROUP_INET_SOCK_RELEASE, CGROUP_INET4_POST_BIND, CGROUP_INET6_POST_BIND: + return linux.BPF_PROG_TYPE_CGROUP_SOCK + case CGROUP_INET4_BIND, CGROUP_INET6_BIND, CGROUP_INET4_CONNECT, CGROUP_INET6_CONNECT, CGROUP_UNIX_CONNECT, CGROUP_INET4_GETPEERNAME, CGROUP_INET6_GETPEERNAME, CGROUP_UNIX_GETPEERNAME, CGROUP_INET4_GETSOCKNAME, CGROUP_INET6_GETSOCKNAME, CGROUP_UNIX_GETSOCKNAME, CGROUP_UDP4_SENDMSG, CGROUP_UDP6_SENDMSG, CGROUP_UNIX_SENDMSG, CGROUP_UDP4_RECVMSG, CGROUP_UDP6_RECVMSG, CGROUP_UNIX_RECVMSG: + return linux.BPF_PROG_TYPE_CGROUP_SOCK_ADDR + case CGROUP_SOCK_OPS: + return linux.BPF_PROG_TYPE_SOCK_OPS + case CGROUP_DEVICE: + return linux.BPF_PROG_TYPE_CGROUP_DEVICE + case CGROUP_SYSCTL: + return linux.BPF_PROG_TYPE_CGROUP_SYSCTL + case CGROUP_GETSOCKOPT, CGROUP_SETSOCKOPT: + return linux.BPF_PROG_TYPE_CGROUP_SOCKOPT + default: + return linux.BPF_PROG_TYPE_UNSPEC + } +} + +// ParseAttachmentType takes a raw linux.BPFAttachType provided by userspace and converts +// it an appropriately-typed BPFAttachType object depending on which subsystem it is to be +// attached to. +// +// ParseAttachmentType returns nil if the attachment type is unknown or unsupported. +func ParseAttachmentType(b linux.BPFAttachType) AttachType { + switch b { + case linux.BPF_CGROUP_INET_INGRESS: + return CGROUP_INET_INGRESS + case linux.BPF_CGROUP_INET_EGRESS: + return CGROUP_INET_EGRESS + case linux.BPF_CGROUP_INET_SOCK_CREATE: + return CGROUP_INET_SOCK_CREATE + case linux.BPF_CGROUP_SOCK_OPS: + return CGROUP_SOCK_OPS + case linux.BPF_CGROUP_DEVICE: + return CGROUP_DEVICE + case linux.BPF_CGROUP_INET4_BIND: + return CGROUP_INET4_BIND + case linux.BPF_CGROUP_INET6_BIND: + return CGROUP_INET6_BIND + case linux.BPF_CGROUP_INET4_CONNECT: + return CGROUP_INET4_CONNECT + case linux.BPF_CGROUP_INET6_CONNECT: + return CGROUP_INET6_CONNECT + case linux.BPF_CGROUP_UNIX_CONNECT: + return CGROUP_UNIX_CONNECT + case linux.BPF_CGROUP_INET4_POST_BIND: + return CGROUP_INET4_POST_BIND + case linux.BPF_CGROUP_INET6_POST_BIND: + return CGROUP_INET6_POST_BIND + case linux.BPF_CGROUP_UDP4_SENDMSG: + return CGROUP_UDP4_SENDMSG + case linux.BPF_CGROUP_UDP6_SENDMSG: + return CGROUP_UDP6_SENDMSG + case linux.BPF_CGROUP_UNIX_SENDMSG: + return CGROUP_UNIX_SENDMSG + case linux.BPF_CGROUP_SYSCTL: + return CGROUP_SYSCTL + case linux.BPF_CGROUP_UDP4_RECVMSG: + return CGROUP_UDP4_RECVMSG + case linux.BPF_CGROUP_UDP6_RECVMSG: + return CGROUP_UDP6_RECVMSG + case linux.BPF_CGROUP_UNIX_RECVMSG: + return CGROUP_UNIX_RECVMSG + case linux.BPF_CGROUP_GETSOCKOPT: + return CGROUP_GETSOCKOPT + case linux.BPF_CGROUP_SETSOCKOPT: + return CGROUP_SETSOCKOPT + case linux.BPF_CGROUP_INET4_GETPEERNAME: + return CGROUP_INET4_GETPEERNAME + case linux.BPF_CGROUP_INET6_GETPEERNAME: + return CGROUP_INET6_GETPEERNAME + case linux.BPF_CGROUP_UNIX_GETPEERNAME: + return CGROUP_UNIX_GETPEERNAME + case linux.BPF_CGROUP_INET4_GETSOCKNAME: + return CGROUP_INET4_GETSOCKNAME + case linux.BPF_CGROUP_INET6_GETSOCKNAME: + return CGROUP_INET6_GETSOCKNAME + case linux.BPF_CGROUP_UNIX_GETSOCKNAME: + return CGROUP_UNIX_GETSOCKNAME + case linux.BPF_CGROUP_INET_SOCK_RELEASE: + return CGROUP_INET_SOCK_RELEASE + } + + return nil +} diff --git a/pkg/errors/linuxerr/internal.go b/pkg/errors/linuxerr/internal.go index 004d4d7..ea3797e 100644 --- a/pkg/errors/linuxerr/internal.go +++ b/pkg/errors/linuxerr/internal.go @@ -122,7 +122,7 @@ func SyscallRestartErrorFromReturn(rv uintptr) (*errors.Error, bool) { // ConvertIntr converts the provided error code (err) to another one (intr) if // the first error corresponds to an interrupted operation. func ConvertIntr(err, intr error) error { - if err == ErrInterrupted { + if Equals(ErrInterrupted, err) { return intr } return err diff --git a/pkg/fd/fd.go b/pkg/fd/fd.go index 4f1d039..551f51b 100644 --- a/pkg/fd/fd.go +++ b/pkg/fd/fd.go @@ -271,3 +271,13 @@ func (f *FD) File() (*os.File, error) { func (f *FD) ReleaseToFile(name string) *os.File { return os.NewFile(uintptr(f.Release()), name) } + +// ReleaseToFiles returns the result of calling ReleaseToFile on each FD in +// the given slice. +func ReleaseToFiles(fds []*FD, name string) []*os.File { + files := make([]*os.File, len(fds)) + for i, fd := range fds { + files[i] = fd.ReleaseToFile(name) + } + return files +} diff --git a/pkg/fdnotifier/fdnotifier.go b/pkg/fdnotifier/fdnotifier.go index da936cc..3749e19 100644 --- a/pkg/fdnotifier/fdnotifier.go +++ b/pkg/fdnotifier/fdnotifier.go @@ -41,6 +41,9 @@ type notifier struct { // notifications. epFD int + // pauseMu synchronizes notifications with save/restore. + pauseMu sync.Mutex + // mu protects fdMap. mu sync.Mutex @@ -162,6 +165,7 @@ func (n *notifier) waitAndNotify() error { } notified := false + n.pauseMu.Lock() n.mu.Lock() for i := 0; i < v; i++ { if fi, ok := n.fdMap[e[i].Fd]; ok { @@ -170,6 +174,7 @@ func (n *notifier) waitAndNotify() error { } } n.mu.Unlock() + n.pauseMu.Unlock() if notified { // Let goroutines woken by Notify get a chance to run before we // epoll_wait again. @@ -178,18 +183,31 @@ func (n *notifier) waitAndNotify() error { } } +// pause suspends notifications until resume is called. +func (n *notifier) pause() { + n.pauseMu.Lock() +} + +// resume ends the effect of a previous call to pause. +func (n *notifier) resume() { + n.pauseMu.Unlock() +} + var shared struct { notifier *notifier once sync.Once initErr error } -// AddFD adds an FD to the list of observed FDs. -func AddFD(fd int32, queue *waiter.Queue) error { +func ensureSharedNotifier() { shared.once.Do(func() { shared.notifier, shared.initErr = newNotifier() }) +} +// AddFD adds an FD to the list of observed FDs. +func AddFD(fd int32, queue *waiter.Queue) error { + ensureSharedNotifier() if shared.initErr != nil { return shared.initErr } @@ -213,3 +231,14 @@ func RemoveFD(fd int32) { func HasFD(fd int32) bool { return shared.notifier.hasFD(fd) } + +// Pause suspends notifications until Resume is called. +func Pause() { + ensureSharedNotifier() + shared.notifier.pause() +} + +// Resume ends the effect of a previous call to Pause. +func Resume() { + shared.notifier.resume() +} diff --git a/pkg/flipcall/flipcall.go b/pkg/flipcall/flipcall.go index 05c57d1..ddead6c 100644 --- a/pkg/flipcall/flipcall.go +++ b/pkg/flipcall/flipcall.go @@ -170,6 +170,22 @@ func (ep *Endpoint) DataCap() uint32 { return ep.dataCap } +// DataAddr returns the start address of the datagram part of ep's packet +// window as a uintptr. Equivalently, DataAddr returns &ep.Data()[0] as a +// uintptr. It returns uintptr instead of unsafe.Pointer so that code using +// flipcall cannot obtain unsafe.Pointers without importing the unsafe package +// directly. +func (ep *Endpoint) DataAddr() uintptr { + return ep.packet + PacketHeaderBytes +} + +// DataEndAddr returns the address of the first byte after ep's packet window, +// as a uintptr. Note that in Go (unlike in C), such an address cannot safely +// be converted to a pointer. +func (ep *Endpoint) DataEndAddr() uintptr { + return ep.packet + PacketHeaderBytes + uintptr(ep.dataCap) +} + // Connection state. const ( // The client is, by definition, initially active, so this must be 0. @@ -180,6 +196,9 @@ const ( // Connect blocks until the peer Endpoint has called Endpoint.RecvFirst(). // +// Flipcall uses the packet window to negotiate the connection, so callers must +// not use the packet window until Connect returns. +// // Preconditions: // - ep is a client Endpoint. // - ep.Connect(), ep.RecvFirst(), ep.SendRecv(), and ep.SendLast() have never @@ -195,6 +214,9 @@ func (ep *Endpoint) Connect() error { // RecvFirst blocks until the peer Endpoint calls Endpoint.SendRecv(), then // returns the datagram length specified by that call. // +// Flipcall uses the packet window to negotiate the connection, so callers must +// not use the packet window until RecvFirst returns. +// // Preconditions: // - ep is a server Endpoint. // - ep.SendRecv(), ep.RecvFirst(), and ep.SendLast() have never been called. diff --git a/pkg/flipcall/flipcall_unsafe.go b/pkg/flipcall/flipcall_unsafe.go index c2928c6..b2d27bb 100644 --- a/pkg/flipcall/flipcall_unsafe.go +++ b/pkg/flipcall/flipcall_unsafe.go @@ -15,7 +15,6 @@ package flipcall import ( - "reflect" "unsafe" "github.com/sagernet/gvisor/pkg/atomicbitops" @@ -62,12 +61,9 @@ func (ep *Endpoint) dataLen() *atomicbitops.Uint32 { // - Writers must not assume that they will read back the same data that they // have written. In other words, writers should avoid reading from Data() at // all. -func (ep *Endpoint) Data() (bs []byte) { - bshdr := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) - bshdr.Data = ep.packet + PacketHeaderBytes - bshdr.Len = int(ep.dataCap) - bshdr.Cap = int(ep.dataCap) - return +func (ep *Endpoint) Data() []byte { + ptr := unsafe.Pointer(ep.packet + PacketHeaderBytes) + return unsafe.Slice((*byte)(ptr), int(ep.dataCap)) } // ioSync is a dummy variable used to indicate synchronization to the Go race diff --git a/pkg/fspath/fspath.go b/pkg/fspath/fspath.go index 4c983d5..3f9f614 100644 --- a/pkg/fspath/fspath.go +++ b/pkg/fspath/fspath.go @@ -161,10 +161,7 @@ func (it Iterator) Next() Iterator { // aren't at the end of the path, there is definitely another path // component. i := it.end + 1 - for { - if it.partialPathname[i] != pathSep { - break - } + for it.partialPathname[i] == pathSep { i++ } nextPartialPathname := it.partialPathname[i:] diff --git a/pkg/fsutil/fsutil.go b/pkg/fsutil/fsutil.go index e331b9e..3f07896 100644 --- a/pkg/fsutil/fsutil.go +++ b/pkg/fsutil/fsutil.go @@ -18,12 +18,18 @@ package fsutil import "golang.org/x/sys/unix" +// SelfFilestorePrefix is the prefix of the self filestore file name. +const SelfFilestorePrefix = ".gvisor.filestore." + // DirentHandler is a function that handles a dirent. type DirentHandler func(ino uint64, off int64, ftype uint8, name string, reclen uint16) // ForEachDirent retrieves all dirents from dirfd using getdents64(2) and // invokes handleDirent on them. func ForEachDirent(dirfd int, handleDirent DirentHandler) error { + if _, err := unix.Seek(dirfd, unix.SEEK_SET, 0); err != nil { + return err + } var direntsBuf [8192]byte for { n, err := unix.Getdents(dirfd, direntsBuf[:]) @@ -46,3 +52,10 @@ func DirentNames(dirfd int) ([]string, error) { }) return names, err } + +// RenameAt is a convenience wrapper to make the renameat syscall. +// +// RenameAt is equivalent to RenameAt2 with a zero flags argument. +func RenameAt(oldDirFD int, oldName string, newDirFD int, newName string) error { + return RenameAt2(oldDirFD, oldName, newDirFD, newName, 0) +} diff --git a/pkg/fsutil/fsutil_unsafe.go b/pkg/fsutil/fsutil_unsafe.go index bc898fa..058a499 100644 --- a/pkg/fsutil/fsutil_unsafe.go +++ b/pkg/fsutil/fsutil_unsafe.go @@ -15,6 +15,7 @@ package fsutil import ( + "bytes" "unsafe" "github.com/sagernet/gvisor/pkg/syserr" @@ -53,9 +54,8 @@ func Utimensat(dirFd int, name string, times [2]unix.Timespec, flags int) error return nil } -// RenameAt is a convenience wrapper to make the renameat(2) syscall. It -// additionally handles empty names. -func RenameAt(oldDirFD int, oldName string, newDirFD int, newName string) error { +// RenameAt2 is a convenience wrapper to make the renameat syscall. +func RenameAt2(oldDirFD int, oldName string, newDirFD int, newName string, flags uint32) error { var oldNamePtr unsafe.Pointer if oldName != "" { nameBytes, err := unix.BytePtrFromString(oldName) @@ -74,12 +74,12 @@ func RenameAt(oldDirFD int, oldName string, newDirFD int, newName string) error } if _, _, errno := unix.Syscall6( - unix.SYS_RENAMEAT, + unix.SYS_RENAMEAT2, uintptr(oldDirFD), uintptr(oldNamePtr), uintptr(newDirFD), uintptr(newNamePtr), - 0, + uintptr(flags), 0); errno != 0 { return syserr.FromHost(errno).ToError() } @@ -93,22 +93,15 @@ func ParseDirents(buf []byte, handleDirent DirentHandler) { // Interpret the buf populated by unix.Getdents as unix.Dirent. dirent := *(*unix.Dirent)(unsafe.Pointer(&buf[0])) + // Extract the name from buf, since len(unix.Dirent.Name) == 256 is + // incorrect in general. The name is null-terminated. + nameBuf := buf[unsafe.Offsetof(dirent.Name):dirent.Reclen] + nameLen := bytes.IndexByte(nameBuf, 0) + name := string(nameBuf[:nameLen]) + // Advance buf for the next dirent. buf = buf[dirent.Reclen:] - // Extracting the name is pretty tedious... - var nameBuf [unix.NAME_MAX]byte - var nameLen int - for i := 0; i < len(dirent.Name); i++ { - // The name is null terminated. - if dirent.Name[i] == 0 { - nameLen = i - break - } - nameBuf[i] = byte(dirent.Name[i]) - } - name := string(nameBuf[:nameLen]) - // Skip `.` and `..` entries. It is anyways ignored by the client. We also // don't want to leak information about `..`. if name == "." || name == ".." { diff --git a/pkg/goid/goid_122_arm64.s b/pkg/goid/goid_122_arm64.s deleted file mode 100644 index ec59b4b..0000000 --- a/pkg/goid/goid_122_arm64.s +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !go1.23 - -#include "textflag.h" - -#define GOID_OFFSET 152 // +checkoffset runtime g.goid - -// func goid() int64 -TEXT ·goid(SB),NOSPLIT,$0-8 - MOVD g, R0 // g - MOVD GOID_OFFSET(R0), R0 - MOVD R0, ret+0(FP) - RET diff --git a/pkg/goid/goid_123_arm64.s b/pkg/goid/goid_123_arm64.s deleted file mode 100644 index c5ea245..0000000 --- a/pkg/goid/goid_123_arm64.s +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build go1.23 && !go1.25 - -#include "textflag.h" - -#define GOID_OFFSET 160 // +checkoffset runtime g.goid - -// func goid() int64 -TEXT ·goid(SB),NOSPLIT,$0-8 - MOVD g, R0 // g - MOVD GOID_OFFSET(R0), R0 - MOVD R0, ret+0(FP) - RET diff --git a/pkg/goid/goid_125_amd64.s b/pkg/goid/goid_asm_impl_amd64.s similarity index 91% rename from pkg/goid/goid_125_amd64.s rename to pkg/goid/goid_asm_impl_amd64.s index 796459d..1087871 100644 --- a/pkg/goid/goid_125_amd64.s +++ b/pkg/goid/goid_asm_impl_amd64.s @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.25 +//go:build amd64 #include "textflag.h" -#define GOID_OFFSET 152 // +checkoffset runtime g.goid +#define GOID_OFFSET 152 // func goid() int64 TEXT ·goid(SB),NOSPLIT|NOFRAME,$0-8 diff --git a/pkg/goid/goid_125_arm64.s b/pkg/goid/goid_asm_impl_arm64.s similarity index 91% rename from pkg/goid/goid_125_arm64.s rename to pkg/goid/goid_asm_impl_arm64.s index e6879a1..9bd8bed 100644 --- a/pkg/goid/goid_125_arm64.s +++ b/pkg/goid/goid_asm_impl_arm64.s @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.25 +//go:build arm64 #include "textflag.h" -#define GOID_OFFSET 152 // +checkoffset runtime g.goid +#define GOID_OFFSET 152 // func goid() int64 TEXT ·goid(SB),NOSPLIT,$0-8 diff --git a/pkg/gomaxprocs/gomaxprocs_mutex.go b/pkg/gomaxprocs/gomaxprocs_mutex.go index 512e803..b54d72c 100644 --- a/pkg/gomaxprocs/gomaxprocs_mutex.go +++ b/pkg/gomaxprocs/gomaxprocs_mutex.go @@ -60,5 +60,5 @@ func gomaxprocsinitLockNames() {} func init() { gomaxprocsinitLockNames() - gomaxprocsprefixIndex = locking.NewMutexClass(reflect.TypeOf(gomaxprocsMutex{}), gomaxprocslockNames) + gomaxprocsprefixIndex = locking.NewMutexClass(reflect.TypeFor[gomaxprocsMutex](), gomaxprocslockNames) } diff --git a/pkg/hostarch/addr.go b/pkg/hostarch/addr.go index 2f7dcf1..11c41ea 100644 --- a/pkg/hostarch/addr.go +++ b/pkg/hostarch/addr.go @@ -101,6 +101,18 @@ func (v Addr) ToRange(length uint64) (AddrRange, bool) { return AddrRange{v, end}, ok } +// MustToRange is equivalent to ToRange, but panics if the end of the range +// wraps around. +// +//go:nosplit +func (v Addr) MustToRange(length uint64) AddrRange { + ar, ok := v.ToRange(length) + if !ok { + panic("hostarch.Addr.ToRange() wraps") + } + return ar +} + // IsPageAligned returns true if ar.Start.IsPageAligned() and // ar.End.IsPageAligned(). func (ar AddrRange) IsPageAligned() bool { diff --git a/pkg/hostarch/hostarch.go b/pkg/hostarch/hostarch.go index af0bc93..da45836 100644 --- a/pkg/hostarch/hostarch.go +++ b/pkg/hostarch/hostarch.go @@ -4,5 +4,23 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd. -// Package hostarch contains host arch address operations for user memory. +// Package hostarch contains machine architecture parameters. package hostarch + +import ( + "encoding/binary" + "fmt" +) + +// EndianString returns "little" if the invoking process is little-endian and +// "big" if the invoking process is big-endian. +func EndianString() string { + switch val := binary.NativeEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04}); val { + case 0x01020304: + return "big" + case 0x04030201: + return "little" + default: + panic(fmt.Sprintf("unknown endianness: [01 02 03 04] => %#x", val)) + } +} diff --git a/pkg/hostarch/hostarch_arm64.go b/pkg/hostarch/hostarch_arm64.go index 82666eb..4016264 100644 --- a/pkg/hostarch/hostarch_arm64.go +++ b/pkg/hostarch/hostarch_arm64.go @@ -13,36 +13,30 @@ // limitations under the License. //go:build arm64 -// +build arm64 package hostarch -import ( - "encoding/binary" - - "golang.org/x/sys/unix" -) +import "encoding/binary" const ( // PageSize is the system page size. // arm64 support 4K/16K/64K page size, // which can be get by unix.Getpagesize(). - // Currently, only 4K page size is supported. + // Currently, only 4K page size is supported by default, + // or 64K pages by using the pagesize_64k build tag. PageSize = 1 << PageShift // HugePageSize is the system huge page size. HugePageSize = 1 << HugePageShift + // JumboPageSize is the 1GB jumbo page size. + JumboPageSize = 1 << JumboPageShift + // CacheLineSize is the size of the cache line. CacheLineSize = 1 << CacheLineShift - // PageShift is the binary log of the system page size. - PageShift = 12 - - // HugePageShift is the binary log of the system huge page size. - // Should be calculated by "PageShift + (PageShift - 3)" - // when multiple page size support is ready. - HugePageShift = 21 + // JumboPageShift is the binary log of jumbo page whose size is 1GB. + JumboPageShift = 30 // CacheLineShift is the binary log of the cache line size. CacheLineShift = 6 @@ -89,10 +83,3 @@ func ESRAccessType(code uint64) AccessType { func UntaggedUserAddr(addr Addr) Addr { return Addr(int64(addr<<8) >> 8) } - -func init() { - // Make sure the page size is 4K on arm64 platform. - if size := unix.Getpagesize(); size != PageSize { - panic("Only 4K page size is supported on arm64!") - } -} diff --git a/pkg/sync/runtime_constants_not_go125.go b/pkg/hostarch/hostarch_arm64_4k.go similarity index 51% rename from pkg/sync/runtime_constants_not_go125.go rename to pkg/hostarch/hostarch_arm64_4k.go index b27a899..d1c012a 100644 --- a/pkg/sync/runtime_constants_not_go125.go +++ b/pkg/hostarch/hostarch_arm64_4k.go @@ -1,4 +1,4 @@ -// Copyright 2023 The gVisor Authors. +// Copyright 2026 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,14 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -// https://go.dev/cl/670497 (1.25) adds a new wait reason, adjusting the value of waitReasonSemacquire. -//go:build !go1.25 +//go:build arm64 && !pagesize_64k -package sync +package hostarch + +import "golang.org/x/sys/unix" -// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. const ( - WaitReasonSelect uint8 = 9 // +checkconst runtime waitReasonSelect - WaitReasonChanReceive uint8 = 14 // +checkconst runtime waitReasonChanReceive - WaitReasonSemacquire uint8 = 18 // +checkconst runtime waitReasonSemacquire + // PageShift is the binary log of the system page size. + // 4K pages: 2^12 = 4096 + PageShift = 12 + + // HugePageShift is the binary log of the system huge page size. + // For 4K pages: PageShift + (PageShift - 3) = 12 + 9 = 21 + // This gives 2MB huge pages. + HugePageShift = 21 ) + +func init() { + if size := unix.Getpagesize(); size != PageSize { + println("WARNING: host page size mismatch - running on non-4K host") + } +} diff --git a/pkg/hostarch/hostarch_arm64_4k_state_autogen.go b/pkg/hostarch/hostarch_arm64_4k_state_autogen.go new file mode 100644 index 0000000..5a78957 --- /dev/null +++ b/pkg/hostarch/hostarch_arm64_4k_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build arm64 && !pagesize_64k +// +build arm64,!pagesize_64k + +package hostarch diff --git a/pkg/hostarch/hostarch_arm64_64k.go b/pkg/hostarch/hostarch_arm64_64k.go new file mode 100644 index 0000000..0d93be4 --- /dev/null +++ b/pkg/hostarch/hostarch_arm64_64k.go @@ -0,0 +1,36 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 && pagesize_64k + +package hostarch + +import "golang.org/x/sys/unix" + +const ( + // PageShift is the binary log of the system page size. + // 64K pages: 2^16 = 65536 + PageShift = 16 + + // HugePageShift is the binary log of the system huge page size. + // For 64K pages: PageShift + (PageShift - 3) = 16 + 13 = 29 + // This gives 512MB huge pages. + HugePageShift = 29 +) + +func init() { + if size := unix.Getpagesize(); size != PageSize { + println("WARNING: host page size mismatch - running on non-64K host") + } +} diff --git a/pkg/hostarch/hostarch_arm64_64k_state_autogen.go b/pkg/hostarch/hostarch_arm64_64k_state_autogen.go new file mode 100644 index 0000000..567e323 --- /dev/null +++ b/pkg/hostarch/hostarch_arm64_64k_state_autogen.go @@ -0,0 +1,6 @@ +// automatically generated by stateify. + +//go:build arm64 && pagesize_64k +// +build arm64,pagesize_64k + +package hostarch diff --git a/pkg/hostarch/hostarch_x86.go b/pkg/hostarch/hostarch_x86.go index eb0405b..9c04aae 100644 --- a/pkg/hostarch/hostarch_x86.go +++ b/pkg/hostarch/hostarch_x86.go @@ -26,6 +26,9 @@ const ( // HugePageSize is the system huge page size. HugePageSize = 1 << HugePageShift + // JumboPageSize is the 1GB jumbo page size. + JumboPageSize = 1 << JumboPageShift + // CacheLineSize is the size of the cache line. CacheLineSize = 1 << CacheLineShift @@ -35,6 +38,9 @@ const ( // HugePageShift is the binary log of the system huge page size. HugePageShift = 21 + // JumboPageShift is the binary log of jumbo page whose size is 1GB. + JumboPageShift = 30 + // CacheLineShift is the binary log of the cache line size. CacheLineShift = 6 ) diff --git a/pkg/hostarch/sizes_util.go b/pkg/hostarch/sizes_util.go index d798675..d53c9b1 100644 --- a/pkg/hostarch/sizes_util.go +++ b/pkg/hostarch/sizes_util.go @@ -11,10 +11,11 @@ const ( PageMask = PageSize - 1 HugePageMask = HugePageSize - 1 CacheLineMask = CacheLineSize - 1 + JumboPageMask = ^uintptr(JumboPageSize - 1) ) type bytecount interface { - ~uint | ~uint16 | ~uint32 | ~uint64 | ~uintptr + ~uint | ~uint32 | ~uint64 | ~uintptr } type hugebytecount interface { @@ -112,3 +113,13 @@ func CacheLineRoundUp[T bytecount](x T) (val T, ok bool) { ok = val >= x return } + +// MustCacheLineRoundUp is equivalent to CacheLineRoundUp, but panics if +// rounding up overflows. +func MustCacheLineRoundUp[T bytecount](x T) T { + val, ok := CacheLineRoundUp(x) + if !ok { + panic("CacheLineRoundUp overflows") + } + return val +} diff --git a/pkg/hosttid/hosttid_124_amd64.s b/pkg/hosttid/hosttid_124_amd64.s deleted file mode 100644 index 3047ce0..0000000 --- a/pkg/hosttid/hosttid_124_amd64.s +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !go1.25 - -#include "textflag.h" - -#define M_OFFSET 48 // +checkoffset runtime g.m -#define PROCID_OFFSET 72 // +checkoffset runtime m.procid - -TEXT ·Current(SB),NOSPLIT|NOFRAME,$0-8 - // procid is in getg().m.procid. - MOVQ TLS, AX - MOVQ 0(AX)(TLS*1), AX - MOVQ M_OFFSET(AX), AX // gp.m - MOVQ PROCID_OFFSET(AX), AX // mp.procid - MOVQ AX, ret+0(FP) - RET diff --git a/pkg/hosttid/hosttid_124_arm64.s b/pkg/hosttid/hosttid_124_arm64.s deleted file mode 100644 index 9dd1011..0000000 --- a/pkg/hosttid/hosttid_124_arm64.s +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !go1.25 - -#include "textflag.h" - -#define M_OFFSET 48 // +checkoffset runtime g.m -#define PROCID_OFFSET 72 // +checkoffset runtime m.procid - -TEXT ·Current(SB),NOSPLIT,$0-8 - // procid is in getg().m.procid. - MOVD g, R0 // g - MOVD M_OFFSET(R0), R0 // gp.m - MOVD PROCID_OFFSET(R0), R0 // mp.procid - MOVD R0, ret+0(FP) - RET diff --git a/pkg/hosttid/hosttid_125_amd64.s b/pkg/hosttid/hosttid_asm_impl_amd64.s similarity index 86% rename from pkg/hosttid/hosttid_125_amd64.s rename to pkg/hosttid/hosttid_asm_impl_amd64.s index 37c915a..fc1720c 100644 --- a/pkg/hosttid/hosttid_125_amd64.s +++ b/pkg/hosttid/hosttid_asm_impl_amd64.s @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.25 +//go:build amd64 #include "textflag.h" -#define M_OFFSET 48 // +checkoffset runtime g.m -#define PROCID_OFFSET 64 // +checkoffset runtime m.procid +#define M_OFFSET 48 +#define PROCID_OFFSET 64 TEXT ·Current(SB),NOSPLIT|NOFRAME,$0-8 // procid is in getg().m.procid. diff --git a/pkg/hosttid/hosttid_125_arm64.s b/pkg/hosttid/hosttid_asm_impl_arm64.s similarity index 86% rename from pkg/hosttid/hosttid_125_arm64.s rename to pkg/hosttid/hosttid_asm_impl_arm64.s index e450ad0..59e8aad 100644 --- a/pkg/hosttid/hosttid_125_arm64.s +++ b/pkg/hosttid/hosttid_asm_impl_arm64.s @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.25 +//go:build arm64 #include "textflag.h" -#define M_OFFSET 48 // +checkoffset runtime g.m -#define PROCID_OFFSET 64 // +checkoffset runtime m.procid +#define M_OFFSET 48 +#define PROCID_OFFSET 64 TEXT ·Current(SB),NOSPLIT,$0-8 // procid is in getg().m.procid. diff --git a/pkg/log/bug.go b/pkg/log/bug.go index 234d63e..a253c39 100644 --- a/pkg/log/bug.go +++ b/pkg/log/bug.go @@ -40,14 +40,14 @@ func reportBugErr(caller int, err error) { func reportBug(caller int, msg string, vars []any) { var b strings.Builder if _, file, line, ok := runtime.Caller(caller); ok { - b.WriteString(fmt.Sprintf(warnFmtStr, file, line)) + fmt.Fprintf(&b, warnFmtStr, file, line) } else { b.WriteString(warnUnknownLineStr) } b.WriteByte('\n') if len(msg) > 0 { if len(vars) > 0 { - b.WriteString(fmt.Sprintf(msg, vars...)) + fmt.Fprintf(&b, msg, vars...) } else { b.WriteString(msg) } @@ -77,11 +77,11 @@ func reportBugOnce(caller int, msg string, vars []any) { defer warnedMu.Unlock() if _, ok = warnedSet[key]; !ok { - b.WriteString(fmt.Sprintf(warnFmtStr, file, line)) + fmt.Fprintf(&b, warnFmtStr, file, line) b.WriteByte('\n') if len(msg) > 0 { if len(vars) > 0 { - b.WriteString(fmt.Sprintf(msg, vars...)) + fmt.Fprintf(&b, msg, vars...) } else { b.WriteString(msg) } @@ -102,7 +102,7 @@ func reportBugOnce(caller int, msg string, vars []any) { b.WriteByte('\n') if len(msg) > 0 { if len(vars) > 0 { - b.WriteString(fmt.Sprintf(msg, vars...)) + fmt.Fprintf(&b, msg, vars...) } else { b.WriteString(msg) } diff --git a/pkg/log/file.go b/pkg/log/file.go new file mode 100644 index 0000000..d200b97 --- /dev/null +++ b/pkg/log/file.go @@ -0,0 +1,60 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "fmt" + "os" + "path/filepath" +) + +// FileOpts contains options for creating a log file. +type FileOpts interface { + // Build constructs the log file path based on the given pattern. + Build(logPattern string) string +} + +// DefaultFileOpts is the default implementation of FileOpts which supports no +// variable substitution. +type DefaultFileOpts struct{} + +// Build implements FileOpts.Build. +func (f *DefaultFileOpts) Build(logPattern string) string { + return logPattern +} + +// OpenFile opens a log file using the specified flags. It uses `opts` to +// construct the log file path based on the given `logPattern`. +func OpenFile(logPattern string, flags int, opts FileOpts) (*os.File, error) { + if len(logPattern) == 0 { + return nil, nil + } + + // Replace variables in the log pattern. + logPath := opts.Build(logPattern) + + // Create parent directory if it doesn't exist. + dir := filepath.Dir(logPath) + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("error creating dir %q: %v", dir, err) + } + + // Open file with the specified flags. + f, err := os.OpenFile(logPath, flags, 0o644) + if err != nil { + return nil, fmt.Errorf("error opening file %q: %v", logPath, err) + } + return f, nil +} diff --git a/pkg/log/log.go b/pkg/log/log.go index 7201a19..8bf64c8 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -327,7 +327,7 @@ func Stacks(all bool) []byte { } // stackRegexp matches one level within a stack trace. -var stackRegexp = regexp.MustCompile("(?m)^\\S+\\(.*\\)$\\r?\\n^\\t\\S+:\\d+.*$\\r?\\n") +var stackRegexp = regexp.MustCompile(`(?m)^\S+\(.*\)$\r?\n^\t\S+:\d+.*$\r?\n`) // LocalStack returns the local goroutine stack, excluding the top N entries. // LocalStack's own entry is excluded by default and does not need to be counted in excludeTopN. diff --git a/pkg/memutil/memutil_unsafe.go b/pkg/memutil/memutil_unsafe.go index 3c5ebd7..24d811c 100644 --- a/pkg/memutil/memutil_unsafe.go +++ b/pkg/memutil/memutil_unsafe.go @@ -16,7 +16,6 @@ package memutil import ( - "reflect" "unsafe" "golang.org/x/sys/unix" @@ -28,17 +27,13 @@ func MapSlice(addr, size, prot, flags, fd, offset uintptr) ([]byte, error) { if err != nil { return nil, err } - var slice []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - hdr.Data = addr - hdr.Len = int(size) - hdr.Cap = int(size) - return slice, nil + + return unsafe.Slice((*byte)(unsafe.Pointer(addr)), int(size)), nil } // UnmapSlice unmaps a mapping returned by MapSlice. func UnmapSlice(slice []byte) error { - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - _, _, err := unix.RawSyscall6(unix.SYS_MUNMAP, uintptr(unsafe.Pointer(hdr.Data)), uintptr(hdr.Cap), 0, 0, 0, 0) + ptr := unsafe.SliceData(slice) + _, _, err := unix.RawSyscall6(unix.SYS_MUNMAP, uintptr(unsafe.Pointer(ptr)), uintptr(cap(slice)), 0, 0, 0, 0) return err } diff --git a/pkg/rawfile/rawfile_unsafe.go b/pkg/rawfile/rawfile_unsafe.go index 4ec671a..1bdf38c 100644 --- a/pkg/rawfile/rawfile_unsafe.go +++ b/pkg/rawfile/rawfile_unsafe.go @@ -19,7 +19,6 @@ package rawfile import ( - "reflect" "unsafe" "golang.org/x/sys/unix" @@ -43,12 +42,9 @@ func IovecFromBytes(bs []byte) unix.Iovec { return iov } -func bytesFromIovec(iov unix.Iovec) (bs []byte) { - sh := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) - sh.Data = uintptr(unsafe.Pointer(iov.Base)) - sh.Len = int(iov.Len) - sh.Cap = int(iov.Len) - return +func bytesFromIovec(iov unix.Iovec) []byte { + ptr := unsafe.Pointer(iov.Base) + return unsafe.Slice((*byte)(ptr), int(iov.Len)) } // AppendIovecFromBytes returns append(iovs, IovecFromBytes(bs)). If len(bs) == diff --git a/pkg/rdma/collect.go b/pkg/rdma/collect.go new file mode 100644 index 0000000..997324e --- /dev/null +++ b/pkg/rdma/collect.go @@ -0,0 +1,442 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rdma + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" +) + +// UverbsSpec identifies one /dev/infiniband/uverbs* device from the +// container's OCI spec. +type UverbsSpec struct { + // Name is the device file name, e.g. "uverbs0". + Name string + // Major and Minor are the char device numbers from the spec. + Major int64 + Minor int64 +} + +// Static PCI attribute files captured at every level of the hierarchy. +// uevent is captured verbatim (NCCL parses PCI_SLOT_NAME out of it; +// libibverbs matches drivers on modalias). +var pciAttrNames = []string{ + "class", "vendor", "device", "subsystem_vendor", "subsystem_device", + "revision", "numa_node", "local_cpus", "local_cpulist", + "max_link_speed", "max_link_width", "current_link_speed", + "current_link_width", "modalias", "uevent", +} + +// Static identity attributes of /sys/class/infiniband//. +var ibAttrNames = []string{ + "node_type", "node_guid", "sys_image_guid", "fw_ver", "board_id", + "hca_type", "hw_rev", "node_desc", +} + +// Per-port attributes served live (they change at runtime: the RoCE GID +// table repopulates when netdevs move namespaces and acquire addresses; +// link state and rate can change on retrain). +var portLiveAttrNames = []string{ + "state", "phys_state", "rate", "lid", "sm_lid", "sm_sl", +} + +// Per-port attributes that are fixed for the sandbox lifetime. +var portStaticAttrNames = []string{"link_layer", "cap_mask"} + +// Curated netdev attribute set (see NetDev for why these are static). +var netAttrNames = []string{ + "address", "addr_len", "type", "dev_id", "dev_port", "mtu", "speed", + "duplex", "operstate", "carrier", "ifindex", +} + +// NUMA aggregate range files. +var numaAggregateNames = []string{ + "online", "possible", "has_cpu", "has_memory", "has_normal_memory", +} + +// GPU/accelerator PCI classes included as leaves (beyond the NIC leaves +// derived from the spec): 3D controller, VGA controller, NVSwitch bridge. +var gpuClassPrefixes = []string{"0x0302", "0x0300", "0x0680"} + +// Collect builds a snapshot for the given spec devices by reading host +// sysfs rooted at sysRoot (normally "/sys"; overridable for tests). +// +// The snapshot is collected by runsc during sandbox creation (while host +// sysfs is still reachable, before pivot_root) and consumed by the sentry's +// sysfs implementation to build a virtual /sys tree that is path-identical +// to the host layout: one canonical /sys/devices/pci... subtree per device, +// with /sys/class/* and /sys/bus/pci/devices as pure symlink farms. Path +// identity matters because consumers (libibverbs, NCCL) resolve symlinks +// with realpath() and walk the resolved paths. +// +// Only devices derived from the container's OCI spec are captured: the +// closure starts at the spec's /dev/infiniband/uverbs* entries and expands +// to their ibdevs, associated netdevs, PCI ancestor chains, GPU PCI +// functions, and the NUMA node topology. Nothing outside that closure is +// exposed to the sandbox. +// +// It must run while the RDMA netdevs are still in the host netns. +func Collect(sysRoot string, uverbs []UverbsSpec) (*Snapshot, error) { + if len(uverbs) == 0 { + return nil, nil + } + abi, err := mustReadAttr(path.Join(sysRoot, "class/infiniband_verbs/abi_version")) + if err != nil { + return nil, fmt.Errorf("reading verbs abi_version: %w", err) + } + s := &Snapshot{VerbsABIVersion: abi} + + // pciPaths accumulates every PCI directory (relative to sysRoot) in + // the closure; expanded with ancestors below. + pciPaths := make(map[string]bool) + + for _, u := range uverbs { + if !SafeName(u.Name) { + return nil, fmt.Errorf("unsafe uverbs device name %q", u.Name) + } + uvDir := path.Join(sysRoot, "class/infiniband_verbs", u.Name) + // dev is reproduced verbatim as the uverbs "dev" file, but the + // major:minor check needs the value without its trailing newline. + dev, err := mustReadAttr(path.Join(uvDir, "dev")) + if err != nil { + return nil, fmt.Errorf("uverbs device %q: reading dev: %w", u.Name, err) + } + if want := fmt.Sprintf("%d:%d", u.Major, u.Minor); strings.TrimSpace(dev) != want { + return nil, fmt.Errorf("uverbs device %q: sysfs dev is %q but the "+ + "OCI spec says %q; the host may have renumbered devices since "+ + "the spec was built", u.Name, strings.TrimSpace(dev), want) + } + // ibdev is a directory name (used in paths and symlink targets), so + // keep the trimmed value; its file reproduction re-adds the newline. + ibdevRaw, err := mustReadAttr(path.Join(uvDir, "ibdev")) + if err != nil { + return nil, fmt.Errorf("uverbs device %q: reading ibdev: %w", u.Name, err) + } + ibdev := strings.TrimSpace(ibdevRaw) + if !SafeName(ibdev) { + return nil, fmt.Errorf("uverbs device %q: unusable ibdev name %q", u.Name, ibdev) + } + ibDir := path.Join(sysRoot, "class/infiniband", ibdev) + + leaf, err := relRealpath(sysRoot, path.Join(ibDir, "device")) + if err != nil { + return nil, fmt.Errorf("resolving PCI device of %s: %w", ibdev, err) + } + if err := addWithAncestors(pciPaths, leaf); err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + + abiVersion, err := mustReadAttr(path.Join(uvDir, "abi_version")) + if err != nil { + return nil, fmt.Errorf("uverbs device %q: reading abi_version: %w", u.Name, err) + } + ibAttrs, err := readAttrs(ibDir, ibAttrNames) + if err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + d := Device{ + Uverbs: u.Name, + IBDev: ibdev, + LeafPCI: leaf, + Dev: dev, + ABIVersion: abiVersion, + IBAttrs: ibAttrs, + Ports: map[string]Port{}, + } + if err := d.collectPorts(ibDir); err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + if err := d.collectNetDevs(sysRoot, path.Join(ibDir, "device/net")); err != nil { + return nil, fmt.Errorf("ibdev %s: %w", ibdev, err) + } + s.Devices = append(s.Devices, d) + } + + // GPU/accelerator leaves by PCI class scan; fold each into the PCI + // closure so its ancestors are materialized alongside the NIC leaves. + gpus, err := gpuLeaves(sysRoot) + if err != nil { + return nil, err + } + for _, g := range gpus { + if err := addWithAncestors(pciPaths, g); err != nil { + return nil, err + } + } + + // Materialize every PCI node with its static attributes. + for p := range pciPaths { + attrs, err := readAttrs(path.Join(sysRoot, p), pciAttrNames) + if err != nil { + return nil, fmt.Errorf("PCI node %q: %w", p, err) + } + s.PCINodes = append(s.PCINodes, PCINode{Path: p, Attrs: attrs}) + } + sort.Slice(s.PCINodes, func(i, j int) bool { return s.PCINodes[i].Path < s.PCINodes[j].Path }) + + numa, err := collectNUMA(sysRoot) + if err != nil { + return nil, err + } + s.NUMA = numa + return s, nil +} + +// addWithAncestors records leaf and every ancestor directory whose name is +// a PCI function or root complex. The walk is textual: no symlinks are +// followed (leaf is already fully resolved). +func addWithAncestors(set map[string]bool, leaf string) error { + if !strings.HasPrefix(leaf, "devices/pci") { + return fmt.Errorf("PCI path %q is not under devices/pci*", leaf) + } + p := leaf + for { + base := path.Base(p) + if bdfRE.MatchString(base) { + set[p] = true + p = path.Dir(p) + continue + } + if pciRootRE.MatchString(base) { + set[p] = true + return nil + } + return fmt.Errorf("unexpected component %q in PCI path %q", base, leaf) + } +} + +func (d *Device) collectPorts(ibDir string) error { + portsDir := path.Join(ibDir, "ports") + ents, err := os.ReadDir(portsDir) + if err != nil { + return fmt.Errorf("reading %q: %w", portsDir, err) + } + for _, e := range ents { + num := e.Name() + if !SafeName(num) { + continue + } + pDir := path.Join(portsDir, num) + staticAttrs, err := readAttrs(pDir, portStaticAttrNames) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + // gids is a mandatory part of every IB/RoCE port; its absence + // means a broken port tree, so any failure is fatal. counters and + // hw_counters vary by device, so a missing directory is tolerated. + gids, err := listSafeNames(path.Join(pDir, "gids")) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + counters, err := listOptionalNames(path.Join(pDir, "counters")) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + hwCounters, err := listOptionalNames(path.Join(pDir, "hw_counters")) + if err != nil { + return fmt.Errorf("port %s: %w", num, err) + } + d.Ports[num] = Port{ + StaticAttrs: staticAttrs, + LiveAttrs: append([]string{}, portLiveAttrNames...), + GIDNames: gids, + CounterNames: counters, + HWCounterNames: hwCounters, + } + } + if len(d.Ports) == 0 { + return fmt.Errorf("no ports under %q", portsDir) + } + return nil +} + +func (d *Device) collectNetDevs(sysRoot, netDir string) error { + ents, err := os.ReadDir(netDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // InfiniBand-link-layer devices may have no netdev; fine. + return nil + } + return fmt.Errorf("reading %q: %w", netDir, err) + } + for _, e := range ents { + name := e.Name() + if !SafeName(name) { + continue + } + attrs, err := readAttrs(path.Join(sysRoot, "class/net", name), netAttrNames) + if err != nil { + return err + } + d.NetDevs = append(d.NetDevs, NetDev{Name: name, Attrs: attrs}) + } + sort.Slice(d.NetDevs, func(i, j int) bool { return d.NetDevs[i].Name < d.NetDevs[j].Name }) + return nil +} + +// gpuLeaves returns the leaf paths (relative to sysRoot) of every host GPU, +// found by scanning /sys/bus/pci/devices for the GPU PCI classes. NCCL needs +// the GPU PCI positions to compute GPU<->NIC distance. We include all host +// GPUs: the exposure is read-only PCI metadata (NCCL discovers GPUs via CUDA, +// not sysfs, so extra nodes are inert). +func gpuLeaves(sysRoot string) ([]string, error) { + busDir := path.Join(sysRoot, "bus/pci/devices") + ents, err := os.ReadDir(busDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // No PCI bus exposed (e.g. tests); GPU-less snapshot. + return nil, nil + } + return nil, fmt.Errorf("reading %q: %w", busDir, err) + } + var leaves []string + for _, e := range ents { + // This scans unrelated host PCI devices, so a device that vanishes + // mid-scan (absent class) is skipped rather than failing collection; + // other read errors are still surfaced. + class, present, err := readAttr(path.Join(busDir, e.Name(), "class")) + if err != nil { + return nil, fmt.Errorf("reading class of PCI device %q: %w", e.Name(), err) + } + if !present { + continue + } + match := false + for _, p := range gpuClassPrefixes { + if strings.HasPrefix(strings.TrimSpace(class), p) { + match = true + break + } + } + if !match { + continue + } + leaf, err := relRealpath(sysRoot, path.Join(busDir, e.Name())) + if err != nil { + return nil, fmt.Errorf("resolving PCI path of GPU %q: %w", e.Name(), err) + } + leaves = append(leaves, leaf) + } + return leaves, nil +} + +func collectNUMA(sysRoot string) (*NUMA, error) { + nodeDir := path.Join(sysRoot, "devices/system/node") + if _, err := os.Stat(nodeDir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Non-NUMA system (or tests): no node topology to expose. + return nil, nil + } + return nil, fmt.Errorf("statting %q: %w", nodeDir, err) + } + agg, err := readAttrs(nodeDir, numaAggregateNames) + if err != nil { + return nil, err + } + return &NUMA{Aggregate: agg}, nil +} + +// relRealpath resolves p and returns it relative to sysRoot. +func relRealpath(sysRoot, p string) (string, error) { + r, err := filepath.EvalSymlinks(p) + if err != nil { + return "", err + } + rel, err := filepath.Rel(sysRoot, r) + if err != nil || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("%q resolves outside %q", p, sysRoot) + } + return rel, nil +} + +// readAttr reads the attribute file at p and returns its contents. present is +// false only when the file is absent (ENOENT). Any other error (permissions, +// I/O) is returned rather than swallowed. +func readAttr(p string) (content string, present bool, err error) { + b, err := os.ReadFile(p) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, nil + } + return "", false, err + } + return string(b), true, nil +} + +// mustReadAttr reads an attribute file that is expected to have. Unlike +// readAttr, it treats absence (ENOENT) as an error. +func mustReadAttr(p string) (string, error) { + content, present, err := readAttr(p) + if err != nil { + return "", err + } + if !present { + return "", fmt.Errorf("required sysfs file %q is missing", p) + } + return content, nil +} + +// readAttrs reads the named attribute files under dir. Absent files (ENOENT) +// are omitted so the sandbox mirrors the host's exact set of files. Any +// non-ENOENT read error aborts collection. +func readAttrs(dir string, names []string) (map[string]string, error) { + m := make(map[string]string) + for _, name := range names { + p := path.Join(dir, name) + content, present, err := readAttr(p) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", p, err) + } + if present { + m[name] = content + } + } + return m, nil +} + +// listSafeNames returns the safe entry names under dir, sorted. The ReadDir +// error is returned (never swallowed); callers decide whether an absent +// directory is acceptable (see listOptionalNames). +func listSafeNames(dir string) ([]string, error) { + ents, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", dir, err) + } + var names []string + for _, e := range ents { + if SafeName(e.Name()) { + names = append(names, e.Name()) + } + } + sort.Strings(names) + return names, nil +} + +// listOptionalNames is listSafeNames for a directory that may legitimately be +// absent: a missing directory yields (nil, nil), but a read error is still +// surfaced. +func listOptionalNames(dir string) ([]string, error) { + names, err := listSafeNames(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return names, err +} diff --git a/pkg/abi/linux/linux_unsafe_state_autogen.go b/pkg/rdma/rdma_state_autogen.go similarity index 74% rename from pkg/abi/linux/linux_unsafe_state_autogen.go rename to pkg/rdma/rdma_state_autogen.go index bddf5a7..2699604 100644 --- a/pkg/abi/linux/linux_unsafe_state_autogen.go +++ b/pkg/rdma/rdma_state_autogen.go @@ -1,3 +1,3 @@ // automatically generated by stateify. -package linux +package rdma diff --git a/pkg/rdma/snapshot.go b/pkg/rdma/snapshot.go new file mode 100644 index 0000000..b8aed5b --- /dev/null +++ b/pkg/rdma/snapshot.go @@ -0,0 +1,154 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rdma hosts RDMA configuration and utilities shared between the +// sentry and runsc. +package rdma + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" +) + +// Path is the location inside the sandbox chroot where the serialized +// snapshot is stored. Needed to communicate this information across the boot +// process re-exec boundary. +const Path = "/var/lib/gvisor/rdma_sysfs.json" + +// PCINode is one directory in the /sys/devices/pci... hierarchy (a root +// complex, bridge, or leaf function) with its static attribute files. +type PCINode struct { + // Path is relative to /sys, e.g. + // "devices/pci0000:07/0000:07:01.0/0000:0c:00.0". + Path string `json:"path"` + // Attrs maps attribute file name to contents (verbatim, including any + // trailing newline). + Attrs map[string]string `json:"attrs"` +} + +// Port is the per-IB-port state. Attributes split into static (immutable +// for the sandbox lifetime, snapshotted) and live (served by reading the +// host file at access time through a bind mount; the RoCE GID table +// changes when netdevs move between namespaces and acquire addresses). +type Port struct { + StaticAttrs map[string]string `json:"static_attrs"` + LiveAttrs []string `json:"live_attrs"` + // GIDNames are the entry names of the GID table (typically "0".."255"). + // The same names index gids/, gid_attrs/types/ and + // gid_attrs/ndevs/, all served live. + GIDNames []string `json:"gid_names"` + CounterNames []string `json:"counter_names"` + HWCounterNames []string `json:"hw_counter_names"` +} + +// NetDev is a network device associated with an RDMA device, with its +// curated static attribute set. +type NetDev struct { + Name string `json:"name"` + Attrs map[string]string `json:"attrs"` +} + +// Device is one uverbs device and everything hanging off it. +type Device struct { + // Uverbs is the device name, e.g. "uverbs0". + Uverbs string `json:"uverbs"` + // IBDev is the InfiniBand device name, e.g. "mlx5_0". + IBDev string `json:"ibdev"` + // LeafPCI is the PCI function directory, relative to /sys. + LeafPCI string `json:"leaf_pci"` + // Dev is the host "major:minor" of the uverbs char device. + Dev string `json:"dev"` + ABIVersion string `json:"abi_version"` + // IBAttrs are the static identity attributes of + // /sys/class/infiniband// (node_guid, fw_ver, ...). + IBAttrs map[string]string `json:"ib_attrs"` + // Ports maps port number ("1") to its state. + Ports map[string]Port `json:"ports"` + // NetDevs are the netdevs bound to this ibdev via device/net/. + NetDevs []NetDev `json:"netdevs"` +} + +// NUMA is the /sys/devices/system/node subtree. +type NUMA struct { + // Aggregate holds the top-level range files (online, possible, ...). + Aggregate map[string]string `json:"aggregate"` +} + +// Snapshot is the host sysfs snapshot for RDMA device topology. +type Snapshot struct { + VerbsABIVersion string `json:"verbs_abi_version"` + // PCINodes contains every PCI directory in the closure: the leaves + // (NICs, GPUs) and all their ancestor bridges/roots. Sorted by Path, + // which places parents before children. + PCINodes []PCINode `json:"pci_nodes"` + Devices []Device `json:"devices"` + NUMA *NUMA `json:"numa,omitempty"` +} + +// safeName matches names we are willing to reproduce inside the sandbox or +// join into host paths. Kernel-generated sysfs names satisfy this; anything +// else is dropped at collection time so the construction side never handles +// a name containing a path separator or dot-dot. +var safeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:+-]*$`) + +// SafeName reports whether name may be used as a sysfs entry name. +func SafeName(name string) bool { + return safeName.MatchString(name) +} + +// bdfRE matches a PCI function directory name ("0000:0c:00.0"). The domain +// is at least 4 hex digits: the kernel prints it with %04x, and synthetic +// domains above 0xffff (e.g. Intel VMD) print wider. +var bdfRE = regexp.MustCompile(`^[0-9a-f]{4,}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +// pciRootRE matches a PCI root complex directory name ("pci0000:07"). +var pciRootRE = regexp.MustCompile(`^pci[0-9a-f]{4,}:[0-9a-f]{2}$`) + +// IsBDF reports whether name is a PCI function directory name. +func IsBDF(name string) bool { return bdfRE.MatchString(name) } + +// Save serializes the snapshot to dst, creating parent directories. +func (s *Snapshot) Save(dst string) error { + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating %q: %w", dir, err) + } + b, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("marshaling RDMA sysfs snapshot: %w", err) + } + return os.WriteFile(dst, b, 0o644) +} + +// Load deserializes a snapshot from src. Returns (nil, nil) if the file +// does not exist. +func Load(src string) (*Snapshot, error) { + b, err := os.ReadFile(src) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var s Snapshot + if err := json.Unmarshal(b, &s); err != nil { + return nil, fmt.Errorf("unmarshaling RDMA sysfs snapshot %q: %w", src, err) + } + return &s, nil +} diff --git a/pkg/refs/refs_map.go b/pkg/refs/refs_map.go index 2c15fd5..fb89d98 100644 --- a/pkg/refs/refs_map.go +++ b/pkg/refs/refs_map.go @@ -162,7 +162,7 @@ func doLeakCheck() { skip = o.LeakCheckDisabled() } if skip { - log.Debugf(obj.LeakMessage()) + log.Debugf("%s", obj.LeakMessage()) continue } msg += obj.LeakMessage() + "\n" @@ -174,6 +174,6 @@ func doLeakCheck() { if leakCheckPanicEnabled() { panic(msg) } - log.Warningf(msg) + log.Warningf("%s", msg) } } diff --git a/pkg/sleep/sleep_unsafe_state_autogen.go b/pkg/sleep/sleep_unsafe_state_autogen.go index 5ca2094..1c243aa 100644 --- a/pkg/sleep/sleep_unsafe_state_autogen.go +++ b/pkg/sleep/sleep_unsafe_state_autogen.go @@ -25,8 +25,8 @@ func (s *Sleeper) beforeSave() {} // +checklocksignore func (s *Sleeper) StateSave(stateSinkObject state.Sink) { s.beforeSave() - var sharedListValue *Waker - sharedListValue = s.saveSharedList() + sharedListValue := s.saveSharedList() + _ = (*Waker)(sharedListValue) stateSinkObject.SaveValue(0, sharedListValue) stateSinkObject.Save(1, &s.localList) stateSinkObject.Save(2, &s.allWakers) @@ -58,8 +58,8 @@ func (w *Waker) beforeSave() {} // +checklocksignore func (w *Waker) StateSave(stateSinkObject state.Sink) { w.beforeSave() - var sValue wakerState - sValue = w.saveS() + sValue := w.saveS() + _ = (wakerState)(sValue) stateSinkObject.SaveValue(0, sValue) stateSinkObject.Save(1, &w.next) stateSinkObject.Save(2, &w.allWakersNext) diff --git a/pkg/state/addr_set.go b/pkg/state/addr_set.go index 1069baa..1aad463 100644 --- a/pkg/state/addr_set.go +++ b/pkg/state/addr_set.go @@ -384,7 +384,7 @@ func (s *addrSet) InsertWithoutMergingUnchecked(gap addrGapIterator, r addrRange if splitMaxGap { gap.node.updateMaxGapLeaf() } - return addrIterator{gap.node, gap.index} + return addrIterator(gap) } // InsertRange inserts the given segment into the set. If the new segment can @@ -512,7 +512,7 @@ func (s *addrSet) Remove(seg addrIterator) addrGapIterator { if addrtrackGaps != 0 { seg.node.updateMaxGapLeaf() } - return seg.node.rebalanceAfterRemove(addrGapIterator{seg.node, seg.index}) + return seg.node.rebalanceAfterRemove(addrGapIterator(seg)) } // RemoveAll removes all segments from the set. All existing iterators are @@ -597,6 +597,19 @@ func (s *addrSet) RemoveFullRangeWith(r addrRange, f func(seg addrIterator)) add } } +// MoveFrom moves all segments from s2 to s, replacing all existing segments in +// s and leaving s2 empty. +func (s *addrSet) MoveFrom(s2 *addrSet) { + *s = *s2 + for _, child := range s.root.children { + if child == nil { + break + } + child.parent = &s.root + } + s2.RemoveAll() +} + // Merge attempts to merge two neighboring segments. If successful, Merge // returns an iterator to the merged segment, and all existing iterators are // invalidated. Otherwise, Merge returns a terminal iterator. @@ -1600,7 +1613,7 @@ func (seg addrIterator) PrevGap() addrGapIterator { if seg.node.hasChildren { return seg.node.children[seg.index].lastSegment().NextGap() } - return addrGapIterator{seg.node, seg.index} + return addrGapIterator(seg) } // NextGap returns the gap immediately after the iterated segment. @@ -1889,26 +1902,26 @@ func (n *addrnode) String() string { func (n *addrnode) writeDebugString(buf *bytes.Buffer, prefix string) { if n.hasChildren != (n.nrSegments > 0 && n.children[0] != nil) { buf.WriteString(prefix) - buf.WriteString(fmt.Sprintf("WARNING: inconsistent value of hasChildren: got %v, want %v\n", n.hasChildren, !n.hasChildren)) + fmt.Fprintf(buf, "WARNING: inconsistent value of hasChildren: got %v, want %v\n", n.hasChildren, !n.hasChildren) } for i := 0; i < n.nrSegments; i++ { if child := n.children[i]; child != nil { cprefix := fmt.Sprintf("%s- % 3d ", prefix, i) if child.parent != n || child.parentIndex != i { buf.WriteString(cprefix) - buf.WriteString(fmt.Sprintf("WARNING: inconsistent linkage to parent: got (%p, %d), want (%p, %d)\n", child.parent, child.parentIndex, n, i)) + fmt.Fprintf(buf, "WARNING: inconsistent linkage to parent: got (%p, %d), want (%p, %d)\n", child.parent, child.parentIndex, n, i) } child.writeDebugString(buf, fmt.Sprintf("%s- % 3d ", prefix, i)) } buf.WriteString(prefix) if n.hasChildren { if addrtrackGaps != 0 { - buf.WriteString(fmt.Sprintf("- % 3d: %v => %v, maxGap: %d\n", i, n.keys[i], n.values[i], n.maxGap.Get())) + fmt.Fprintf(buf, "- % 3d: %v => %v, maxGap: %d\n", i, n.keys[i], n.values[i], n.maxGap.Get()) } else { - buf.WriteString(fmt.Sprintf("- % 3d: %v => %v\n", i, n.keys[i], n.values[i])) + fmt.Fprintf(buf, "- % 3d: %v => %v\n", i, n.keys[i], n.values[i]) } } else { - buf.WriteString(fmt.Sprintf("- % 3d: %v => %v\n", i, n.keys[i], n.values[i])) + fmt.Fprintf(buf, "- % 3d: %v => %v\n", i, n.keys[i], n.values[i]) } } if child := n.children[n.nrSegments]; child != nil { diff --git a/pkg/state/decode.go b/pkg/state/decode.go index 9689eb9..b4363f0 100644 --- a/pkg/state/decode.go +++ b/pkg/state/decode.go @@ -694,6 +694,7 @@ func (ds *decodeState) Load(obj reflect.Value) { // iterations required to finish all objects. if err := safely(func() { for elem := ds.leaves.Front(); elem != nil; elem = elem.Next() { + ods = elem.ods ds.checkComplete(elem.ods) } }); err != nil { @@ -714,7 +715,7 @@ func (ds *decodeState) Load(obj reflect.Value) { fmt.Fprintf(&buf, "%q", cycleOS.obj.Type()) } buf.WriteString("}") - Failf("incomplete graph: %s", string(buf.Bytes())) + Failf("incomplete graph: %s", buf.String()) } } diff --git a/pkg/state/encode.go b/pkg/state/encode.go index e8786ba..f1e7a48 100644 --- a/pkg/state/encode.go +++ b/pkg/state/encode.go @@ -770,8 +770,11 @@ func (es *encodeState) Save(obj reflect.Value) { es.encodeObject(oes.obj, oes.how, &oes.encoded) } }); err != nil { - // Include the object in the error message. - Failf("encoding error: %w\nfor object %#v", err, oes.obj.Interface()) + // Include the object in the error message, if available. + if oes != nil && oes.obj.IsValid() { + Failf("encoding error: %w\nfor object %#v", err, oes.obj.Interface()) + } + Failf("encoding error: %w", err) } // Check that we have objects to serialize. @@ -802,14 +805,19 @@ func (es *encodeState) Save(obj reflect.Value) { }) for _, id := range ids { // Encode the id. + oes = nil wire.Save(&es.w, wire.Uint(id)) // Marshal the object. - oes := es.pending[id] + oes = es.pending[id] wire.Save(&es.w, oes.encoded) } }); err != nil { - // Include the object and the error. - Failf("error serializing object %#v: %w", oes.encoded, err) + if oes != nil { + // Include the object and the error. + Failf("error serializing object %#v: %w", oes.encoded, err) + } else { + Failf("error serializing type or ID: %w", err) + } } } diff --git a/pkg/state/pretty/pretty.go b/pkg/state/pretty/pretty.go index 6d02e79..b486881 100644 --- a/pkg/state/pretty/pretty.go +++ b/pkg/state/pretty/pretty.go @@ -48,7 +48,7 @@ func (p *printer) formatRef(x *wire.Ref, graph uint64) string { buf.WriteString(".") buf.WriteString(string(*v)) case wire.Index: - buf.WriteString(fmt.Sprintf("[%d]", v)) + fmt.Fprintf(&buf, "[%d]", v) default: panic(fmt.Sprintf("unreachable: switch should be exhaustive, unhandled case %v", reflect.TypeOf(component))) } @@ -147,7 +147,7 @@ func (p *printer) format(graph uint64, depth int, encoded wire.Object) (string, return strings.Join(items, tabs), len(zeros) < len(x.Contents) case *wire.Struct: tag := fmt.Sprintf("g%dt%d", graph, x.TypeID) - spec, _ := p.typeSpecs[tag] + spec := p.typeSpecs[tag] typ, _ := p.formatType(x.TypeID, graph) if x.Fields() == 0 { return fmt.Sprintf("struct[%s]{}", typ), false diff --git a/pkg/state/state.go b/pkg/state/state.go index 3a8f1f6..3267092 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -188,7 +188,7 @@ func (s Sink) Context() context.Context { // Type is an interface that must be implemented by Struct objects. This allows // these objects to be serialized while minimizing runtime reflection required. // -// All these methods can be automatically generated by the go_statify tool. +// All these methods can be automatically generated by the go_stateify tool. type Type interface { // StateTypeName returns the type's name. // diff --git a/pkg/state/statefile/statefile.go b/pkg/state/statefile/statefile.go index eff4966..0e82794 100644 --- a/pkg/state/statefile/statefile.go +++ b/pkg/state/statefile/statefile.go @@ -87,8 +87,8 @@ var ErrMetadataInvalid = fmt.Errorf("metadata invalid, can't start with _") var ErrInvalidFlags = fmt.Errorf("flags set is invalid") const ( - // CompressionKey is the key for the compression level in the metadata. - CompressionKey = "compression" + // compressionKey is the key for the compression level in the metadata. + compressionKey = "compression" ) // CompressionLevel is the image compression level. @@ -100,7 +100,7 @@ const ( // CompressionLevelNone represents the absence of any compression on an image. CompressionLevelNone = CompressionLevel("none") // CompressionLevelDefault represents the default compression level. - CompressionLevelDefault = CompressionLevelFlateBestSpeed + CompressionLevelDefault = CompressionLevelNone ) func (c CompressionLevel) String() string { @@ -109,7 +109,7 @@ func (c CompressionLevel) String() string { // ToMetadata returns the compression level as a metadata map. func (c CompressionLevel) ToMetadata() map[string]string { - return map[string]string{CompressionKey: string(c)} + return map[string]string{compressionKey: string(c)} } // CompressionLevelFromString parses a string into the CompressionLevel. @@ -127,21 +127,14 @@ func CompressionLevelFromString(val string) (CompressionLevel, error) { } // CompressionLevelFromMetadata returns image compression type stored in the metadata. -// If the metadata doesn't contain compression information the default behavior -// is the "flate-best-speed" state because the default behavior used to be to always -// compress. If the parameter is missing it will be set to default. +// If the metadata doesn't contain compression information, the default behavior +// is "none" (no compression) and it is added to the metadata. func CompressionLevelFromMetadata(metadata map[string]string) (CompressionLevel, error) { - compression := CompressionLevelDefault - - if val, ok := metadata[CompressionKey]; ok { - var err error - if compression, err = CompressionLevelFromString(val); err != nil { - return CompressionLevelNone, err - } - } else { - metadata[CompressionKey] = string(compression) + if val, ok := metadata[compressionKey]; ok { + return CompressionLevelFromString(val) } - + compression := CompressionLevelDefault + metadata[compressionKey] = string(compression) return compression, nil } @@ -166,8 +159,12 @@ func NewWriter(w io.Writer, key []byte, metadata map[string]string) (io.WriteClo } // Create our HMAC function. - h := hmac.New(sha256.New, key) - mw := io.MultiWriter(w, h) + mw := w + var h hash.Hash + if len(key) > 0 { + h = hmac.New(sha256.New, key) + mw = io.MultiWriter(w, h) + } // First, write the header. if _, err := mw.Write(magicHeader); err != nil { @@ -204,12 +201,14 @@ func NewWriter(w io.Writer, key []byte, metadata map[string]string) (io.WriteClo return nil, err } // Write the current hash. - cur := h.Sum(nil) - for done := 0; done < len(cur); { - n, err := mw.Write(cur[done:]) - done += n - if err != nil { - return nil, err + if h != nil { + cur := h.Sum(nil) + for done := 0; done < len(cur); { + n, err := mw.Write(cur[done:]) + done += n + if err != nil { + return nil, err + } } } @@ -240,8 +239,10 @@ func readMetadataLen(r io.Reader) (uint64, error) { // metadata validates the magic header and reads out the metadata from a state // data stream. -func metadata(r io.Reader, h hash.Hash) (map[string]string, error) { - if h != nil { +func metadata(r io.Reader, key []byte) (map[string]string, error) { + var h hash.Hash + if len(key) > 0 { + h = hmac.New(sha256.New, key) r = io.TeeReader(r, h) } @@ -285,6 +286,9 @@ func metadata(r io.Reader, h hash.Hash) (map[string]string, error) { cur := h.Sum(nil) buf := make([]byte, len(cur)) if _, err := io.ReadFull(r, buf); err != nil { + if err == io.EOF { + return nil, io.ErrUnexpectedEOF + } return nil, err } if !hmac.Equal(cur, buf) { @@ -304,8 +308,7 @@ func metadata(r io.Reader, h hash.Hash) (map[string]string, error) { // NewReader returns a reader for a statefile. func NewReader(r io.ReadCloser, key []byte) (io.ReadCloser, map[string]string, error) { // Read the metadata with the hash. - h := hmac.New(sha256.New, key) - metadata, err := metadata(r, h) + metadata, err := metadata(r, key) if err != nil { return nil, nil, err } @@ -321,11 +324,12 @@ func NewReader(r io.ReadCloser, key []byte) (io.ReadCloser, map[string]string, e // Pick correct reader var cr io.ReadCloser - if compression == CompressionLevelFlateBestSpeed { + switch compression { + case CompressionLevelFlateBestSpeed: cr, err = compressio.NewReader(r, key) - } else if compression == CompressionLevelNone { + case CompressionLevelNone: cr = compressio.NewSimpleReader(r, key) - } else { + default: // Should never occur, as it has the default path. return nil, nil, fmt.Errorf("metadata contains invalid compression flag value: %v", compression) } diff --git a/pkg/state/stats.go b/pkg/state/stats.go index eaec664..17c30b6 100644 --- a/pkg/state/stats.go +++ b/pkg/state/stats.go @@ -124,7 +124,7 @@ func (s *Stats) String() string { total time.Duration ) buf.WriteString("\n") - buf.WriteString(fmt.Sprintf("% 16s | % 8s | % 16s | %s\n", "total", "count", "per", "type")) + fmt.Fprintf(&buf, "% 16s | % 8s | % 16s | %s\n", "total", "count", "per", "type") buf.WriteString("-----------------+----------+------------------+----------------\n") for _, se := range ss { if se.entry.count == 0 { @@ -135,11 +135,11 @@ func (s *Stats) String() string { count += se.entry.count total += se.entry.total per := se.entry.total / time.Duration(se.entry.count) - buf.WriteString(fmt.Sprintf("% 16s | %8d | % 16s | %s\n", - se.entry.total, se.entry.count, per, se.name)) + fmt.Fprintf(&buf, "% 16s | %8d | % 16s | %s\n", + se.entry.total, se.entry.count, per, se.name) } buf.WriteString("-----------------+----------+------------------+----------------\n") - buf.WriteString(fmt.Sprintf("% 16s | % 8d | % 16s | [all]", - total, count, total/time.Duration(count))) - return string(buf.Bytes()) + fmt.Fprintf(&buf, "% 16s | % 8d | % 16s | [all]", + total, count, total/time.Duration(count)) + return buf.String() } diff --git a/pkg/state/types.go b/pkg/state/types.go index e6d2e57..a85df4b 100644 --- a/pkg/state/types.go +++ b/pkg/state/types.go @@ -295,23 +295,23 @@ const interfaceType = "interface" var primitiveTypeDatabase = func() map[string]reflect.Type { r := make(map[string]reflect.Type) for _, t := range []reflect.Type{ - reflect.TypeOf(false), - reflect.TypeOf(int(0)), - reflect.TypeOf(int8(0)), - reflect.TypeOf(int16(0)), - reflect.TypeOf(int32(0)), - reflect.TypeOf(int64(0)), - reflect.TypeOf(uint(0)), - reflect.TypeOf(uintptr(0)), - reflect.TypeOf(uint8(0)), - reflect.TypeOf(uint16(0)), - reflect.TypeOf(uint32(0)), - reflect.TypeOf(uint64(0)), - reflect.TypeOf(""), - reflect.TypeOf(float32(0.0)), - reflect.TypeOf(float64(0.0)), - reflect.TypeOf(complex64(0.0)), - reflect.TypeOf(complex128(0.0)), + reflect.TypeFor[bool](), + reflect.TypeFor[int](), + reflect.TypeFor[int8](), + reflect.TypeFor[int16](), + reflect.TypeFor[int32](), + reflect.TypeFor[int64](), + reflect.TypeFor[uint](), + reflect.TypeFor[uintptr](), + reflect.TypeFor[uint8](), + reflect.TypeFor[uint16](), + reflect.TypeFor[uint32](), + reflect.TypeFor[uint64](), + reflect.TypeFor[string](), + reflect.TypeFor[float32](), + reflect.TypeFor[float64](), + reflect.TypeFor[complex64](), + reflect.TypeFor[complex128](), } { r[t.Name()] = t } diff --git a/pkg/state/wire/wire.go b/pkg/state/wire/wire.go index 07a41ac..15c5292 100644 --- a/pkg/state/wire/wire.go +++ b/pkg/state/wire/wire.go @@ -509,7 +509,7 @@ func loadMap(r *Reader) Map { func (m *Map) save(w *Writer) { l := Uint(len(m.Keys)) if int(l) != len(m.Values) { - panic(fmt.Sprintf("mismatched keys (%d) Aand values (%d)", len(m.Keys), len(m.Values))) + panic(fmt.Sprintf("mismatched keys (%d) and values (%d)", len(m.Keys), len(m.Values))) } l.save(w) if l == 0 { diff --git a/pkg/sync/gate_unsafe.go b/pkg/sync/gate_unsafe.go index 40c44da..4cca7bc 100644 --- a/pkg/sync/gate_unsafe.go +++ b/pkg/sync/gate_unsafe.go @@ -110,8 +110,8 @@ func (g *Gate) leaveClosed() { if atomic.LoadUintptr(&g.closingG) == 0 { return } - if g := atomic.SwapUintptr(&g.closingG, 0); g > preparingG { - goready(g, 0) + if cG := atomic.SwapUintptr(&g.closingG, 0); cG > preparingG { + goready(cG, 0) } } @@ -133,8 +133,8 @@ func (g *Gate) Close() { panic("concurrent Close of sync.Gate") } - if g := atomic.SwapUintptr(&g.closingG, preparingG); g != 0 { - panic(fmt.Sprintf("invalid sync.Gate.closingG during Close: %#x", g)) + if cG := atomic.SwapUintptr(&g.closingG, preparingG); cG != 0 { + panic(fmt.Sprintf("invalid sync.Gate.closingG during Close: %#x", cG)) } if atomic.LoadInt32(&g.userCount) == math.MinInt32 { // The last call to Leave arrived while we were setting up closingG. @@ -142,6 +142,7 @@ func (g *Gate) Close() { } // WaitReasonSemacquire/TraceBlockSync are consistent with WaitGroup. gopark(gateCommit, gohacks.Noescape(unsafe.Pointer(&g.closingG)), WaitReasonSemacquire, TraceBlockSync, 0) + RaceAcquire(unsafe.Pointer(&g.closingG)) } //go:norace diff --git a/pkg/goid/goid_122_amd64.s b/pkg/sync/runtime_constants_go_impl.go similarity index 67% rename from pkg/goid/goid_122_amd64.s rename to pkg/sync/runtime_constants_go_impl.go index 5039f73..4888288 100644 --- a/pkg/goid/goid_122_amd64.s +++ b/pkg/sync/runtime_constants_go_impl.go @@ -1,4 +1,4 @@ -// Copyright 2020 The gVisor Authors. +// Copyright 2023 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !go1.23 +package sync -#include "textflag.h" - -#define GOID_OFFSET 152 // +checkoffset runtime g.goid - -// func goid() int64 -TEXT ·goid(SB),NOSPLIT|NOFRAME,$0-8 - MOVQ (TLS), R14 - MOVQ GOID_OFFSET(R14), R14 - MOVQ R14, ret+0(FP) - RET +// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. +const ( + WaitReasonSelect uint8 = 18 + WaitReasonChanReceive uint8 = 19 + WaitReasonSemacquire uint8 = 13 +) diff --git a/pkg/sync/runtime_exectracer2.go b/pkg/sync/runtime_exectracer2_go_impl.go similarity index 81% rename from pkg/sync/runtime_exectracer2.go rename to pkg/sync/runtime_exectracer2_go_impl.go index 58630af..c43868a 100644 --- a/pkg/sync/runtime_exectracer2.go +++ b/pkg/sync/runtime_exectracer2_go_impl.go @@ -16,6 +16,6 @@ package sync // TraceBlockReason constants, from Go's src/runtime/trace2runtime.go. const ( - TraceBlockSelect TraceBlockReason = 3 // +checkconst runtime traceBlockSelect - TraceBlockSync TraceBlockReason = 5 // +checkconst runtime traceBlockSync + TraceBlockSelect TraceBlockReason = 3 + TraceBlockSync TraceBlockReason = 5 ) diff --git a/pkg/sync/runtime_spinning_amd64.s b/pkg/sync/runtime_spinning_amd64.s deleted file mode 100644 index b238c8d..0000000 --- a/pkg/sync/runtime_spinning_amd64.s +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// https://go.dev/cl/669235 (1.25) adds a new schedt field prior to nmspinning. -//go:build amd64 && !go1.25 - -#include "textflag.h" - -#define NMSPINNING_OFFSET 92 // +checkoffset runtime schedt.nmspinning - -TEXT ·addrOfSpinning(SB),NOSPLIT|NOFRAME,$0-8 - LEAQ runtime·sched(SB), AX - ADDQ $NMSPINNING_OFFSET, AX - MOVQ AX, ret+0(FP) - RET diff --git a/pkg/sync/runtime_spinning_go125_amd64.s b/pkg/sync/runtime_spinning_asm_impl_amd64.s similarity index 80% rename from pkg/sync/runtime_spinning_go125_amd64.s rename to pkg/sync/runtime_spinning_asm_impl_amd64.s index 46b2f06..6daa250 100644 --- a/pkg/sync/runtime_spinning_go125_amd64.s +++ b/pkg/sync/runtime_spinning_asm_impl_amd64.s @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// https://go.dev/cl/669235 (1.25) adds a new schedt field prior to nmspinning. -//go:build amd64 && go1.25 +//go:build amd64 #include "textflag.h" -#define NMSPINNING_OFFSET 100 // +checkoffset runtime schedt.nmspinning +#define NMSPINNING_OFFSET 116 TEXT ·addrOfSpinning(SB),NOSPLIT|NOFRAME,$0-8 LEAQ runtime·sched(SB), AX diff --git a/pkg/abi/linux/vfio_unsafe.go b/pkg/sync/runtime_spinning_asm_impl_arm64.s similarity index 71% rename from pkg/abi/linux/vfio_unsafe.go rename to pkg/sync/runtime_spinning_asm_impl_arm64.s index 8448991..c04da8a 100644 --- a/pkg/abi/linux/vfio_unsafe.go +++ b/pkg/sync/runtime_spinning_asm_impl_arm64.s @@ -1,4 +1,4 @@ -// Copyright 2024 The gVisor Authors. +// Copyright 2023 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,11 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package linux +//go:build !amd64 -import "unsafe" - -// Size returns the number of bytes for a VFIOIrqSet object. -func (vfioIrqSet VFIOIrqSet) Size() uint64 { - return uint64(unsafe.Sizeof(vfioIrqSet)) -} +// This file is intentionally left blank. arm64 doesn't use +// addrOfSpinning, but we still need an input to the nogo template rule. diff --git a/pkg/sync/runtime_spinning_other.s b/pkg/sync/runtime_spinning_other.s index b6391d2..bafac6a 100644 --- a/pkg/sync/runtime_spinning_other.s +++ b/pkg/sync/runtime_spinning_other.s @@ -15,4 +15,10 @@ //go:build !amd64 // This file is intentionally left blank. Other arches don't use -// addrOfSpinning, but we still need an input to the nogo template rule. +// addrOfSpinning, but because this package is partially used in Netstack, we +// should support arches that aren't amd64 or arm64. Having this file here +// ensures that `go build` doesn't compile the package with the `-complete` +// flag, because the package isn't made up of just '.go' files. +// This allows Netstack to use the architecture-independent portions of this +// package, because the architecture-dependent portions are never compiled in +// the first place. diff --git a/pkg/tcpip/adapters/gonet/gonet.go b/pkg/tcpip/adapters/gonet/gonet.go index 519f34c..70caef9 100644 --- a/pkg/tcpip/adapters/gonet/gonet.go +++ b/pkg/tcpip/adapters/gonet/gonet.go @@ -184,7 +184,7 @@ func (d *deadlineTimer) setDeadline(cancelCh *chan struct{}, timer **time.Timer, return } - timeout := t.Sub(time.Now()) + timeout := time.Until(t) if timeout <= 0 { close(*cancelCh) return diff --git a/pkg/tcpip/faketime/faketime.go b/pkg/tcpip/faketime/faketime.go index c28cffa..ebabde8 100644 --- a/pkg/tcpip/faketime/faketime.go +++ b/pkg/tcpip/faketime/faketime.go @@ -25,8 +25,6 @@ import ( ) // NullClock implements a clock that never advances. -// -// +stateify savable type NullClock struct{} var _ tcpip.Clock = (*NullClock)(nil) @@ -42,8 +40,6 @@ func (*NullClock) NowMonotonic() tcpip.MonotonicTime { } // nullTimer implements a timer that never fires. -// -// +stateify savable type nullTimer struct{} var _ tcpip.Timer = (*nullTimer)(nil) @@ -96,9 +92,8 @@ func (n *notificationChannels) wait() { } } -// +stateify savable type manualClockMutex struct { - sync.RWMutex `state:"nosave"` + sync.RWMutex // now is the current (fake) time of the clock. now time.Time @@ -334,9 +329,8 @@ func (mc *ManualClock) stopTimer(mt *manualTimer) bool { return true } -// +stateify savable type manualTimerMu struct { - sync.Mutex `state:"nosave"` + sync.Mutex // firesAt is the time when the timer will fire. // @@ -344,13 +338,10 @@ type manualTimerMu struct { firesAt time.Time } -// +stateify savable type manualTimer struct { clock *ManualClock - // TODO(b/341946753): Restore when netstack is savable. - f func() `state:"nosave"` - - mu manualTimerMu + f func() + mu manualTimerMu } var _ tcpip.Timer = (*manualTimer)(nil) diff --git a/pkg/tcpip/faketime/faketime_state_autogen.go b/pkg/tcpip/faketime/faketime_state_autogen.go index eb5e6d2..d2c5026 100644 --- a/pkg/tcpip/faketime/faketime_state_autogen.go +++ b/pkg/tcpip/faketime/faketime_state_autogen.go @@ -8,79 +8,6 @@ import ( "github.com/sagernet/gvisor/pkg/state" ) -func (n *NullClock) StateTypeName() string { - return "pkg/tcpip/faketime.NullClock" -} - -func (n *NullClock) StateFields() []string { - return []string{} -} - -func (n *NullClock) beforeSave() {} - -// +checklocksignore -func (n *NullClock) StateSave(stateSinkObject state.Sink) { - n.beforeSave() -} - -func (n *NullClock) afterLoad(context.Context) {} - -// +checklocksignore -func (n *NullClock) StateLoad(ctx context.Context, stateSourceObject state.Source) { -} - -func (n *nullTimer) StateTypeName() string { - return "pkg/tcpip/faketime.nullTimer" -} - -func (n *nullTimer) StateFields() []string { - return []string{} -} - -func (n *nullTimer) beforeSave() {} - -// +checklocksignore -func (n *nullTimer) StateSave(stateSinkObject state.Sink) { - n.beforeSave() -} - -func (n *nullTimer) afterLoad(context.Context) {} - -// +checklocksignore -func (n *nullTimer) StateLoad(ctx context.Context, stateSourceObject state.Source) { -} - -func (m *manualClockMutex) StateTypeName() string { - return "pkg/tcpip/faketime.manualClockMutex" -} - -func (m *manualClockMutex) StateFields() []string { - return []string{ - "now", - "times", - "timers", - } -} - -func (m *manualClockMutex) beforeSave() {} - -// +checklocksignore -func (m *manualClockMutex) StateSave(stateSinkObject state.Sink) { - m.beforeSave() - stateSinkObject.Save(0, &m.now) - stateSinkObject.Save(1, &m.times) - stateSinkObject.Save(2, &m.timers) -} - -func (m *manualClockMutex) afterLoad(context.Context) {} - -// +checklocksignore -func (m *manualClockMutex) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &m.now) - stateSourceObject.Load(1, &m.times) - stateSourceObject.Load(2, &m.timers) -} - func (mc *ManualClock) StateTypeName() string { return "pkg/tcpip/faketime.ManualClock" } @@ -109,64 +36,6 @@ func (mc *ManualClock) StateLoad(ctx context.Context, stateSourceObject state.So stateSourceObject.Load(1, &mc.mu) } -func (m *manualTimerMu) StateTypeName() string { - return "pkg/tcpip/faketime.manualTimerMu" -} - -func (m *manualTimerMu) StateFields() []string { - return []string{ - "firesAt", - } -} - -func (m *manualTimerMu) beforeSave() {} - -// +checklocksignore -func (m *manualTimerMu) StateSave(stateSinkObject state.Sink) { - m.beforeSave() - stateSinkObject.Save(0, &m.firesAt) -} - -func (m *manualTimerMu) afterLoad(context.Context) {} - -// +checklocksignore -func (m *manualTimerMu) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &m.firesAt) -} - -func (mt *manualTimer) StateTypeName() string { - return "pkg/tcpip/faketime.manualTimer" -} - -func (mt *manualTimer) StateFields() []string { - return []string{ - "clock", - "mu", - } -} - -func (mt *manualTimer) beforeSave() {} - -// +checklocksignore -func (mt *manualTimer) StateSave(stateSinkObject state.Sink) { - mt.beforeSave() - stateSinkObject.Save(0, &mt.clock) - stateSinkObject.Save(1, &mt.mu) -} - -func (mt *manualTimer) afterLoad(context.Context) {} - -// +checklocksignore -func (mt *manualTimer) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &mt.clock) - stateSourceObject.Load(1, &mt.mu) -} - func init() { - state.Register((*NullClock)(nil)) - state.Register((*nullTimer)(nil)) - state.Register((*manualClockMutex)(nil)) state.Register((*ManualClock)(nil)) - state.Register((*manualTimerMu)(nil)) - state.Register((*manualTimer)(nil)) } diff --git a/pkg/tcpip/header/ipv6.go b/pkg/tcpip/header/ipv6.go index 53584df..049319b 100644 --- a/pkg/tcpip/header/ipv6.go +++ b/pkg/tcpip/header/ipv6.go @@ -205,10 +205,80 @@ func (b IPv6) NextHeader() uint8 { } // TransportProtocol implements Network.TransportProtocol. +// +// Deprecated: Use TryParseTransportProtocol instead. +// This function does not parse extension headers and returns the next header +// field of the IPv6 header as the transport +// protocol which may not be the actual transport protocol. +// Use TryParseTransportProtocol to get the transport protocol correctly. func (b IPv6) TransportProtocol() tcpip.TransportProtocolNumber { return tcpip.TransportProtocolNumber(b.NextHeader()) } +// IsExtensionHeader returns true if the next header is a known extension header. +func IsExtensionHeader(nextHdr uint8) bool { + extType := IPv6ExtensionHeaderIdentifier(nextHdr) + switch extType { + case IPv6HopByHopOptionsExtHdrIdentifier, IPv6RoutingExtHdrIdentifier, IPv6FragmentExtHdrIdentifier, IPv6DestinationOptionsExtHdrIdentifier, IPv6AuthenticationExtHdrIdentifier, IPv6NoNextHeaderIdentifier: + return true + default: + return false + } +} + +// TryParseTransportProtocol parses the IPv6 header and extension headers to get the +// transport protocol. +// Reference: net/ipv6/exthdrs_core.c:ipv6_skip_exthdr. +// Returns the transport protocol and a boolean indicating if the transport +// protocol parsing was successful. +func (b IPv6) TryParseTransportProtocol() (tcpip.TransportProtocolNumber, bool) { + if len(b) < IPv6MinimumSize { + return 0, false + } + data := []byte(b[IPv6MinimumSize:]) + nxtHdr := b.NextHeader() + maybeProto := tcpip.TransportProtocolNumber(nxtHdr) + for IsExtensionHeader(nxtHdr) { + dataLen := len(data) + if dataLen < 2 { + return maybeProto, false + } + currHdrLen := 0 + switch IPv6ExtensionHeaderIdentifier(nxtHdr) { + case IPv6FragmentExtHdrIdentifier: + // Fragment extension header is always 8 bytes long. + if dataLen < 8 { + return maybeProto, false + } + // Get the fragment offset from the fragment extension header. + fragOffset := binary.BigEndian.Uint16(data[2:4]) & ^uint16(0x7) + if fragOffset != 0 { + return tcpip.TransportProtocolNumber(data[0]), false + } + currHdrLen = 8 + case IPv6HopByHopOptionsExtHdrIdentifier, IPv6RoutingExtHdrIdentifier, IPv6DestinationOptionsExtHdrIdentifier: + currHdrLen = int(data[1]+1) * 8 + case IPv6AuthenticationExtHdrIdentifier: + // Authentication extension header length calculation is different from + // other extension headers. + currHdrLen = int(data[1]+2) * 4 + default: + // IPv6NoNextHeaderIdentifier or any unknown extension header. + return maybeProto, false + } + if currHdrLen > len(data) { + return maybeProto, false + } + nxtHdr = data[0] + maybeProto = tcpip.TransportProtocolNumber(nxtHdr) + data = data[currHdrLen:] + } + if len(data) == 0 { + return maybeProto, false + } + return maybeProto, true +} + // Payload implements Network.Payload. func (b IPv6) Payload() []byte { return b[IPv6MinimumSize:][:b.PayloadLength()] diff --git a/pkg/tcpip/header/ipv6_extension_headers.go b/pkg/tcpip/header/ipv6_extension_headers.go index 2d64c24..b5bac1f 100644 --- a/pkg/tcpip/header/ipv6_extension_headers.go +++ b/pkg/tcpip/header/ipv6_extension_headers.go @@ -45,6 +45,11 @@ const ( // Destination Options extension header, as per RFC 8200 section 4.6. IPv6DestinationOptionsExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 60 + // IPv6AuthenticationExtHdrIdentifier is the header identifier of an + // Authentication extension header, as per RFC 8200 section 4.1. + // TODO: b/512233021 - Parse Authentication extension header correctly. + IPv6AuthenticationExtHdrIdentifier IPv6ExtensionHeaderIdentifier = 51 + // IPv6NoNextHeaderIdentifier is the header identifier used to signify the end // of an IPv6 payload, as per RFC 8200 section 4.7. IPv6NoNextHeaderIdentifier IPv6ExtensionHeaderIdentifier = 59 diff --git a/pkg/tcpip/header/parse/parse.go b/pkg/tcpip/header/parse/parse.go index f31a62c..9326e77 100644 --- a/pkg/tcpip/header/parse/parse.go +++ b/pkg/tcpip/header/parse/parse.go @@ -158,9 +158,19 @@ traverseExtensions: // // Returns true if the header was successfully parsed. func UDP(pkt *stack.PacketBuffer) bool { - _, ok := pkt.TransportHeader().Consume(header.UDPMinimumSize) + hdr, ok := pkt.TransportHeader().Consume(header.UDPMinimumSize) + if !ok { + return false + } pkt.TransportProtocolNumber = header.UDPProtocolNumber - return ok + // Validate the UDP payload length. + length := int(header.UDP(hdr).Length()) - header.UDPMinimumSize + if length < 0 || length > pkt.Data().Size() { + return false + } + // Trim the payload to the length specified in the UDP header. + pkt.Data().CapLength(length) + return true } // TCP parses a TCP packet found in pkt.Data and populates pkt's transport diff --git a/pkg/tcpip/header/tcp.go b/pkg/tcpip/header/tcp.go index a95d13e..56cf1c3 100644 --- a/pkg/tcpip/header/tcp.go +++ b/pkg/tcpip/header/tcp.go @@ -17,7 +17,6 @@ package header import ( "encoding/binary" - "github.com/google/btree" "github.com/sagernet/gvisor/pkg/tcpip" "github.com/sagernet/gvisor/pkg/tcpip/checksum" "github.com/sagernet/gvisor/pkg/tcpip/seqnum" @@ -175,11 +174,6 @@ type SACKBlock struct { End seqnum.Value } -// Less returns true if r.Start < b.Start. -func (r SACKBlock) Less(b btree.Item) bool { - return r.Start.LessThan(b.(SACKBlock).Start) -} - // Contains returns true if b is completely contained in r. func (r SACKBlock) Contains(b SACKBlock) bool { return r.Start.LessThanEq(b.Start) && b.End.LessThanEq(r.End) @@ -219,9 +213,8 @@ const ( // TCPTotalHeaderMaximumSize is the maximum size of headers from all layers in // a TCP packet. It analogous to MAX_TCP_HEADER in Linux. // - // TODO(b/319936470): Investigate why this needs to be at least 140 bytes. In - // Linux this value is at least 160, but in theory we should be able to use - // 138. In practice anything less than 140 starts to break GSO on gVNIC + // Note: In Linux this value is at least 160, but in theory we should be able + // to use 138. In practice anything less than 140 starts to break GSO on gVNIC // hardware. TCPTotalHeaderMaximumSize = 160 diff --git a/pkg/tcpip/header/udp.go b/pkg/tcpip/header/udp.go index 2fc93a1..1d9176a 100644 --- a/pkg/tcpip/header/udp.go +++ b/pkg/tcpip/header/udp.go @@ -138,20 +138,33 @@ func (b UDP) Encode(u *UDPFields) { // SetSourcePortWithChecksumUpdate implements ChecksummableTransport. func (b UDP) SetSourcePortWithChecksumUpdate(new uint16) { + if b.Checksum() == 0 { + b.SetSourcePort(new) + return + } old := b.SourcePort() b.SetSourcePort(new) - b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) + xsum := ^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new) + b.SetChecksum(normalizeChecksum(xsum)) } // SetDestinationPortWithChecksumUpdate implements ChecksummableTransport. func (b UDP) SetDestinationPortWithChecksumUpdate(new uint16) { + if b.Checksum() == 0 { + b.SetDestinationPort(new) + return + } old := b.DestinationPort() b.SetDestinationPort(new) - b.SetChecksum(^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new)) + xsum := ^checksumUpdate2ByteAlignedUint16(^b.Checksum(), old, new) + b.SetChecksum(normalizeChecksum(xsum)) } // UpdateChecksumPseudoHeaderAddress implements ChecksummableTransport. func (b UDP) UpdateChecksumPseudoHeaderAddress(old, new tcpip.Address, fullChecksum bool) { + if fullChecksum && b.Checksum() == 0 { + return + } xsum := b.Checksum() if fullChecksum { xsum = ^xsum @@ -159,7 +172,7 @@ func (b UDP) UpdateChecksumPseudoHeaderAddress(old, new tcpip.Address, fullCheck xsum = checksumUpdate2ByteAlignedAddress(xsum, old, new) if fullChecksum { - xsum = ^xsum + xsum = normalizeChecksum(^xsum) } b.SetChecksum(xsum) @@ -197,3 +210,14 @@ func UDPValid(hdr UDP, payloadChecksum func() uint16, payloadSize uint16, netPro return true, hdr.IsChecksumValid(srcAddr, dstAddr, payloadChecksum()) } + +func normalizeChecksum(xsum uint16) uint16 { + // RFC 768: + // If the computed UDP checksum is zero, it is transmitted as all ones. + // An all zero transmitted checksum value means that + // the transmitter generated no checksum. + if xsum == 0 { + return 0xFFFF + } + return xsum +} diff --git a/pkg/tcpip/link/channel/endpoint_mutex.go b/pkg/tcpip/link/channel/endpoint_mutex.go index 8c834d2..9aac04a 100644 --- a/pkg/tcpip/link/channel/endpoint_mutex.go +++ b/pkg/tcpip/link/channel/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/channel/queue_mutex.go b/pkg/tcpip/link/channel/queue_mutex.go index c7f2f8f..602b206 100644 --- a/pkg/tcpip/link/channel/queue_mutex.go +++ b/pkg/tcpip/link/channel/queue_mutex.go @@ -92,5 +92,5 @@ func queueinitLockNames() {} func init() { queueinitLockNames() - queueprefixIndex = locking.NewMutexClass(reflect.TypeOf(queueRWMutex{}), queuelockNames) + queueprefixIndex = locking.NewMutexClass(reflect.TypeFor[queueRWMutex](), queuelockNames) } diff --git a/pkg/tcpip/link/ethernet/ethernet.go b/pkg/tcpip/link/ethernet/ethernet.go index 80f3cdb..8404342 100644 --- a/pkg/tcpip/link/ethernet/ethernet.go +++ b/pkg/tcpip/link/ethernet/ethernet.go @@ -56,7 +56,11 @@ func (e *Endpoint) LinkAddress() tcpip.LinkAddress { // MTU implements stack.LinkEndpoint. func (e *Endpoint) MTU() uint32 { - return e.Endpoint.MTU() + // It prevents upper-layers from sending larger than expected packets. + if mtu := e.Endpoint.MTU(); mtu > header.EthernetMinimumSize { + return mtu - header.EthernetMinimumSize + } + return 0 } // DeliverNetworkPacket implements stack.NetworkDispatcher. diff --git a/pkg/tcpip/link/fdbased/endpoint.go b/pkg/tcpip/link/fdbased/endpoint.go index 1561ca7..dd19ddd 100644 --- a/pkg/tcpip/link/fdbased/endpoint.go +++ b/pkg/tcpip/link/fdbased/endpoint.go @@ -203,10 +203,6 @@ type Options struct { // include CapabilitySaveRestore SaveRestore bool - // DisconnectOk if true, indicates that this NIC capability set should - // include CapabilityDisconnectOk. - DisconnectOk bool - // GSOMaxSize is the maximum GSO packet size. It is zero if GSO is // disabled. GSOMaxSize uint32 @@ -240,19 +236,21 @@ type Options struct { // ProcessorsPerChannel is the number of goroutines used to handle packets // from each FD. ProcessorsPerChannel int + + // IsPacketSocket indicates whether each FD is a packet socket. + // If nil, getsockname will be called. + IsPacketSocket []bool + + // PreConfigured indicates that socket setup (getsockname, setsockopt) + // has already been performed on the host. + PreConfigured bool } -// fanoutID is used for AF_PACKET based endpoints to enable PACKET_FANOUT -// support in the host kernel. This allows us to use multiple FD's to receive -// from the same underlying NIC. The fanoutID needs to be the same for a given -// set of FD's that point to the same NIC. Trying to set the PACKET_FANOUT -// option for an FD with a fanoutID already in use by another FD for a different -// NIC will return an EINVAL. -// -// Since fanoutID must be unique within the network namespace, we start with -// the PID to avoid collisions. The only way to be sure of avoiding collisions -// is to run in a new network namespace. -var fanoutID atomicbitops.Int32 = atomicbitops.FromInt32(int32(unix.Getpid())) +// fallbackFanoutID is used only when PACKET_FANOUT_FLAG_UNIQUEID is not +// supported by the host kernel. It preserves the PID-seeded best-effort behavior: +// seed from unix.Getpid() and increment per endpoint. This is not +// collision-free across sentries that share a network namespace. +var fallbackFanoutID atomicbitops.Int32 = atomicbitops.FromInt32(int32(unix.Getpid())) // New creates a new fd-based endpoint. // @@ -279,10 +277,6 @@ func New(opts *Options) (stack.LinkEndpoint, error) { caps |= stack.CapabilitySaveRestore } - if opts.DisconnectOk { - caps |= stack.CapabilityDisconnectOk - } - if len(opts.FDs) == 0 { return nil, fmt.Errorf("opts.FD is empty, at least one FD must be specified") } @@ -307,17 +301,18 @@ func New(opts *Options) (stack.LinkEndpoint, error) { } } - // Increment fanoutID to ensure that we don't re-use the same fanoutID - // for the next endpoint. - fid := fanoutID.Add(1) + // Fanout id allocated by the kernel for this endpoint. All AF_PACKET FDs + // belonging to this endpoint must use the same id. -1 means no AF_PACKET + // FD has allocated an id yet; 0 is a valid fanout id. + fid := int32(-1) // Create per channel dispatchers. - for _, fd := range opts.FDs { + for i, fd := range opts.FDs { if err := unix.SetNonblock(fd, true); err != nil { return nil, fmt.Errorf("unix.SetNonblock(%v) failed: %v", fd, err) } - isSocket, err := isSocketFD(fd) + isSocket, err := IsSocketFD(fd) if err != nil { return nil, err } @@ -334,7 +329,34 @@ func New(opts *Options) (stack.LinkEndpoint, error) { opts.ProcessorsPerChannel = max(1, runtime.GOMAXPROCS(0)/len(opts.FDs)) } - inboundDispatcher, err := createInboundDispatcher(e, fd, isSocket, fid, opts) + var isPacket bool + if opts.PreConfigured { + if opts.IsPacketSocket != nil && i < len(opts.IsPacketSocket) { + isPacket = opts.IsPacketSocket[i] + } else { + return nil, fmt.Errorf("PreConfigured is true but IsPacketSocket is missing or too short (index %d, len %d)", i, len(opts.IsPacketSocket)) + } + } else { + var err error + isPacket, err = IsPacketSocket(fd, isSocket) + if err != nil { + return nil, err + } + } + + if isPacket && !opts.PreConfigured { + var err error + if fid < 0 { + fid, err = CreatePacketFanoutGroup(fd) + } else { + err = JoinPacketFanoutGroup(fd, fid) + } + if err != nil { + return nil, fmt.Errorf("failed to enable PACKET_FANOUT option: %v", err) + } + } + + inboundDispatcher, err := createInboundDispatcher(e, fd, isSocket, opts) if err != nil { return nil, fmt.Errorf("createInboundDispatcher(...) = %v", err) } @@ -344,7 +366,7 @@ func New(opts *Options) (stack.LinkEndpoint, error) { return e, nil } -func createInboundDispatcher(e *endpoint, fd int, isSocket bool, fID int32, opts *Options) (linkDispatcher, error) { +func createInboundDispatcher(e *endpoint, fd int, isSocket bool, opts *Options) (linkDispatcher, error) { // By default use the readv() dispatcher as it works with all kinds of // FDs (tap/tun/unix domain sockets and af_packet). inboundDispatcher, err := newReadVDispatcher(fd, e, opts) @@ -353,38 +375,6 @@ func createInboundDispatcher(e *endpoint, fd int, isSocket bool, fID int32, opts } if isSocket { - sa, err := unix.Getsockname(fd) - if err != nil { - return nil, fmt.Errorf("unix.Getsockname(%d) = %v", fd, err) - } - switch sa.(type) { - case *unix.SockaddrLinklayer: - // Enable PACKET_FANOUT mode if the underlying socket is of type - // AF_PACKET. We do not enable PACKET_FANOUT_FLAG_DEFRAG as that will - // prevent gvisor from receiving fragmented packets and the host does the - // reassembly on our behalf before delivering the fragments. This makes it - // hard to test fragmentation reassembly code in Netstack. - // - // See: include/uapi/linux/if_packet.h (struct fanout_args). - // - // NOTE: We are using SetSockOptInt here even though the underlying - // option is actually a struct. The code follows the example in the - // kernel documentation as described at the link below: - // - // See: https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt - // - // This works out because the actual implementation for the option zero - // initializes the structure and will initialize the max_members field - // to a proper value if zero. - // - // See: https://github.com/torvalds/linux/blob/7acac4b3196caee5e21fb5ea53f8bc124e6a16fc/net/packet/af_packet.c#L3881 - const fanoutType = unix.PACKET_FANOUT_HASH - fanoutArg := (int(fID) & 0xffff) | fanoutType<<16 - if err := unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT, fanoutArg); err != nil { - return nil, fmt.Errorf("failed to enable PACKET_FANOUT option: %v", err) - } - } - switch e.packetDispatchMode { case PacketMMap: inboundDispatcher, err = newPacketMMapDispatcher(fd, e, opts) @@ -407,7 +397,82 @@ func createInboundDispatcher(e *endpoint, fd int, isSocket bool, fID int32, opts return inboundDispatcher, nil } -func isSocketFD(fd int) (bool, error) { +// IsPacketSocket checks if the FD is an AF_PACKET socket. +func IsPacketSocket(fd int, isSocket bool) (bool, error) { + if !isSocket { + return false, nil + } + sa, err := unix.Getsockname(fd) + if err != nil { + return false, fmt.Errorf("unix.Getsockname(%d) = %v", fd, err) + } + _, ok := sa.(*unix.SockaddrLinklayer) + return ok, nil +} + +// CreatePacketFanoutGroup enables PACKET_FANOUT for the first AF_PACKET socket +// in an endpoint and returns the fanout id the group joined. +// +// All AF_PACKET FDs that back the same endpoint must join the same fanout +// group so the host kernel consistently hashes packets for a flow to one FD. +// Fanout ids are unique within the Linux network namespace that owns the +// sockets; reusing an id for a different NIC in that namespace fails with +// EINVAL. +// +// We ask the kernel to allocate the id via PACKET_FANOUT_FLAG_UNIQUEID so the +// id is guaranteed unique within the namespace even when multiple sentries +// share it. If that setsockopt fails (e.g. the host kernel predates +// PACKET_FANOUT_FLAG_UNIQUEID), we fall back to the PID-seeded +// fallbackFanoutID, which is best-effort and not collision-free across +// sentries that share a network namespace. +// +// We do not enable PACKET_FANOUT_FLAG_DEFRAG as that will prevent gvisor from +// receiving fragmented packets and the host does the reassembly on our behalf +// before delivering the fragments. This makes it hard to test fragmentation +// reassembly code in Netstack. +// +// See: include/uapi/linux/if_packet.h (struct fanout_args). +// +// NOTE: We are using SetSockOptInt here even though the underlying option is +// actually a struct. The code follows the example in the kernel documentation +// as described at the link below: +// +// See: https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt +// +// This works out because the actual implementation for the option zero +// initializes the structure and will initialize the max_members field to a +// proper value if zero. +// +// See: https://github.com/torvalds/linux/blob/7acac4b3196caee5e21fb5ea53f8bc124e6a16fc/net/packet/af_packet.c#L3881 +func CreatePacketFanoutGroup(fd int) (int32, error) { + const fanoutType = unix.PACKET_FANOUT_HASH + fanoutArg := (fanoutType | unix.PACKET_FANOUT_FLAG_UNIQUEID) << 16 + if err := unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT, fanoutArg); err != nil { + uniqueIDErr := err + fallbackID := fallbackFanoutID.Add(1) + fanoutArg = (int(fallbackID) & 0xffff) | fanoutType<<16 + if err := unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT, fanoutArg); err != nil { + return 0, fmt.Errorf("UNIQUEID failed (%v); fallback fanout id %d also failed: %v", uniqueIDErr, fanoutArg&0xffff, err) + } + return int32(fanoutArg & 0xffff), nil + } + + fanoutArg, err := unix.GetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT) + if err != nil { + return 0, fmt.Errorf("getsockopt(PACKET_FANOUT) failed: %v", err) + } + return int32(fanoutArg & 0xffff), nil +} + +// JoinPacketFanoutGroup joins the FD to the specified fanout group. +func JoinPacketFanoutGroup(fd int, fID int32) error { + const fanoutType = unix.PACKET_FANOUT_HASH + fanoutArg := (int(fID) & 0xffff) | fanoutType<<16 + return unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT, fanoutArg) +} + +// IsSocketFD checks if the FD is a socket. +func IsSocketFD(fd int) (bool, error) { var stat unix.Stat_t if err := unix.Fstat(fd, &stat); err != nil { return false, fmt.Errorf("unix.Fstat(%v,...) failed: %v", fd, err) @@ -892,7 +957,7 @@ func (e *InjectableEndpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, // NewInjectable creates a new fd-based InjectableEndpoint. func NewInjectable(fd int, mtu uint32, capabilities stack.LinkEndpointCapabilities) (*InjectableEndpoint, error) { unix.SetNonblock(fd, true) - isSocket, err := isSocketFD(fd) + isSocket, err := IsSocketFD(fd) if err != nil { return nil, err } diff --git a/pkg/tcpip/link/fdbased/endpoint_mutex.go b/pkg/tcpip/link/fdbased/endpoint_mutex.go index d05b264..6b92c14 100644 --- a/pkg/tcpip/link/fdbased/endpoint_mutex.go +++ b/pkg/tcpip/link/fdbased/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/fdbased/fdbased_state_autogen.go b/pkg/tcpip/link/fdbased/fdbased_state_autogen.go index c058305..c8f11a5 100644 --- a/pkg/tcpip/link/fdbased/fdbased_state_autogen.go +++ b/pkg/tcpip/link/fdbased/fdbased_state_autogen.go @@ -112,7 +112,6 @@ func (o *Options) StateFields() []string { "ClosedFunc", "Address", "SaveRestore", - "DisconnectOk", "GSOMaxSize", "GVisorGSOEnabled", "PacketDispatchMode", @@ -122,6 +121,8 @@ func (o *Options) StateFields() []string { "InterfaceIndex", "GRO", "ProcessorsPerChannel", + "IsPacketSocket", + "PreConfigured", } } @@ -136,16 +137,17 @@ func (o *Options) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(3, &o.ClosedFunc) stateSinkObject.Save(4, &o.Address) stateSinkObject.Save(5, &o.SaveRestore) - stateSinkObject.Save(6, &o.DisconnectOk) - stateSinkObject.Save(7, &o.GSOMaxSize) - stateSinkObject.Save(8, &o.GVisorGSOEnabled) - stateSinkObject.Save(9, &o.PacketDispatchMode) - stateSinkObject.Save(10, &o.TXChecksumOffload) - stateSinkObject.Save(11, &o.RXChecksumOffload) - stateSinkObject.Save(12, &o.MaxSyscallHeaderBytes) - stateSinkObject.Save(13, &o.InterfaceIndex) - stateSinkObject.Save(14, &o.GRO) - stateSinkObject.Save(15, &o.ProcessorsPerChannel) + stateSinkObject.Save(6, &o.GSOMaxSize) + stateSinkObject.Save(7, &o.GVisorGSOEnabled) + stateSinkObject.Save(8, &o.PacketDispatchMode) + stateSinkObject.Save(9, &o.TXChecksumOffload) + stateSinkObject.Save(10, &o.RXChecksumOffload) + stateSinkObject.Save(11, &o.MaxSyscallHeaderBytes) + stateSinkObject.Save(12, &o.InterfaceIndex) + stateSinkObject.Save(13, &o.GRO) + stateSinkObject.Save(14, &o.ProcessorsPerChannel) + stateSinkObject.Save(15, &o.IsPacketSocket) + stateSinkObject.Save(16, &o.PreConfigured) } func (o *Options) afterLoad(context.Context) {} @@ -158,16 +160,17 @@ func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) stateSourceObject.Load(3, &o.ClosedFunc) stateSourceObject.Load(4, &o.Address) stateSourceObject.Load(5, &o.SaveRestore) - stateSourceObject.Load(6, &o.DisconnectOk) - stateSourceObject.Load(7, &o.GSOMaxSize) - stateSourceObject.Load(8, &o.GVisorGSOEnabled) - stateSourceObject.Load(9, &o.PacketDispatchMode) - stateSourceObject.Load(10, &o.TXChecksumOffload) - stateSourceObject.Load(11, &o.RXChecksumOffload) - stateSourceObject.Load(12, &o.MaxSyscallHeaderBytes) - stateSourceObject.Load(13, &o.InterfaceIndex) - stateSourceObject.Load(14, &o.GRO) - stateSourceObject.Load(15, &o.ProcessorsPerChannel) + stateSourceObject.Load(6, &o.GSOMaxSize) + stateSourceObject.Load(7, &o.GVisorGSOEnabled) + stateSourceObject.Load(8, &o.PacketDispatchMode) + stateSourceObject.Load(9, &o.TXChecksumOffload) + stateSourceObject.Load(10, &o.RXChecksumOffload) + stateSourceObject.Load(11, &o.MaxSyscallHeaderBytes) + stateSourceObject.Load(12, &o.InterfaceIndex) + stateSourceObject.Load(13, &o.GRO) + stateSourceObject.Load(14, &o.ProcessorsPerChannel) + stateSourceObject.Load(15, &o.IsPacketSocket) + stateSourceObject.Load(16, &o.PreConfigured) } func (e *InjectableEndpoint) StateTypeName() string { diff --git a/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go b/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go index 2c813d8..720f7e0 100644 --- a/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go +++ b/pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go @@ -92,5 +92,5 @@ func injectableEndpointinitLockNames() {} func init() { injectableEndpointinitLockNames() - injectableEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(injectableEndpointRWMutex{}), injectableEndpointlockNames) + injectableEndpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[injectableEndpointRWMutex](), injectableEndpointlockNames) } diff --git a/pkg/tcpip/link/fdbased/processor_mutex.go b/pkg/tcpip/link/fdbased/processor_mutex.go index cd297d2..bfea8f9 100644 --- a/pkg/tcpip/link/fdbased/processor_mutex.go +++ b/pkg/tcpip/link/fdbased/processor_mutex.go @@ -60,5 +60,5 @@ func processorinitLockNames() {} func init() { processorinitLockNames() - processorprefixIndex = locking.NewMutexClass(reflect.TypeOf(processorMutex{}), processorlockNames) + processorprefixIndex = locking.NewMutexClass(reflect.TypeFor[processorMutex](), processorlockNames) } diff --git a/pkg/tcpip/link/fdbased/processors.go b/pkg/tcpip/link/fdbased/processors.go index 47450f8..6330e59 100644 --- a/pkg/tcpip/link/fdbased/processors.go +++ b/pkg/tcpip/link/fdbased/processors.go @@ -46,7 +46,6 @@ type processor struct { func (p *processor) start(wg *sync.WaitGroup) { defer wg.Done() - defer p.sleeper.Done() for { switch w := p.sleeper.Fetch(true); { case w == &p.packetWaker: @@ -128,9 +127,10 @@ func (m *processorManager) start() { } // afterLoad is invoked by stateify. -func (m *processorManager) afterLoad(context.Context) { - m.wg.Add(len(m.processors)) - m.start() +func (m *processorManager) afterLoad(ctx context.Context) { + // Close all the old/saved processors. There are new NICs and + // processors created during restore. + m.close() } func (m *processorManager) connectionHash(cid *connectionID) uint32 { @@ -215,34 +215,47 @@ func tcpipConnectionID(pkt *stack.PacketBuffer) (connectionID, bool) { return cid, true } ipHdr := header.IPv6(h) + cid.srcAddr = ipHdr.SourceAddressSlice() + cid.dstAddr = ipHdr.DestinationAddressSlice() + cid.proto = header.IPv6ProtocolNumber - var tcpHdr header.TCP - if tcpip.TransportProtocolNumber(ipHdr.NextHeader()) == header.TCPProtocolNumber { - tcpHdr = header.TCP(h[header.IPv6FixedHeaderSize:][:tcpSrcDstPortLen]) + if !header.IsExtensionHeader(ipHdr.NextHeader()) { + // Known transport protocols(not just TCP) store the src and dst ports + // in the first 4 bytes after the IPv6 fixed header. + tcpHdr := header.TCP(h[header.IPv6FixedHeaderSize:][:tcpSrcDstPortLen]) + cid.srcPort = tcpHdr.SourcePort() + cid.dstPort = tcpHdr.DestinationPort() } else { // Slow path for IPv6 extension headers :(. dataBuf := pkt.Data().ToBuffer() dataBuf.TrimFront(header.IPv6MinimumSize) it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(ipHdr.NextHeader()), dataBuf) defer it.Release() + // All fragment packets need to be processed by the same goroutine, so + // only record the ports if this is not a fragment packet. + var isFragment bool for { hdr, done, err := it.Next() if done || err != nil { break } + if fh, ok := hdr.(header.IPv6FragmentExtHdr); ok && !fh.IsAtomic() { + isFragment = true + } hdr.Release() } - h, ok = pkt.Data().PullUp(int(it.HeaderOffset()) + tcpSrcDstPortLen) - if !ok { - return cid, true + if !isFragment { + h, ok = pkt.Data().PullUp(int(it.HeaderOffset()) + tcpSrcDstPortLen) + if !ok { + return cid, true + } + // Known transport protocols store the src and dst ports + // in the first 4 bytes after the IPv6 fixed header. + tcpHdr := header.TCP(h[it.HeaderOffset():][:tcpSrcDstPortLen]) + cid.srcPort = tcpHdr.SourcePort() + cid.dstPort = tcpHdr.DestinationPort() } - tcpHdr = header.TCP(h[it.HeaderOffset():][:tcpSrcDstPortLen]) } - cid.srcAddr = ipHdr.SourceAddressSlice() - cid.dstAddr = ipHdr.DestinationAddressSlice() - cid.srcPort = tcpHdr.SourcePort() - cid.dstPort = tcpHdr.DestinationPort() - cid.proto = header.IPv6ProtocolNumber default: return cid, true } diff --git a/pkg/tcpip/link/loopback/endpoint_mutex.go b/pkg/tcpip/link/loopback/endpoint_mutex.go index 2d34fc2..2d9134a 100644 --- a/pkg/tcpip/link/loopback/endpoint_mutex.go +++ b/pkg/tcpip/link/loopback/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/muxed/endpoint_mutex.go b/pkg/tcpip/link/muxed/endpoint_mutex.go index 2f39d9a..4324161 100644 --- a/pkg/tcpip/link/muxed/endpoint_mutex.go +++ b/pkg/tcpip/link/muxed/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/pipe/endpoint_mutex.go b/pkg/tcpip/link/pipe/endpoint_mutex.go index 456122f..86e2f16 100644 --- a/pkg/tcpip/link/pipe/endpoint_mutex.go +++ b/pkg/tcpip/link/pipe/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go b/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go index 557341a..c6d91bb 100644 --- a/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go +++ b/pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go @@ -60,5 +60,5 @@ func queueDispatcherinitLockNames() {} func init() { queueDispatcherinitLockNames() - queueDispatcherprefixIndex = locking.NewMutexClass(reflect.TypeOf(queueDispatcherMutex{}), queueDispatcherlockNames) + queueDispatcherprefixIndex = locking.NewMutexClass(reflect.TypeFor[queueDispatcherMutex](), queueDispatcherlockNames) } diff --git a/pkg/tcpip/link/qdisc/fifo/fifo.go b/pkg/tcpip/link/qdisc/fifo/fifo.go index fe3e135..e64609a 100644 --- a/pkg/tcpip/link/qdisc/fifo/fifo.go +++ b/pkg/tcpip/link/qdisc/fifo/fifo.go @@ -22,6 +22,7 @@ import ( "github.com/sagernet/gvisor/pkg/sleep" "github.com/sagernet/gvisor/pkg/sync" "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/link/qdisc" "github.com/sagernet/gvisor/pkg/tcpip/stack" ) @@ -60,7 +61,7 @@ type queueDispatcher struct { mu queueDispatcherMutex `state:"nosave"` // +checklocks:mu - queue packetBufferCircularList + queue qdisc.PacketBufferCircularList newPacketWaker sleep.Waker `state:"nosave"` closeWaker sleep.Waker `state:"nosave"` @@ -78,7 +79,7 @@ func New(lower stack.LinkWriter, n int, queueLen int) stack.QueueingDiscipline { for i := range d.dispatchers { qd := &d.dispatchers[i] qd.lower = lower - qd.queue.init(queueLen) + qd.queue.Init(queueLen) d.wg.Add(1) go func() { @@ -101,19 +102,19 @@ func (qd *queueDispatcher) dispatchLoop() { case &qd.newPacketWaker: case &qd.closeWaker: qd.mu.Lock() - for p := qd.queue.removeFront(); p != nil; p = qd.queue.removeFront() { + for p := qd.queue.RemoveFront(); p != nil; p = qd.queue.RemoveFront() { p.DecRef() } - qd.queue.decRef() + qd.queue.DecRef() qd.mu.Unlock() return default: panic("unknown waker") } qd.mu.Lock() - for pkt := qd.queue.removeFront(); pkt != nil; pkt = qd.queue.removeFront() { + for pkt := qd.queue.RemoveFront(); pkt != nil; pkt = qd.queue.RemoveFront() { batch.PushBack(pkt) - if batch.Len() < BatchSize && !qd.queue.isEmpty() { + if batch.Len() < BatchSize && !qd.queue.IsEmpty() { continue } qd.mu.Unlock() @@ -137,9 +138,13 @@ func (d *discipline) WritePacket(pkt *stack.PacketBuffer) tcpip.Error { } qd := &d.dispatchers[int(pkt.Hash)%len(d.dispatchers)] qd.mu.Lock() - haveSpace := qd.queue.hasSpace() + if d.closed.Load() == qDiscClosed { + qd.mu.Unlock() + return &tcpip.ErrClosedForSend{} + } + haveSpace := qd.queue.HasSpace() if haveSpace { - qd.queue.pushBack(pkt.IncRef()) + qd.queue.PushBack(pkt.IncRef()) } qd.mu.Unlock() if !haveSpace { diff --git a/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go b/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go index 5dccdd1..659e910 100644 --- a/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go +++ b/pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go @@ -64,39 +64,7 @@ func (qd *queueDispatcher) StateLoad(ctx context.Context, stateSourceObject stat stateSourceObject.Load(1, &qd.queue) } -func (pl *packetBufferCircularList) StateTypeName() string { - return "pkg/tcpip/link/qdisc/fifo.packetBufferCircularList" -} - -func (pl *packetBufferCircularList) StateFields() []string { - return []string{ - "pbs", - "head", - "size", - } -} - -func (pl *packetBufferCircularList) beforeSave() {} - -// +checklocksignore -func (pl *packetBufferCircularList) StateSave(stateSinkObject state.Sink) { - pl.beforeSave() - stateSinkObject.Save(0, &pl.pbs) - stateSinkObject.Save(1, &pl.head) - stateSinkObject.Save(2, &pl.size) -} - -func (pl *packetBufferCircularList) afterLoad(context.Context) {} - -// +checklocksignore -func (pl *packetBufferCircularList) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &pl.pbs) - stateSourceObject.Load(1, &pl.head) - stateSourceObject.Load(2, &pl.size) -} - func init() { state.Register((*discipline)(nil)) state.Register((*queueDispatcher)(nil)) - state.Register((*packetBufferCircularList)(nil)) } diff --git a/pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go b/pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go deleted file mode 100644 index 27ce701..0000000 --- a/pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2022 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fifo - -import "github.com/sagernet/gvisor/pkg/tcpip/stack" - -// packetBufferCircularList is a slice-backed circular list. All operations are -// O(1) unless otherwise noted. It only allocates once, during the call to -// init(). -// -// Users should call init() before using packetBufferCircularList. -// -// +stateify savable -type packetBufferCircularList struct { - pbs []*stack.PacketBuffer - head int - size int -} - -// init initializes the list with the given size. -func (pl *packetBufferCircularList) init(size int) { - pl.pbs = make([]*stack.PacketBuffer, size) -} - -// length returns the number of elements in the list. -// -//go:nosplit -func (pl *packetBufferCircularList) length() int { - return pl.size -} - -// hasSpace returns whether there is space left in the list. -// -//go:nosplit -func (pl *packetBufferCircularList) hasSpace() bool { - return pl.size < len(pl.pbs) -} - -// isEmpty returns whether the list is empty. -// -//go:nosplit -func (pl *packetBufferCircularList) isEmpty() bool { - return pl.size == 0 -} - -// pushBack inserts the PacketBuffer at the end of the list. -// -// Users must check beforehand that there is space via a call to hasSpace(). -// Failing to do so may clobber existing entries. -// -//go:nosplit -func (pl *packetBufferCircularList) pushBack(pb *stack.PacketBuffer) { - next := (pl.head + pl.size) % len(pl.pbs) - pl.pbs[next] = pb - pl.size++ -} - -// removeFront returns the first element of the list or nil. -// -//go:nosplit -func (pl *packetBufferCircularList) removeFront() *stack.PacketBuffer { - if pl.isEmpty() { - return nil - } - ret := pl.pbs[pl.head] - pl.pbs[pl.head] = nil - pl.head = (pl.head + 1) % len(pl.pbs) - pl.size-- - return ret -} - -// decRef decreases the reference count on each stack.PacketBuffer stored in -// the list. -// -// NOTE: runs in O(n) time. -// -//go:nosplit -func (pl *packetBufferCircularList) decRef() { - for i := 0; i < pl.size; i++ { - pl.pbs[(pl.head+i)%len(pl.pbs)].DecRef() - } -} diff --git a/pkg/tcpip/link/qdisc/packet_buffer_circular_list.go b/pkg/tcpip/link/qdisc/packet_buffer_circular_list.go new file mode 100644 index 0000000..fd60477 --- /dev/null +++ b/pkg/tcpip/link/qdisc/packet_buffer_circular_list.go @@ -0,0 +1,108 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package qdisc provides shared building blocks used by queueing disciplines. +package qdisc + +import "github.com/sagernet/gvisor/pkg/tcpip/stack" + +// PacketBufferCircularList is a slice-backed circular list. All operations are +// O(1) unless otherwise noted. It only allocates once, during the call to +// Init(). +// +// Users should call Init() before using PacketBufferCircularList. +// +// +stateify savable +type PacketBufferCircularList struct { + pbs []*stack.PacketBuffer + head int + size int +} + +// Init initializes the list with the given size. +func (pl *PacketBufferCircularList) Init(size int) { + pl.pbs = make([]*stack.PacketBuffer, size) +} + +// Length returns the number of elements in the list. +// +//go:nosplit +func (pl *PacketBufferCircularList) Length() int { + return pl.size +} + +// HasSpace returns whether there is space left in the list. +// +//go:nosplit +func (pl *PacketBufferCircularList) HasSpace() bool { + return pl.size < len(pl.pbs) +} + +// IsEmpty returns whether the list is empty. +// +//go:nosplit +func (pl *PacketBufferCircularList) IsEmpty() bool { + return pl.size == 0 +} + +// PushBack inserts the PacketBuffer at the end of the list. +// +// Users must check beforehand that there is space via a call to HasSpace(). +// Failing to do so may clobber existing entries. +// +//go:nosplit +func (pl *PacketBufferCircularList) PushBack(pb *stack.PacketBuffer) { + next := (pl.head + pl.size) % len(pl.pbs) + pl.pbs[next] = pb + pl.size++ +} + +// PeekFront returns the first element of the list without removing it, or nil +// if empty. The list retains its reference; the caller must not DecRef. To take +// ownership, call RemoveFront, which returns the same pointer. The returned +// pointer is only valid until the next mutation of the list. +// +//go:nosplit +func (pl *PacketBufferCircularList) PeekFront() *stack.PacketBuffer { + if pl.IsEmpty() { + return nil + } + return pl.pbs[pl.head] +} + +// RemoveFront returns the first element of the list or nil. +// +//go:nosplit +func (pl *PacketBufferCircularList) RemoveFront() *stack.PacketBuffer { + if pl.IsEmpty() { + return nil + } + ret := pl.pbs[pl.head] + pl.pbs[pl.head] = nil + pl.head = (pl.head + 1) % len(pl.pbs) + pl.size-- + return ret +} + +// DecRef decreases the reference count on each stack.PacketBuffer stored in +// the list. +// +// NOTE: runs in O(n) time. +// +//go:nosplit +func (pl *PacketBufferCircularList) DecRef() { + for i := 0; i < pl.size; i++ { + pl.pbs[(pl.head+i)%len(pl.pbs)].DecRef() + } +} diff --git a/pkg/tcpip/link/qdisc/qdisc_state_autogen.go b/pkg/tcpip/link/qdisc/qdisc_state_autogen.go new file mode 100644 index 0000000..631571c --- /dev/null +++ b/pkg/tcpip/link/qdisc/qdisc_state_autogen.go @@ -0,0 +1,44 @@ +// automatically generated by stateify. + +package qdisc + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (pl *PacketBufferCircularList) StateTypeName() string { + return "pkg/tcpip/link/qdisc.PacketBufferCircularList" +} + +func (pl *PacketBufferCircularList) StateFields() []string { + return []string{ + "pbs", + "head", + "size", + } +} + +func (pl *PacketBufferCircularList) beforeSave() {} + +// +checklocksignore +func (pl *PacketBufferCircularList) StateSave(stateSinkObject state.Sink) { + pl.beforeSave() + stateSinkObject.Save(0, &pl.pbs) + stateSinkObject.Save(1, &pl.head) + stateSinkObject.Save(2, &pl.size) +} + +func (pl *PacketBufferCircularList) afterLoad(context.Context) {} + +// +checklocksignore +func (pl *PacketBufferCircularList) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &pl.pbs) + stateSourceObject.Load(1, &pl.head) + stateSourceObject.Load(2, &pl.size) +} + +func init() { + state.Register((*PacketBufferCircularList)(nil)) +} diff --git a/pkg/tcpip/link/qdisc/tbf/queue_mutex.go b/pkg/tcpip/link/qdisc/tbf/queue_mutex.go new file mode 100644 index 0000000..f056f6b --- /dev/null +++ b/pkg/tcpip/link/qdisc/tbf/queue_mutex.go @@ -0,0 +1,64 @@ +package tbf + +import ( + "reflect" + + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/sync/locking" +) + +// Mutex is sync.Mutex with the correctness validator. +type queueMutex struct { + mu sync.Mutex +} + +var queueprefixIndex *locking.MutexClass + +// lockNames is a list of user-friendly lock names. +// Populated in init. +var queuelockNames []string + +// lockNameIndex is used as an index passed to NestedLock and NestedUnlock, +// referring to an index within lockNames. +// Values are specified using the "consts" field of go_template_instance. +type queuelockNameIndex int + +// DO NOT REMOVE: The following function automatically replaced with lock index constants. +// LOCK_NAME_INDEX_CONSTANTS +const () + +// Lock locks m. +// +checklocksignore +func (m *queueMutex) Lock() { + locking.AddGLock(queueprefixIndex, -1) + m.mu.Lock() +} + +// NestedLock locks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *queueMutex) NestedLock(i queuelockNameIndex) { + locking.AddGLock(queueprefixIndex, int(i)) + m.mu.Lock() +} + +// Unlock unlocks m. +// +checklocksignore +func (m *queueMutex) Unlock() { + locking.DelGLock(queueprefixIndex, -1) + m.mu.Unlock() +} + +// NestedUnlock unlocks m knowing that another lock of the same type is held. +// +checklocksignore +func (m *queueMutex) NestedUnlock(i queuelockNameIndex) { + locking.DelGLock(queueprefixIndex, int(i)) + m.mu.Unlock() +} + +// DO NOT REMOVE: The following function is automatically replaced. +func queueinitLockNames() {} + +func init() { + queueinitLockNames() + queueprefixIndex = locking.NewMutexClass(reflect.TypeFor[queueMutex](), queuelockNames) +} diff --git a/pkg/tcpip/link/qdisc/tbf/tbf.go b/pkg/tcpip/link/qdisc/tbf/tbf.go new file mode 100644 index 0000000..4f27e0c --- /dev/null +++ b/pkg/tcpip/link/qdisc/tbf/tbf.go @@ -0,0 +1,239 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tbf provides a simplified Token Bucket Filter queueing discipline +// modeled on Linux's net/sched/sch_tbf.c. Only the single-rate bucket is +// implemented; peakrate/peakburst (Linux's second bucket) is not. +package tbf + +import ( + "fmt" + "time" + + "github.com/sagernet/gvisor/pkg/atomicbitops" + "github.com/sagernet/gvisor/pkg/sleep" + "github.com/sagernet/gvisor/pkg/sync" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/link/qdisc" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +const ( + // BatchSize is the number of packets to write in each syscall. It is 47 + // because when GVisorGSO is in use then a single 65KB TCP segment can get + // split into 46 segments of 1420 bytes and a single 216 byte segment. + BatchSize = 47 + + qDiscClosed = 1 +) + +var _ stack.QueueingDiscipline = (*discipline)(nil) + +// +stateify savable +type discipline struct { + // Immutable configuration set by New. + lower stack.LinkWriter + clock tcpip.Clock `state:"nosave"` + rate uint64 // max sustained throughput, bytes/sec + burst uint32 // largest packet this TBF will pass, bytes + buffer int64 // nanoseconds needed to transmit burst bytes at rate + + // Shutdown state. + wg sync.WaitGroup `state:"nosave"` + closed atomicbitops.Int32 + + // Wakers driving dispatchLoop. + newPacketWaker sleep.Waker `state:"nosave"` + tokenWaker sleep.Waker `state:"nosave"` + closeWaker sleep.Waker `state:"nosave"` + + mu queueMutex `state:"nosave"` + // +checklocks:mu + queue qdisc.PacketBufferCircularList + + // Dispatcher state: mutated only inside dispatchLoop and + // thus not protected by mu. + tokens int64 // current bucket level, ns + timeCheckpoint tcpip.MonotonicTime + watchdog tcpip.Timer `state:"nosave"` +} + +// len2TimeNS returns the number of ns to transmit len bytes at rate bytes/sec. +// Linux's psched_l2t_ns avoids the divide via a precomputed mult/shift; see +// psched_ratecfg_precompute__ in net/sched/sch_generic.c. +func len2TimeNS(rate uint64, len uint32) uint64 { + const nsecPerSec = 1000000000 + return uint64(len) * nsecPerSec / rate +} + +func (d *discipline) dispatchLoop() { + s := sleep.Sleeper{} + s.AddWaker(&d.newPacketWaker) + s.AddWaker(&d.tokenWaker) + s.AddWaker(&d.closeWaker) + defer s.Done() + + var batch stack.PacketBufferList + for { + switch w := s.Fetch(true); w { + case &d.newPacketWaker, &d.tokenWaker: + case &d.closeWaker: + if d.watchdog != nil { + d.watchdog.Stop() + } + d.mu.Lock() + for p := d.queue.RemoveFront(); p != nil; p = d.queue.RemoveFront() { + p.DecRef() + } + d.queue.DecRef() + d.mu.Unlock() + return + default: + panic("unknown waker") + } + + d.mu.Lock() + for pkt := d.queue.PeekFront(); pkt != nil; pkt = d.queue.PeekFront() { + pktLen := pkt.Size() + now := d.clock.NowMonotonic() + toks := min(now.Sub(d.timeCheckpoint).Nanoseconds(), d.buffer) + toks += d.tokens + if toks > d.buffer { + toks = d.buffer + } + toks -= int64(len2TimeNS(d.rate, uint32(pktLen))) + sufficientTokens := toks >= 0 + if !sufficientTokens { + // -toks is the deficit in ns: how long until enough tokens accumulate. + if d.watchdog != nil { + d.watchdog.Stop() + } + d.watchdog = d.clock.AfterFunc(time.Duration(-toks), d.tokenWaker.Assert) + break + } + d.queue.RemoveFront() + d.timeCheckpoint = now + d.tokens = toks + batch.PushBack(pkt) + + possiblyAnotherPacket := batch.Len() < BatchSize && !d.queue.IsEmpty() + if possiblyAnotherPacket { + continue + } + d.mu.Unlock() + _, _ = d.lower.WritePackets(batch) + batch.Reset() + d.mu.Lock() + } + if batch.Len() > 0 { + d.mu.Unlock() + _, _ = d.lower.WritePackets(batch) + batch.Reset() + d.mu.Lock() + } + d.mu.Unlock() + } +} + +// New creates a new TBF queueing discipline that will rate-limit lower to +// rate bytes/sec with bursts of up to burst bytes, queueing up to queueLen +// packets of backlog before dropping. Note that queueLen counts packets, +// not bytes as in Linux's sch_tbf.c, for consistency with the fifo qdisc. +// +// +checklocksignore: we don't have to hold locks during initialization. +func New(lower stack.LinkEndpoint, clock tcpip.Clock, rate uint64, burst, queueLen uint32) (stack.QueueingDiscipline, error) { + if rate == 0 { + return nil, fmt.Errorf("qdisc=tbf requires setting qdisc-tbf-rate") + } + + if burst == 0 { + return nil, fmt.Errorf("qdisc=tbf requires setting qdisc-tbf-burst") + } + + if gsoEP, ok := lower.(stack.GSOEndpoint); ok { + // HostGSOSupported endpoints can hand WritePacket a single GSO + // super-packet up to GSOMaxSize+MaxHeaderLength bytes, so the bucket + // must be able to hold one. GVisorGSOSupported segments above the + // qdisc and GSONotSupported never produces packets above the link + // MTU, both covered by the next check. + maxGSOPktLen := gsoEP.GSOMaxSize() + uint32(lower.MaxHeaderLength()) + if gsoEP.SupportedGSO() == stack.HostGSOSupported && burst < uint32(maxGSOPktLen) { + return nil, fmt.Errorf("burst (%d bytes) is smaller than link's max GSO packet size (%d bytes); either increase burst or disable host GSO via --gso=false", burst, maxGSOPktLen) + } + } + + maxPktLen := lower.MTU() + uint32(lower.MaxHeaderLength()) + if burst < maxPktLen { + return nil, fmt.Errorf("burst (%d bytes) is smaller than max packet length (%d bytes)", burst, maxPktLen) + } + + buffer := int64(len2TimeNS(rate, burst)) + if buffer == 0 { + return nil, fmt.Errorf("rate (%d bytes/sec) is too high relative to burst (%d bytes); reduce qdisc-tbf-rate or increase qdisc-tbf-burst", rate, burst) + } + + d := &discipline{ + lower: lower, + clock: clock, + rate: rate, + burst: burst, + buffer: buffer, + tokens: buffer, + timeCheckpoint: clock.NowMonotonic(), + } + d.queue.Init(int(queueLen)) + d.wg.Add(1) + go func() { + defer d.wg.Done() + d.dispatchLoop() + }() + return d, nil +} + +// WritePacket implements stack.QueueingDiscipline.WritePacket. +func (d *discipline) WritePacket(pkt *stack.PacketBuffer) tcpip.Error { + if d.closed.Load() == qDiscClosed { + return &tcpip.ErrClosedForSend{} + } + + if uint32(pkt.Size()) > d.burst { + // if the burst parameter is not smaller than the expected packet size, + // oversize packets should be impossible with New's GSO check + return &tcpip.ErrMessageTooLong{} + } + + d.mu.Lock() + if d.closed.Load() == qDiscClosed { + d.mu.Unlock() + return &tcpip.ErrClosedForSend{} + } + haveSpace := d.queue.HasSpace() + if haveSpace { + d.queue.PushBack(pkt.IncRef()) + } + d.mu.Unlock() + if !haveSpace { + return &tcpip.ErrNoBufferSpace{} + } + + d.newPacketWaker.Assert() + return nil +} + +// Close implements stack.QueueingDiscipline.Close. +func (d *discipline) Close() { + d.closed.Store(qDiscClosed) + d.closeWaker.Assert() + d.wg.Wait() +} diff --git a/pkg/tcpip/link/qdisc/tbf/tbf_state_autogen.go b/pkg/tcpip/link/qdisc/tbf/tbf_state_autogen.go new file mode 100644 index 0000000..529f016 --- /dev/null +++ b/pkg/tcpip/link/qdisc/tbf/tbf_state_autogen.go @@ -0,0 +1,59 @@ +// automatically generated by stateify. + +package tbf + +import ( + "context" + + "github.com/sagernet/gvisor/pkg/state" +) + +func (d *discipline) StateTypeName() string { + return "pkg/tcpip/link/qdisc/tbf.discipline" +} + +func (d *discipline) StateFields() []string { + return []string{ + "lower", + "rate", + "burst", + "buffer", + "closed", + "queue", + "tokens", + "timeCheckpoint", + } +} + +func (d *discipline) beforeSave() {} + +// +checklocksignore +func (d *discipline) StateSave(stateSinkObject state.Sink) { + d.beforeSave() + stateSinkObject.Save(0, &d.lower) + stateSinkObject.Save(1, &d.rate) + stateSinkObject.Save(2, &d.burst) + stateSinkObject.Save(3, &d.buffer) + stateSinkObject.Save(4, &d.closed) + stateSinkObject.Save(5, &d.queue) + stateSinkObject.Save(6, &d.tokens) + stateSinkObject.Save(7, &d.timeCheckpoint) +} + +func (d *discipline) afterLoad(context.Context) {} + +// +checklocksignore +func (d *discipline) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &d.lower) + stateSourceObject.Load(1, &d.rate) + stateSourceObject.Load(2, &d.burst) + stateSourceObject.Load(3, &d.buffer) + stateSourceObject.Load(4, &d.closed) + stateSourceObject.Load(5, &d.queue) + stateSourceObject.Load(6, &d.tokens) + stateSourceObject.Load(7, &d.timeCheckpoint) +} + +func init() { + state.Register((*discipline)(nil)) +} diff --git a/pkg/tcpip/link/sharedmem/endpoint_mutex.go b/pkg/tcpip/link/sharedmem/endpoint_mutex.go index e970f75..264dfac 100644 --- a/pkg/tcpip/link/sharedmem/endpoint_mutex.go +++ b/pkg/tcpip/link/sharedmem/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go b/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go index e892e5c..ec9374f 100644 --- a/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go +++ b/pkg/tcpip/link/sharedmem/server_endpoint_mutex.go @@ -92,5 +92,5 @@ func serverEndpointinitLockNames() {} func init() { serverEndpointinitLockNames() - serverEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(serverEndpointRWMutex{}), serverEndpointlockNames) + serverEndpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[serverEndpointRWMutex](), serverEndpointlockNames) } diff --git a/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go b/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go index 3abfdae..88edd6f 100644 --- a/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go +++ b/pkg/tcpip/link/sharedmem/sharedmem_unsafe.go @@ -16,7 +16,6 @@ package sharedmem import ( "fmt" - "reflect" "unsafe" "github.com/sagernet/gvisor/pkg/atomicbitops" @@ -48,12 +47,7 @@ func getBuffer(fd int) ([]byte, error) { return nil, fmt.Errorf("failed to map memory for buffer fd: %d, error: %s", fd, err) } - // Use unsafe to convert addr into a []byte. - var b []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - hdr.Data = addr - hdr.Len = int(s.Size) - hdr.Cap = int(s.Size) + b := unsafe.Slice((*byte)(unsafe.Pointer(addr)), int(s.Size)) return b, nil } diff --git a/pkg/tcpip/link/sniffer/sniffer.go b/pkg/tcpip/link/sniffer/sniffer.go index 3ad3c9c..685edd5 100644 --- a/pkg/tcpip/link/sniffer/sniffer.go +++ b/pkg/tcpip/link/sniffer/sniffer.go @@ -353,8 +353,13 @@ func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumbe details += fmt.Sprintf("invalid packet: tcp data offset too small %d", offset) break } - if size := clone.Data().Size() + len(tcp); offset > size && !moreFragments { - details += fmt.Sprintf("invalid packet: tcp data offset %d larger than tcp packet length %d", offset, size) + + if size := clone.Data().Size() + len(tcp); offset > size { + if !moreFragments { + details += fmt.Sprintf("invalid packet: tcp data offset %d larger than tcp packet length %d", offset, size) + } else { + details += fmt.Sprintf("truncated options (tcp data offset %d, tcp packet length %d)", offset, size) + } break } diff --git a/pkg/tcpip/link/tun/device.go b/pkg/tcpip/link/tun/device.go index f94b6e6..b80ac5e 100644 --- a/pkg/tcpip/link/tun/device.go +++ b/pkg/tcpip/link/tun/device.go @@ -17,7 +17,6 @@ package tun import ( "fmt" - "github.com/sagernet/gvisor/pkg/atomicbitops" "github.com/sagernet/gvisor/pkg/buffer" "github.com/sagernet/gvisor/pkg/context" "github.com/sagernet/gvisor/pkg/errors/linuxerr" @@ -258,10 +257,15 @@ func (d *Device) Write(data *buffer.View) (int64, error) { case d.flags.TUN: // TUN interface with IFF_NO_PI enabled, thus // we need to determine protocol from version field + if data.Size() == 0 { + // Ignore bad packet. + return dataLen, nil + } version := data.AsSlice()[0] >> 4 - if version == 4 { + switch version { + case 4: protocol = header.IPv4ProtocolNumber - } else if version == 6 { + case 6: protocol = header.IPv6ProtocolNumber } } @@ -362,22 +366,26 @@ type tunEndpoint struct { tunEndpointRefs *channel.Endpoint - stack *stack.Stack - nicID tcpip.NICID - name string - isTap bool - persistent atomicbitops.Bool - closed atomicbitops.Bool + stack *stack.Stack + nicID tcpip.NICID + name string + isTap bool mu endpointMutex `state:"nosave"` onCloseAction func() `state:"nosave"` + persistent bool + closed bool } func (e *tunEndpoint) setPersistent(v bool) { - old := e.persistent.Swap(v) - if old == v { + e.mu.Lock() + if e.persistent == v || e.closed { + e.mu.Unlock() return } + e.persistent = v + e.mu.Unlock() + // Update refs without holding the lock. if v { e.IncRef() } else { @@ -386,17 +394,19 @@ func (e *tunEndpoint) setPersistent(v bool) { } func (e *tunEndpoint) Close() { - if e.closed.Swap(true) { + e.mu.Lock() + if e.closed { + e.mu.Unlock() return } - - if e.persistent.Load() { - e.DecRef(context.Background()) - } - e.mu.Lock() + e.closed = true + decref := e.persistent action := e.onCloseAction e.onCloseAction = nil e.mu.Unlock() + if decref { + e.DecRef(context.Background()) + } if action != nil { action() } diff --git a/pkg/tcpip/link/tun/device_mutex.go b/pkg/tcpip/link/tun/device_mutex.go index 8381889..5ed0930 100644 --- a/pkg/tcpip/link/tun/device_mutex.go +++ b/pkg/tcpip/link/tun/device_mutex.go @@ -92,5 +92,5 @@ func deviceinitLockNames() {} func init() { deviceinitLockNames() - deviceprefixIndex = locking.NewMutexClass(reflect.TypeOf(deviceRWMutex{}), devicelockNames) + deviceprefixIndex = locking.NewMutexClass(reflect.TypeFor[deviceRWMutex](), devicelockNames) } diff --git a/pkg/tcpip/link/tun/endpoint_mutex.go b/pkg/tcpip/link/tun/endpoint_mutex.go index a57d0e1..80d7902 100644 --- a/pkg/tcpip/link/tun/endpoint_mutex.go +++ b/pkg/tcpip/link/tun/endpoint_mutex.go @@ -60,5 +60,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/veth/endpoint_mutex.go b/pkg/tcpip/link/veth/endpoint_mutex.go index d5dbcd4..b7388a8 100644 --- a/pkg/tcpip/link/veth/endpoint_mutex.go +++ b/pkg/tcpip/link/veth/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/veth/veth.go b/pkg/tcpip/link/veth/veth.go index cf6a6ce..45c8fe9 100644 --- a/pkg/tcpip/link/veth/veth.go +++ b/pkg/tcpip/link/veth/veth.go @@ -163,8 +163,7 @@ func (e *Endpoint) SetMTU(mtu uint32) { // Capabilities implements stack.LinkEndpoint.Capabilities. func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities { - // TODO(b/352384218): Enable CapabilityTXChecksumOffload. - return stack.CapabilityRXChecksumOffload | stack.CapabilitySaveRestore + return stack.CapabilityRXChecksumOffload | stack.CapabilitySaveRestore | stack.CapabilityTXChecksumOffload } // GSOMaxSize implements stack.GSOEndpoint. diff --git a/pkg/tcpip/link/veth/veth_mutex.go b/pkg/tcpip/link/veth/veth_mutex.go index 2f420df..08bc3fd 100644 --- a/pkg/tcpip/link/veth/veth_mutex.go +++ b/pkg/tcpip/link/veth/veth_mutex.go @@ -92,5 +92,5 @@ func vethinitLockNames() {} func init() { vethinitLockNames() - vethprefixIndex = locking.NewMutexClass(reflect.TypeOf(vethRWMutex{}), vethlockNames) + vethprefixIndex = locking.NewMutexClass(reflect.TypeFor[vethRWMutex](), vethlockNames) } diff --git a/pkg/tcpip/link/waitable/endpoint_mutex.go b/pkg/tcpip/link/waitable/endpoint_mutex.go index b671008..e1f136f 100644 --- a/pkg/tcpip/link/waitable/endpoint_mutex.go +++ b/pkg/tcpip/link/waitable/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/link/xdp/endpoint.go b/pkg/tcpip/link/xdp/endpoint.go index 6990dce..48d282e 100644 --- a/pkg/tcpip/link/xdp/endpoint.go +++ b/pkg/tcpip/link/xdp/endpoint.go @@ -88,10 +88,6 @@ type Options struct { // include CapabilitySaveRestore SaveRestore bool - // DisconnectOk if true, indicates that this NIC capability set should - // include CapabilityDisconnectOk. - DisconnectOk bool - // TXChecksumOffload if true, indicates that this endpoints capability // set should include CapabilityTXChecksumOffload. TXChecksumOffload bool @@ -109,6 +105,9 @@ type Options struct { // GRO enables generic receive offload. GRO bool + + // QueueID is the ID of the RX queue to which the AF_XDP socket is attached. + QueueID uint32 } // New creates a new endpoint from an AF_XDP socket. @@ -126,10 +125,6 @@ func New(opts *Options) (stack.LinkEndpoint, error) { caps |= stack.CapabilitySaveRestore } - if opts.DisconnectOk { - caps |= stack.CapabilityDisconnectOk - } - if err := unix.SetNonblock(opts.FD, true); err != nil { return nil, fmt.Errorf("unix.SetNonblock(%v) failed: %v", opts.FD, err) } @@ -164,7 +159,7 @@ func New(opts *Options) (stack.LinkEndpoint, error) { NDescriptors: nFrames / 2, Bind: opts.Bind, } - ep.control, err = xdp.NewFromSocket(opts.FD, uint32(opts.InterfaceIndex), 0 /* queueID */, xdpOpts) + ep.control, err = xdp.NewFromSocket(opts.FD, uint32(opts.InterfaceIndex), opts.QueueID, xdpOpts) if err != nil { return nil, fmt.Errorf("failed to create AF_XDP dispatcher: %v", err) } diff --git a/pkg/tcpip/link/xdp/endpoint_mutex.go b/pkg/tcpip/link/xdp/endpoint_mutex.go index c527707..0dc93f5 100644 --- a/pkg/tcpip/link/xdp/endpoint_mutex.go +++ b/pkg/tcpip/link/xdp/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/network/arp/arp.go b/pkg/tcpip/network/arp/arp.go index 13b7479..562cbf5 100644 --- a/pkg/tcpip/network/arp/arp.go +++ b/pkg/tcpip/network/arp/arp.go @@ -130,6 +130,11 @@ func (e *endpoint) MTU() uint32 { return lmtu - uint32(e.MaxHeaderLength()) } +// EndpointHeaderSize returns the size necessary for the ARP header. +func (e *endpoint) EndpointHeaderSize() uint32 { + return header.ARPSize +} + func (e *endpoint) MaxHeaderLength() uint16 { return e.nic.MaxHeaderLength() + header.ARPSize } diff --git a/pkg/tcpip/network/internal/fragmentation/fragmentation.go b/pkg/tcpip/network/internal/fragmentation/fragmentation.go index a53603c..cab4019 100644 --- a/pkg/tcpip/network/internal/fragmentation/fragmentation.go +++ b/pkg/tcpip/network/internal/fragmentation/fragmentation.go @@ -302,6 +302,7 @@ type PacketFragmenter struct { fragmentCount int currentFragment int fragmentOffset int + mark uint32 } // MakePacketFragmenter prepares the struct needed for packet fragmentation. @@ -332,6 +333,7 @@ func MakePacketFragmenter(pkt *stack.PacketBuffer, fragmentPayloadLen uint32, re reserve: reserve, fragmentPayloadLen: int(fragmentPayloadLen), fragmentCount: int(fragmentCount), + mark: pkt.Mark, } } @@ -351,6 +353,7 @@ func (pf *PacketFragmenter) BuildNextFragment() (*stack.PacketBuffer, int, int, fragPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ ReserveHeaderBytes: pf.reserve, + Mark: pf.mark, }) // Copy data for the fragment. diff --git a/pkg/tcpip/network/internal/fragmentation/reassembler.go b/pkg/tcpip/network/internal/fragmentation/reassembler.go index e490487..6e6bf93 100644 --- a/pkg/tcpip/network/internal/fragmentation/reassembler.go +++ b/pkg/tcpip/network/internal/fragmentation/reassembler.go @@ -103,8 +103,21 @@ func (r *reassembler) process(first, last uint16, more bool, proto uint8, pkt *s } holeFound = true + // IPv6: rfc8200#section-4.5 + // Changed the text to require that IPv6 nodes must not create + // overlapping fragments. Also, when reassembling an IPv6 + // datagram, if one or more its constituent fragments is + // determined to be an overlapping fragment, the entire datagram + // (and any constituent fragments) must be silently discarded. + // Includes a clarification that no ICMP error message should be + // sent if overlapping fragments are received. if currentHole.filled { + // Incoming fragment is a subset of an existing fragment. + if first != currentHole.first || last != currentHole.last { + return nil, 0, false, 0, ErrFragmentOverlap + } // Incoming fragment is a duplicate. + // Not dropping packet incase of duplicates. continue } diff --git a/pkg/tcpip/network/internal/ip/duplicate_address_detection.go b/pkg/tcpip/network/internal/ip/duplicate_address_detection.go index 6be50f2..f275013 100644 --- a/pkg/tcpip/network/internal/ip/duplicate_address_detection.go +++ b/pkg/tcpip/network/internal/ip/duplicate_address_detection.go @@ -39,7 +39,7 @@ type dadState struct { extendRequest extendRequest done *bool - timer tcpip.Timer + timer tcpip.Timer `state:"nosave"` completionHandlers []stack.DADCompletionHandler } diff --git a/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go b/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go index 1eaeee7..c44a2bf 100644 --- a/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go +++ b/pkg/tcpip/network/internal/ip/generic_multicast_protocol.go @@ -291,13 +291,13 @@ type GenericMulticastProtocolState struct { robustnessVariable uint8 queryInterval time.Duration mode protocolMode - modeTimer tcpip.Timer + modeTimer tcpip.Timer `state:"nosave"` - generalQueryV2Timer tcpip.Timer + generalQueryV2Timer tcpip.Timer `state:"nosave"` // TODO(b/341946753): Restore when netstack is savable. generalQueryV2TimerFiresAt time.Time `state:"nosave"` - stateChangedReportV2Timer tcpip.Timer + stateChangedReportV2Timer tcpip.Timer `state:"nosave"` stateChangedReportV2TimerSet bool } diff --git a/pkg/tcpip/network/internal/ip/ip_state_autogen.go b/pkg/tcpip/network/internal/ip/ip_state_autogen.go index 2aaa1cc..a58ed22 100644 --- a/pkg/tcpip/network/internal/ip/ip_state_autogen.go +++ b/pkg/tcpip/network/internal/ip/ip_state_autogen.go @@ -17,7 +17,6 @@ func (d *dadState) StateFields() []string { "nonce", "extendRequest", "done", - "timer", "completionHandlers", } } @@ -30,8 +29,7 @@ func (d *dadState) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(0, &d.nonce) stateSinkObject.Save(1, &d.extendRequest) stateSinkObject.Save(2, &d.done) - stateSinkObject.Save(3, &d.timer) - stateSinkObject.Save(4, &d.completionHandlers) + stateSinkObject.Save(3, &d.completionHandlers) } func (d *dadState) afterLoad(context.Context) {} @@ -41,8 +39,7 @@ func (d *dadState) StateLoad(ctx context.Context, stateSourceObject state.Source stateSourceObject.Load(0, &d.nonce) stateSourceObject.Load(1, &d.extendRequest) stateSourceObject.Load(2, &d.done) - stateSourceObject.Load(3, &d.timer) - stateSourceObject.Load(4, &d.completionHandlers) + stateSourceObject.Load(3, &d.completionHandlers) } func (d *DADOptions) StateTypeName() string { @@ -237,9 +234,6 @@ func (g *GenericMulticastProtocolState) StateFields() []string { "robustnessVariable", "queryInterval", "mode", - "modeTimer", - "generalQueryV2Timer", - "stateChangedReportV2Timer", "stateChangedReportV2TimerSet", } } @@ -254,10 +248,7 @@ func (g *GenericMulticastProtocolState) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(2, &g.robustnessVariable) stateSinkObject.Save(3, &g.queryInterval) stateSinkObject.Save(4, &g.mode) - stateSinkObject.Save(5, &g.modeTimer) - stateSinkObject.Save(6, &g.generalQueryV2Timer) - stateSinkObject.Save(7, &g.stateChangedReportV2Timer) - stateSinkObject.Save(8, &g.stateChangedReportV2TimerSet) + stateSinkObject.Save(5, &g.stateChangedReportV2TimerSet) } func (g *GenericMulticastProtocolState) afterLoad(context.Context) {} @@ -269,10 +260,7 @@ func (g *GenericMulticastProtocolState) StateLoad(ctx context.Context, stateSour stateSourceObject.Load(2, &g.robustnessVariable) stateSourceObject.Load(3, &g.queryInterval) stateSourceObject.Load(4, &g.mode) - stateSourceObject.Load(5, &g.modeTimer) - stateSourceObject.Load(6, &g.generalQueryV2Timer) - stateSourceObject.Load(7, &g.stateChangedReportV2Timer) - stateSourceObject.Load(8, &g.stateChangedReportV2TimerSet) + stateSourceObject.Load(5, &g.stateChangedReportV2TimerSet) } func (m *MultiCounterIPForwardingStats) StateTypeName() string { diff --git a/pkg/tcpip/network/internal/ip/reject_with_reset.go b/pkg/tcpip/network/internal/ip/reject_with_reset.go new file mode 100644 index 0000000..218ed10 --- /dev/null +++ b/pkg/tcpip/network/internal/ip/reject_with_reset.go @@ -0,0 +1,293 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ip + +import ( + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/stack" +) + +// ipv6FragmentOffset returns the fragment offset of the IPv6 packet +// if present. +func ipv6FragmentOffset(pkt *stack.PacketBuffer, ipHdr header.IPv6) (uint16, bool) { + if !header.IsExtensionHeader(ipHdr.NextHeader()) { + return 0, false + } + + netHeaderSlice := pkt.NetworkHeader().Slice() + if len(netHeaderSlice) <= header.IPv6MinimumSize { + return 0, false + } + + // Make an iterator to walk the extension headers. + buf := buffer.MakeWithData(netHeaderSlice[header.IPv6MinimumSize:]) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(ipHdr.NextHeader()), buf) + defer it.Release() + + for { + extHdr, done, err := it.Next() + if err != nil || done { + break + } + switch extHdr := extHdr.(type) { + case header.IPv6FragmentExtHdr: + offset := extHdr.FragmentOffset() + extHdr.Release() + return offset, true + default: + extHdr.Release() + } + } + + return 0, false +} + +// buildResetPayloadV4 builds an IPv4 + TCP Reset packet in a buffer. +func buildResetPayloadV4(ttl uint8, src, dst tcpip.Address, tcpHdr header.TCP, seq, ack uint32, flags header.TCPFlags) *buffer.View { + totalHdrLen := header.IPv4MinimumSize + header.TCPMinimumSize + v := buffer.NewViewSize(totalHdrLen) + buf := v.AsSlice() + + rstIPHdr := header.IPv4(buf[:header.IPv4MinimumSize]) + rstIPHdr.Encode(&header.IPv4Fields{ + TotalLength: uint16(totalHdrLen), + TTL: ttl, + Protocol: uint8(header.TCPProtocolNumber), + TOS: stack.DefaultTOS, + Flags: header.IPv4FlagDontFragment, + // Flip source and destination addresses. + SrcAddr: dst, + DstAddr: src, + }) + + rstTCPHdr := header.TCP(buf[header.IPv4MinimumSize:]) + rstTCPHdr.Encode(&header.TCPFields{ + SrcPort: tcpHdr.DestinationPort(), + DstPort: tcpHdr.SourcePort(), + SeqNum: seq, + AckNum: ack, + DataOffset: header.TCPMinimumSize, + Flags: flags, + }) + + xsum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, dst, src, header.TCPMinimumSize) + rstTCPHdr.SetChecksum(0) + rstTCPHdr.SetChecksum(^rstTCPHdr.CalculateChecksum(xsum)) + + return v +} + +// buildResetPayloadV6 builds an IPv6 + TCP Reset packet in a buffer. +func buildResetPayloadV6(hopLimit uint8, src, dst tcpip.Address, tcpHdr header.TCP, seq, ack uint32, flags header.TCPFlags) *buffer.View { + totalHdrLen := header.IPv6MinimumSize + header.TCPMinimumSize + v := buffer.NewViewSize(totalHdrLen) + buf := v.AsSlice() + + rstIPHdr := header.IPv6(buf[:header.IPv6MinimumSize]) + rstIPHdr.Encode(&header.IPv6Fields{ + PayloadLength: uint16(header.TCPMinimumSize), + TransportProtocol: header.TCPProtocolNumber, + HopLimit: hopLimit, + // Flip source and destination addresses. + SrcAddr: dst, + DstAddr: src, + }) + + rstTCPHdr := header.TCP(buf[header.IPv6MinimumSize:]) + rstTCPHdr.Encode(&header.TCPFields{ + SrcPort: tcpHdr.DestinationPort(), + DstPort: tcpHdr.SourcePort(), + SeqNum: seq, + AckNum: ack, + DataOffset: header.TCPMinimumSize, + Flags: flags, + }) + + // Compute TCP checksum. + xsum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, dst, src, header.TCPMinimumSize) + rstTCPHdr.SetChecksum(0) + rstTCPHdr.SetChecksum(^rstTCPHdr.CalculateChecksum(xsum)) + + return v +} + +// RejectWithTCPReset sends a TCP reset in response to the packet. +// +// Ref: net/ipv[4|6]/netfilter/nf_reject_ipv[4|6].c:nf_send_reset[6]() +func RejectWithTCPReset(pkt *stack.PacketBuffer, netProto tcpip.NetworkProtocolNumber, stk *stack.Stack, deliveredLocally bool) tcpip.Error { + var src, dst tcpip.Address + var ttl uint8 + isFragment := false + + switch netProto { + case header.IPv4ProtocolNumber: + // Ref: net/ipv4/netfilter/nf_reject_ipv4.c:nf_reject_ip_tcphdr_get + ipHdr := header.IPv4(pkt.NetworkHeader().Slice()) + if len(ipHdr) < header.IPv4MinimumSize { + return nil + } + if ipHdr.Protocol() != uint8(header.TCPProtocolNumber) { + return nil + } + if ipHdr.FragmentOffset() != 0 { + return nil + } + isFragment = ipHdr.More() + src = ipHdr.SourceAddress() + dst = ipHdr.DestinationAddress() + + // Ref: net/ipv4/netfilter/nf_reject_ipv4.c:nf_send_reset + if header.IsV4MulticastAddress(dst) || + header.IsV4MulticastAddress(src) || + pkt.NetworkPacketInfo.LocalAddressBroadcast || + pkt.PktType == tcpip.PacketBroadcast || pkt.PktType == tcpip.PacketMulticast || + src == header.IPv4Any || dst == header.IPv4Any { + return nil + } + + case header.IPv6ProtocolNumber: + // Ref: net/ipv6/netfilter/nf_reject_ipv6.c:nf_reject_ip6_tcphdr_get + ipHdr := header.IPv6(pkt.NetworkHeader().Slice()) + if len(ipHdr) < header.IPv6MinimumSize { + return nil + } + fragOffset, ok := ipv6FragmentOffset(pkt, ipHdr) + if ok && fragOffset != 0 { + return nil + } + isFragment = ok + src = ipHdr.SourceAddress() + dst = ipHdr.DestinationAddress() + + // Ref: net/ipv6/netfilter/nf_reject_ipv6.c:nf_send_reset6 + if header.IsV6MulticastAddress(src) || header.IsV6MulticastAddress(dst) || + header.IsV4MappedAddress(src) || header.IsV4MappedAddress(dst) || + src == header.IPv6Any || dst == header.IPv6Any || + pkt.PktType == tcpip.PacketBroadcast || pkt.PktType == tcpip.PacketMulticast { + return nil + } + + default: + return nil + } + + tcpHdr := func(pkt *stack.PacketBuffer) header.TCP { + // If 0 < len(transportHdr) < header.TCPMinimumSize, then the TCP header is invalid. + // Assuming a TCP packet, + // if the TCP header was parsed, the + // len should be >= header.TCPMinimumSize; + // else the TCP header was not parsed and the len should be 0. + transportHdr := pkt.TransportHeader().Slice() + if len(transportHdr) >= header.TCPMinimumSize { + return header.TCP(transportHdr) + } + if len(transportHdr) != 0 { + return nil + } + // In the case of fragmented TCP packets, the TCP header may not be parsed. + // Pull up the TCP header from the payload. + b, ok := pkt.Data().PullUp(header.TCPMinimumSize) + if !ok { + return nil + } + hdr := header.TCP(b) + hdrLen := int(hdr.DataOffset()) + if hdrLen < header.TCPMinimumSize || pkt.Data().Size() < hdrLen { + return nil + } + tcpHdr, ok := pkt.Data().Consume(hdrLen) + if !ok { + return nil + } + pkt.TransportProtocolNumber = header.TCPProtocolNumber + return header.TCP(tcpHdr) + }(pkt) + if tcpHdr == nil { + return nil + } + + // Ref: net/ipv[4|6]/netfilter/nf_reject_ipv[4|6].c:nf_reject_ip[6]_tcphdr_get() + // No RST for RST as this will cause a loop. + if tcpHdr.Flags().Contains(header.TCPFlagRst) { + return nil + } + + // Check checksum integrity only for non-fragmented packets. + // We don't support refragmentation(nf_defrag) before REJECT, + // so we can't validate the checksum for fragmented packets. + if !isFragment { + // Check checksum integrity. + if !pkt.RXChecksumValidated && !tcpHdr.IsChecksumValid(src, dst, pkt.Data().Checksum(), uint16(pkt.Data().Size())) { + return nil + } + } + + localAddr := dst + if !deliveredLocally { + // If the packet wasn't delivered locally, do not use the packet's destination + // address as the response's source address as we should not own the + // destination address. + localAddr = tcpip.Address{} + } + + route, err := stk.FindRoute(0 /*nicID*/, localAddr, src, netProto, false /* multicastLoop */) + if err != nil { + return err + } + defer route.Release() + ttl = route.DefaultTTL() + + var seq uint32 + var ack uint32 + payloadLen := uint32(pkt.Data().Size()) + flags := header.TCPFlagRst + + // Ref: net/ipv[4|6]/netfilter/nf_reject_ipv[4|6].c:nf_reject_ip[6]_tcphdr_put() + if tcpHdr.Flags()&header.TCPFlagAck != 0 { + seq = tcpHdr.AckNumber() + } else { + flags |= header.TCPFlagAck + ack = tcpHdr.SequenceNumber() + payloadLen + if tcpHdr.Flags()&header.TCPFlagSyn != 0 { + ack++ + } + if tcpHdr.Flags()&header.TCPFlagFin != 0 { + ack++ + } + } + + var v *buffer.View + if netProto == header.IPv4ProtocolNumber { + v = buildResetPayloadV4(ttl, src, dst, tcpHdr, seq, ack, flags) + } else { + v = buildResetPayloadV6(ttl, src, dst, tcpHdr, seq, ack, flags) + } + + rstPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(route.MaxHeaderLength()), + Payload: buffer.MakeWithView(v), + }) + rstPkt.TransportProtocolNumber = header.TCPProtocolNumber + defer rstPkt.DecRef() + + // TODO: b/521536712 - Add support for mark propagation. + if err := route.WriteHeaderIncludedPacket(rstPkt); err != nil { + return err + } + + return nil +} diff --git a/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go b/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go index 1af40ce..1492b8a 100644 --- a/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go +++ b/pkg/tcpip/network/internal/multicast/multicast_state_autogen.go @@ -16,7 +16,6 @@ func (r *RouteTable) StateFields() []string { return []string{ "installedRoutes", "pendingRoutes", - "cleanupPendingRoutesTimer", "isCleanupRoutineRunning", "config", } @@ -29,9 +28,8 @@ func (r *RouteTable) StateSave(stateSinkObject state.Sink) { r.beforeSave() stateSinkObject.Save(0, &r.installedRoutes) stateSinkObject.Save(1, &r.pendingRoutes) - stateSinkObject.Save(2, &r.cleanupPendingRoutesTimer) - stateSinkObject.Save(3, &r.isCleanupRoutineRunning) - stateSinkObject.Save(4, &r.config) + stateSinkObject.Save(2, &r.isCleanupRoutineRunning) + stateSinkObject.Save(3, &r.config) } func (r *RouteTable) afterLoad(context.Context) {} @@ -40,9 +38,8 @@ func (r *RouteTable) afterLoad(context.Context) {} func (r *RouteTable) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &r.installedRoutes) stateSourceObject.Load(1, &r.pendingRoutes) - stateSourceObject.Load(2, &r.cleanupPendingRoutesTimer) - stateSourceObject.Load(3, &r.isCleanupRoutineRunning) - stateSourceObject.Load(4, &r.config) + stateSourceObject.Load(2, &r.isCleanupRoutineRunning) + stateSourceObject.Load(3, &r.config) } func (r *InstalledRoute) StateTypeName() string { diff --git a/pkg/tcpip/network/internal/multicast/route_table.go b/pkg/tcpip/network/internal/multicast/route_table.go index 490e83d..5bfb417 100644 --- a/pkg/tcpip/network/internal/multicast/route_table.go +++ b/pkg/tcpip/network/internal/multicast/route_table.go @@ -57,7 +57,7 @@ type RouteTable struct { // cleanupPendingRoutesTimer is a timer that triggers a routine to remove // pending routes that are expired. // +checklocks:pendingMu - cleanupPendingRoutesTimer tcpip.Timer + cleanupPendingRoutesTimer tcpip.Timer `state:"nosave"` // +checklocks:pendingMu isCleanupRoutineRunning bool diff --git a/pkg/tcpip/network/ipv4/icmp.go b/pkg/tcpip/network/ipv4/icmp.go index 5644c14..822848e 100644 --- a/pkg/tcpip/network/ipv4/icmp.go +++ b/pkg/tcpip/network/ipv4/icmp.go @@ -16,11 +16,13 @@ package ipv4 import ( "fmt" + "math" "github.com/sagernet/gvisor/pkg/buffer" "github.com/sagernet/gvisor/pkg/tcpip" "github.com/sagernet/gvisor/pkg/tcpip/checksum" "github.com/sagernet/gvisor/pkg/tcpip/header" + "github.com/sagernet/gvisor/pkg/tcpip/header/parse" "github.com/sagernet/gvisor/pkg/tcpip/stack" ) @@ -343,10 +345,47 @@ func (e *endpoint) handleICMP(pkt *stack.PacketBuffer) { switch h.Type() { case header.ICMPv4Echo: received.echoRequest.Increment() - e.dispatcher.DeliverTransportPacket(header.ICMPv4ProtocolNumber, pkt) + + // DeliverTransportPacket may modify pkt so don't use it beyond + // this point. Make a deep copy of the data before pkt gets sent as we will + // be modifying fields. Both the ICMP header (with its type modified to + // EchoReply) and payload are reused in the reply packet. + // + // TODO(gvisor.dev/issue/4399): The copy may not be needed if there are no + // waiting endpoints. Consider moving responsibility for doing the copy to + // DeliverTransportPacket so that is is only done when needed. + replyData := stack.PayloadSince(pkt.TransportHeader()) + defer replyData.Release() + localAddressTemporary := pkt.NetworkPacketInfo.LocalAddressTemporary + localAddressBroadcast := pkt.NetworkPacketInfo.LocalAddressBroadcast + + // It's possible that a raw socket or per-stack default handler expects + // to receive this packet. + defaultHandlerHandled := false + if dispatcher, ok := e.dispatcher.(stack.TransportDispatcherWithDefaultHandlerResult); ok { + _, defaultHandlerHandled = dispatcher.DeliverTransportPacketWithDefaultHandlerResult(header.ICMPv4ProtocolNumber, pkt) + } else { + e.dispatcher.DeliverTransportPacket(header.ICMPv4ProtocolNumber, pkt) + } + pkt = nil + + // Skip the built-in ICMP echo reply if the request was consumed by a + // per-stack default handler. Also preserve the IPv4 behavior for + // temporary local addresses: the packet is delivered above, but the + // stack does not synthesize an echo reply for it. + if defaultHandlerHandled || localAddressTemporary { + return + } + + e.sendICMPEchoReply(replyData, iph, newOptions, localAddressBroadcast) + case header.ICMPv4EchoReply: received.echoReply.Increment() + + // ICMP sockets expect the ICMP header to be present, so we don't consume + // the ICMP header. e.dispatcher.DeliverTransportPacket(header.ICMPv4ProtocolNumber, pkt) + case header.ICMPv4DstUnreachable: received.dstUnreachable.Increment() @@ -411,6 +450,98 @@ func (e *endpoint) handleICMP(pkt *stack.PacketBuffer) { } } +func (e *endpoint) sendICMPEchoReply(replyData *buffer.View, ipHdr header.IPv4, newOptions header.IPv4Options, localAddressBroadcast bool) { + sent := e.stats.icmp.packetsSent + if !e.protocol.allowICMPReply(header.ICMPv4EchoReply, header.ICMPv4UnusedCode) { + sent.rateLimited.Increment() + return + } + + // As per RFC 1122 section 3.2.1.3, when a host sends any datagram, the IP + // source address MUST be one of its own IP addresses (but not a broadcast + // or multicast address). + localAddr := ipHdr.DestinationAddress() + if localAddressBroadcast || header.IsV4MulticastAddress(localAddr) { + localAddr = tcpip.Address{} + } + + r, err := e.protocol.stack.FindRoute(e.nic.ID(), localAddr, ipHdr.SourceAddress(), ProtocolNumber, false /* multicastLoop */) + if err != nil { + // If we cannot find a route to the destination, silently drop the packet. + return + } + defer r.Release() + + outgoingEP, ok := e.protocol.getEndpointForNIC(r.NICID()) + if !ok { + // The outgoing NIC went away. + sent.dropped.Increment() + return + } + + // Because IP and ICMP are so closely intertwined, we need to handcraft our + // IP header to be able to follow RFC 792. The wording on page 13 is as + // follows: + // IP Fields: + // Addresses + // The address of the source in an echo message will be the + // destination of the echo reply message. To form an echo reply + // message, the source and destination addresses are simply reversed, + // the type code changed to 0, and the checksum recomputed. + // + // This was interpreted by early implementors to mean that all options must + // be copied from the echo request IP header to the echo reply IP header + // and this behaviour is still relied upon by some applications. + // + // Create a copy of the IP header we received, options and all, and change + // The fields we need to alter. + // + // We need to produce the entire packet in the data segment in order to + // use WriteHeaderIncludedPacket(). WriteHeaderIncludedPacket sets the + // total length and the header checksum so we don't need to set those here. + // + // Take the base of the incoming request IP header but replace the options. + replyHeaderLength := uint8(header.IPv4MinimumSize + len(newOptions)) + replyIPHdrView := buffer.NewView(int(replyHeaderLength)) + replyIPHdrView.Write(ipHdr[:header.IPv4MinimumSize]) + replyIPHdrView.Write(newOptions) + replyIPHdr := header.IPv4(replyIPHdrView.AsSlice()) + replyIPHdr.SetHeaderLength(replyHeaderLength) + replyIPHdr.SetSourceAddress(r.LocalAddress()) + replyIPHdr.SetDestinationAddress(r.RemoteAddress()) + replyIPHdr.SetTTL(r.DefaultTTL()) + replyIPHdr.SetTotalLength(uint16(len(replyIPHdr) + len(replyData.AsSlice()))) + replyIPHdr.SetChecksum(0) + replyIPHdr.SetChecksum(^replyIPHdr.CalculateChecksum()) + + replyICMPHdr := header.ICMPv4(replyData.AsSlice()) + replyICMPHdr.SetType(header.ICMPv4EchoReply) + replyICMPHdr.SetChecksum(0) + replyICMPHdr.SetChecksum(^checksum.Checksum(replyData.AsSlice(), 0)) + + replyBuf := buffer.MakeWithView(replyIPHdrView) + replyBuf.Append(replyData.Clone()) + replyPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(r.MaxHeaderLength()), + Payload: replyBuf, + }) + defer replyPkt.DecRef() + // Populate the network/transport headers in the packet buffer so the + // ICMP packet goes through IPTables. + if ok := parse.IPv4(replyPkt); !ok { + panic("expected to parse IPv4 header we just created") + } + if ok := parse.ICMPv4(replyPkt); !ok { + panic("expected to parse ICMPv4 header we just created") + } + + if err := outgoingEP.writePacket(r, replyPkt); err != nil { + sent.dropped.Increment() + return + } + sent.echoReply.Increment() +} + // ======= ICMP Error packet generation ========= // icmpReason is a marker interface for IPv4 specific ICMP errors. @@ -479,7 +610,12 @@ func (*icmpReasonNetworkUnreachable) isICMPReason() {} // icmpReasonFragmentationNeeded is an error where a packet requires // fragmentation while also having the Don't Fragment flag set, as per RFC 792 // page 3, Destination Unreachable Message. -type icmpReasonFragmentationNeeded struct{} +type icmpReasonFragmentationNeeded struct { + // mtu is the MTU of the next-hop link. Per RFC 1191 §4, this value + // must be included in the ICMP Fragmentation Needed message so the + // sender can update its path MTU cache. + mtu uint32 +} func (*icmpReasonFragmentationNeeded) isICMPReason() {} @@ -584,30 +720,36 @@ func (p *protocol) returnError(reason icmpReason, pkt *stack.PacketBuffer, deliv } sent := netEP.stats.icmp.packetsSent - icmpType, icmpCode, counter, pointer := func() (header.ICMPv4Type, header.ICMPv4Code, tcpip.MultiCounterStat, byte) { + icmpType, icmpCode, counter, pointer, nextHopMTU := func() (header.ICMPv4Type, header.ICMPv4Code, tcpip.MultiCounterStat, byte, uint16) { switch reason := reason.(type) { case *icmpReasonNetworkProhibited: - return header.ICMPv4DstUnreachable, header.ICMPv4NetProhibited, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4NetProhibited, sent.dstUnreachable, 0, 0 case *icmpReasonHostProhibited: - return header.ICMPv4DstUnreachable, header.ICMPv4HostProhibited, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4HostProhibited, sent.dstUnreachable, 0, 0 case *icmpReasonAdministrativelyProhibited: - return header.ICMPv4DstUnreachable, header.ICMPv4AdminProhibited, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4AdminProhibited, sent.dstUnreachable, 0, 0 case *icmpReasonPortUnreachable: - return header.ICMPv4DstUnreachable, header.ICMPv4PortUnreachable, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4PortUnreachable, sent.dstUnreachable, 0, 0 case *icmpReasonProtoUnreachable: - return header.ICMPv4DstUnreachable, header.ICMPv4ProtoUnreachable, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4ProtoUnreachable, sent.dstUnreachable, 0, 0 case *icmpReasonNetworkUnreachable: - return header.ICMPv4DstUnreachable, header.ICMPv4NetUnreachable, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4NetUnreachable, sent.dstUnreachable, 0, 0 case *icmpReasonHostUnreachable: - return header.ICMPv4DstUnreachable, header.ICMPv4HostUnreachable, sent.dstUnreachable, 0 + return header.ICMPv4DstUnreachable, header.ICMPv4HostUnreachable, sent.dstUnreachable, 0, 0 case *icmpReasonFragmentationNeeded: - return header.ICMPv4DstUnreachable, header.ICMPv4FragmentationNeeded, sent.dstUnreachable, 0 + // Per RFC 1191 §4, include the next-hop MTU in the ICMP message. + // Cap at MaxUint16 since the field is 16 bits wide. + mtu := reason.mtu + if mtu > math.MaxUint16 { + mtu = math.MaxUint16 + } + return header.ICMPv4DstUnreachable, header.ICMPv4FragmentationNeeded, sent.dstUnreachable, 0, uint16(mtu) case *icmpReasonTTLExceeded: - return header.ICMPv4TimeExceeded, header.ICMPv4TTLExceeded, sent.timeExceeded, 0 + return header.ICMPv4TimeExceeded, header.ICMPv4TTLExceeded, sent.timeExceeded, 0, 0 case *icmpReasonReassemblyTimeout: - return header.ICMPv4TimeExceeded, header.ICMPv4ReassemblyTimeout, sent.timeExceeded, 0 + return header.ICMPv4TimeExceeded, header.ICMPv4ReassemblyTimeout, sent.timeExceeded, 0, 0 case *icmpReasonParamProblem: - return header.ICMPv4ParamProblem, header.ICMPv4UnusedCode, sent.paramProblem, reason.pointer + return header.ICMPv4ParamProblem, header.ICMPv4UnusedCode, sent.paramProblem, reason.pointer, 0 default: panic(fmt.Sprintf("unsupported ICMP type %T", reason)) } @@ -676,6 +818,7 @@ func (p *protocol) returnError(reason icmpReason, pkt *stack.PacketBuffer, deliv icmpHdr.SetCode(icmpCode) icmpHdr.SetType(icmpType) icmpHdr.SetPointer(pointer) + icmpHdr.SetMTU(nextHopMTU) icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, icmpPkt.Data().Checksum())) if err := route.WritePacket( diff --git a/pkg/tcpip/network/ipv4/ipv4.go b/pkg/tcpip/network/ipv4/ipv4.go index 4a9402b..5cbdbd2 100644 --- a/pkg/tcpip/network/ipv4/ipv4.go +++ b/pkg/tcpip/network/ipv4/ipv4.go @@ -26,6 +26,7 @@ import ( "github.com/sagernet/gvisor/pkg/log" "github.com/sagernet/gvisor/pkg/sync" "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" "github.com/sagernet/gvisor/pkg/tcpip/header" "github.com/sagernet/gvisor/pkg/tcpip/header/parse" "github.com/sagernet/gvisor/pkg/tcpip/network/hash" @@ -84,6 +85,7 @@ var ( _ IGMPEndpoint = (*endpoint)(nil) ) +// +checklocksalias:igmp.ep.mu=mu // +stateify savable type endpoint struct { nic stack.NetworkInterface @@ -128,13 +130,11 @@ func (e *endpoint) GetIGMPVersion() IGMPVersion { } // +checklocks:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) setIGMPVersionLocked(v IGMPVersion) IGMPVersion { return e.igmp.setVersion(v) } // +checklocksread:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) getIGMPVersionLocked() IGMPVersion { return e.igmp.getVersion() } @@ -293,7 +293,6 @@ func (e *endpoint) Enable() tcpip.Error { } // +checklocks:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) enableLocked() tcpip.Error { // If the NIC is not enabled, the endpoint can't do anything meaningful so // don't enable the endpoint. @@ -364,7 +363,6 @@ func (e *endpoint) Disable() { } // +checklocks:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) disableLocked() { if !e.isEnabled() { return @@ -428,6 +426,11 @@ func (e *endpoint) MTU() uint32 { return networkMTU } +// EndpointHeaderSize returns the size necessary for the IPv4 header. +func (e *endpoint) EndpointHeaderSize() uint32 { + return header.IPv4MinimumSize +} + // MaxHeaderLength returns the maximum length needed by ipv4 headers (and // underlying protocols). func (e *endpoint) MaxHeaderLength() uint16 { @@ -519,6 +522,48 @@ func (e *endpoint) handleFragments(_ *stack.Route, networkMTU uint32, pkt *stack } } +// recalculateChecksum recalculates the checksum of a TCP packet. +func recalculateChecksum(pkt *stack.PacketBuffer, r *stack.Route) tcpip.Error { + // RXChecksumValidated indicates that checksum verification may be + // safely skipped. + if pkt.RXChecksumValidated { + return nil + } + // NeedsCsum is set if the checksum offload is enabled, so no need to + // calculate the checksum. + if pkt.GSOOptions.Type != stack.GSONone && pkt.GSOOptions.NeedsCsum { + return nil + } + transportHeader := pkt.TransportHeader().Slice() + netHdr := header.IPv4(pkt.NetworkHeader().Slice()) + switch pkt.TransportProtocolNumber { + case header.TCPProtocolNumber: + if len(transportHeader) < header.TCPMinimumSize { + return &tcpip.ErrMalformedHeader{} + } + tcp := header.TCP(transportHeader) + xsum := r.PseudoHeaderChecksum(header.TCPProtocolNumber, netHdr.PayloadLength()) + xsum = checksum.Combine(xsum, pkt.Data().Checksum()) + tcp.SetChecksum(0) + tcp.SetChecksum(^tcp.CalculateChecksum(xsum)) + case header.UDPProtocolNumber: + if len(transportHeader) < header.UDPMinimumSize { + return &tcpip.ErrMalformedHeader{} + } + udp := header.UDP(transportHeader) + xsum := r.PseudoHeaderChecksum(header.UDPProtocolNumber, netHdr.PayloadLength()) + xsum = checksum.Combine(xsum, pkt.Data().Checksum()) + udp.SetChecksum(0) + csum := ^udp.CalculateChecksum(xsum) + // RFC 768: If the computed checksum is zero, it is transmitted as all ones. + if csum == 0 { + csum = 0xFFFF + } + udp.SetChecksum(csum) + } + return nil +} + // WritePacket writes a packet to the given destination address and protocol. func (e *endpoint) WritePacket(r *stack.Route, params stack.NetworkHeaderParams, pkt *stack.PacketBuffer) tcpip.Error { if err := e.addIPHeader(r.LocalAddress(), r.RemoteAddress(), pkt, params, nil /* options */); err != nil { @@ -531,16 +576,23 @@ func (e *endpoint) WritePacket(r *stack.Route, params stack.NetworkHeaderParams, func (e *endpoint) writePacket(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Error { netHeader := header.IPv4(pkt.NetworkHeader().Slice()) dstAddr := netHeader.DestinationAddress() + stk := e.protocol.stack - // iptables filtering. All packets that reach here are locally - // generated. - outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) - if ok := e.protocol.stack.IPTables().CheckOutput(pkt, r, outNicName); !ok { + // iptables filtering. All packets that reach here are locally generated. + outNicName := stk.FindNICNameFromID(e.nic.ID()) + if ok := stk.IPTables().CheckOutput(pkt, r, outNicName); !ok { // iptables is telling us to drop the packet. e.stats.ip.IPTablesOutputDropped.Increment() return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckOutput(pkt, r, stack.IP) { + // nftables is telling us to drop the packet. + return nil + } + } + // If the packet is manipulated as per DNAT Output rules, handle packet // based on destination address and do not send the packet to link // layer. @@ -555,6 +607,37 @@ func (e *endpoint) writePacket(r *stack.Route, pkt *stack.PacketBuffer) tcpip.Er ep.handleLocalPacket(pkt, true /* canSkipRXChecksum */) return nil } + + // Similar to the `ip_route_me_harder` in the kernel, + // we need to find a new route for the packet. + // Implementation is similar to the func forwardUnicastPacket. + stk := e.protocol.stack + newRoute, err := stk.FindRoute(0 /* nic id */, netHeader.SourceAddress(), newDstAddr, header.IPv4ProtocolNumber, false /* multicastLoop */) + if err != nil { + return err // Drop the packet + } + // Release the new route on exit. + defer newRoute.Release() + + // Check if we need to recalculate the checksum. + // If the original route did not require a checksum but the new one does, + // we must calculate the full checksum; otherwise, NAT should have already + // done it. + if !r.RequiresTXTransportChecksum() && newRoute.RequiresTXTransportChecksum() { + if err := recalculateChecksum(pkt, newRoute); err != nil { + return err // Drop the packet + } + } + + // Update the route to the new route. + r = newRoute + + // Use the new endpoint to write the packet. + forwardToEp, ok := e.protocol.getEndpointForNIC(r.NICID()) + if !ok { + return &tcpip.ErrUnknownNICID{} + } + return forwardToEp.writePacketPostRouting(r, pkt, true /* headerIncluded */) } return e.writePacketPostRouting(r, pkt, false /* headerIncluded */) @@ -571,15 +654,23 @@ func (e *endpoint) writePacketPostRouting(r *stack.Route, pkt *stack.PacketBuffe return nil } + stk := e.protocol.stack // Postrouting NAT can only change the source address, and does not alter the // route or outgoing interface of the packet. - outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) - if ok := e.protocol.stack.IPTables().CheckPostrouting(pkt, r, e, outNicName); !ok { + outNicName := stk.FindNICNameFromID(e.nic.ID()) + if ok := stk.IPTables().CheckPostrouting(pkt, r, e, outNicName); !ok { // iptables is telling us to drop the packet. e.stats.ip.IPTablesPostroutingDropped.Increment() return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckPostrouting(pkt, r, stack.IP) { + // nftables is telling us to drop the packet. + return nil + } + } + stats := e.stats.ip networkMTU, err := calculateNetworkMTU(e.nic.MTU(), uint32(len(pkt.NetworkHeader().Slice()))) @@ -690,6 +781,13 @@ func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketB return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckForward(pkt, route, stack.IP) { + // nftables is telling us to drop the packet. + return nil + } + } + // We need to do a deep copy of the IP packet because // WriteHeaderIncludedPacket may modify the packet buffer, but we do // not own it. @@ -725,6 +823,10 @@ func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketB newHdr.SetChecksum(0) newHdr.SetChecksum(^newHdr.CalculateChecksum()) + if route.RequiresTXTransportChecksum() { + newPkt.CalculateTransportChecksum() + } + switch err := forwardToEp.writePacketPostRouting(route, newPkt, true /* headerIncluded */); err.(type) { case nil: return nil @@ -738,7 +840,9 @@ func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketB // WriteHeaderIncludedPacket checks for the presence of the Don't Fragment bit // while sending the packet and returns this error iff fragmentation is // necessary and the bit is also set. - _ = e.protocol.returnError(&icmpReasonFragmentationNeeded{}, pkt, false /* deliveredLocally */) + _ = e.protocol.returnError(&icmpReasonFragmentationNeeded{ + mtu: forwardToEp.nic.MTU(), + }, pkt, false /* deliveredLocally */) return &ip.ErrMessageTooLong{} case *tcpip.ErrNoBufferSpace: return &ip.ErrOutgoingDeviceNoBufferSpace{} @@ -790,6 +894,13 @@ func (e *endpoint) forwardUnicastPacket(pkt *stack.PacketBuffer) ip.ForwardingEr return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckForward(pkt, nil /* route */, stack.IP) { + // nftables is telling us to drop the packet. + return nil + } + } + // The packet originally arrived on e so provide its NIC as the input NIC. ep.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) return nil @@ -857,7 +968,8 @@ func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { } } - if e.protocol.stack.HandleLocal() { + stk := e.protocol.stack + if stk.HandleLocal() { addressEndpoint := e.AcquireAssignedAddress(header.IPv4(pkt.NetworkHeader().Slice()).SourceAddress(), e.nic.Promiscuous(), stack.CanBePrimaryEndpoint, true /* readOnly */) if addressEndpoint != nil { // The source address is one of our own, so we never should have gotten @@ -868,13 +980,22 @@ func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { } } + nicID := e.nic.ID() // Loopback traffic skips the prerouting chain. - inNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) - if ok := e.protocol.stack.IPTables().CheckPrerouting(pkt, e, inNicName); !ok { + inNicName := stk.FindNICNameFromID(nicID) + pkt.InputNICID = nicID + if ok := stk.IPTables().CheckPrerouting(pkt, e, inNicName); !ok { // iptables is telling us to drop the packet. stats.IPTablesPreroutingDropped.Increment() return } + + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckPrerouting(pkt, nil /* route */, stack.IP) { + // nftables is telling us to drop the packet. + return + } + } } // CheckPrerouting can modify the backing storage of the packet, so refresh // the header. @@ -1208,14 +1329,22 @@ func (e *endpoint) handleForwardingError(err ip.ForwardingError) { func (e *endpoint) deliverPacketLocally(h header.IPv4, pkt *stack.PacketBuffer, inNICName string) { stats := e.stats + stk := e.protocol.stack // iptables filtering. All packets that reach here are intended for // this machine and will not be forwarded. - if ok := e.protocol.stack.IPTables().CheckInput(pkt, inNICName); !ok { + if ok := stk.IPTables().CheckInput(pkt, inNICName); !ok { // iptables is telling us to drop the packet. stats.ip.IPTablesInputDropped.Increment() return } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckInput(pkt, nil /* route */, stack.IP) { + // nftables is telling us to drop the packet. + return + } + } + if h.More() || h.FragmentOffset() != 0 { if pkt.Data().Size()+len(pkt.TransportHeader().Slice()) == 0 { // Drop the packet as it's marked as a fragment but has @@ -1327,7 +1456,7 @@ func (e *endpoint) deliverPacketLocally(h header.IPv4, pkt *stack.PacketBuffer, } if p == header.IGMPProtocolNumber { e.mu.Lock() - e.igmp.handleIGMP(pkt, hasRouterAlertOption) // +checklocksforce: e == e.igmp.ep. + e.igmp.handleIGMP(pkt, hasRouterAlertOption) e.mu.Unlock() return } @@ -1377,7 +1506,6 @@ func (e *endpoint) AddAndAcquirePermanentAddress(addr tcpip.AddressWithPrefix, p // sendQueuedReports sends queued igmp reports. // // +checklocks:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) sendQueuedReports() { e.igmp.sendQueuedReports() } @@ -1463,7 +1591,6 @@ func (e *endpoint) JoinGroup(addr tcpip.Address) tcpip.Error { // joinGroupLocked is like JoinGroup but with locking requirements. // // +checklocks:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) joinGroupLocked(addr tcpip.Address) tcpip.Error { if !header.IsV4MulticastAddress(addr) { return &tcpip.ErrBadAddress{} @@ -1483,7 +1610,6 @@ func (e *endpoint) LeaveGroup(addr tcpip.Address) tcpip.Error { // leaveGroupLocked is like LeaveGroup but with locking requirements. // // +checklocks:e.mu -// +checklocksalias:e.igmp.ep.mu=e.mu func (e *endpoint) leaveGroupLocked(addr tcpip.Address) tcpip.Error { return e.igmp.leaveGroup(addr) } @@ -1492,7 +1618,7 @@ func (e *endpoint) leaveGroupLocked(addr tcpip.Address) tcpip.Error { func (e *endpoint) IsInGroup(addr tcpip.Address) bool { e.mu.RLock() defer e.mu.RUnlock() - return e.igmp.isInGroup(addr) // +checklocksforce: e.mu==e.igmp.ep.mu. + return e.igmp.isInGroup(addr) } // Stats implements stack.NetworkEndpoint. @@ -1860,6 +1986,8 @@ func (p *protocol) SendRejectionError(pkt *stack.PacketBuffer, rejectWith stack. return p.returnError(&icmpReasonHostProhibited{}, pkt, inputHook) case stack.RejectIPv4WithICMPAdminProhibited: return p.returnError(&icmpReasonAdministrativelyProhibited{}, pkt, inputHook) + case stack.RejectIPv4WithTCPReset: + return ip.RejectWithTCPReset(pkt, ProtocolNumber, p.stack, inputHook) default: panic(fmt.Sprintf("unhandled %[1]T = %[1]d", rejectWith)) } diff --git a/pkg/tcpip/network/ipv6/icmp.go b/pkg/tcpip/network/ipv6/icmp.go index f6e8f33..d140a4a 100644 --- a/pkg/tcpip/network/ipv6/icmp.go +++ b/pkg/tcpip/network/ipv6/icmp.go @@ -654,11 +654,29 @@ func (e *endpoint) handleICMP(pkt *stack.PacketBuffer, hasFragmentHeader bool, r case header.ICMPv6EchoRequest: received.echoRequest.Increment() - if len(h) < header.ICMPv6EchoMinimumSize { - received.invalid.Increment() + replyPayload := pkt.Data().ToBuffer() + replyHeader := make([]byte, header.ICMPv6EchoMinimumSize) + copy(replyHeader, h[:header.ICMPv6EchoMinimumSize]) + + // It's possible that a raw socket or per-stack default handler expects + // to receive this packet. + defaultHandlerHandled := false + if dispatcher, ok := e.dispatcher.(stack.TransportDispatcherWithDefaultHandlerResult); ok { + _, defaultHandlerHandled = dispatcher.DeliverTransportPacketWithDefaultHandlerResult(header.ICMPv6ProtocolNumber, pkt) + } else { + e.dispatcher.DeliverTransportPacket(header.ICMPv6ProtocolNumber, pkt) + } + pkt = nil + + // Skip the built-in ICMP echo reply if the request was consumed by a + // per-stack default handler. + if defaultHandlerHandled { + replyPayload.Release() return } - e.dispatcher.DeliverTransportPacket(header.ICMPv6ProtocolNumber, pkt) + + e.sendICMPEchoReply(replyPayload, replyHeader, srcAddr, dstAddr, iph) + case header.ICMPv6EchoReply: received.echoReply.Increment() if len(h) < header.ICMPv6EchoMinimumSize { @@ -666,6 +684,7 @@ func (e *endpoint) handleICMP(pkt *stack.PacketBuffer, hasFragmentHeader bool, r return } e.dispatcher.DeliverTransportPacket(header.ICMPv6ProtocolNumber, pkt) + case header.ICMPv6TimeExceeded: received.timeExceeded.Increment() @@ -852,6 +871,62 @@ func (e *endpoint) handleICMP(pkt *stack.PacketBuffer, hasFragmentHeader bool, r } } +func (e *endpoint) sendICMPEchoReply(replyPayload buffer.Buffer, replyHeader []byte, srcAddr, dstAddr tcpip.Address, ipHdr header.IPv6) { + sent := e.stats.icmp.packetsSent + + // As per RFC 4291 section 2.7, multicast addresses must not be used as + // source addresses in IPv6 packets. + localAddr := dstAddr + if header.IsV6MulticastAddress(dstAddr) { + localAddr = tcpip.Address{} + } + + r, err := e.protocol.stack.FindRoute(e.nic.ID(), localAddr, srcAddr, ProtocolNumber, false /* multicastLoop */) + if err != nil { + // If we cannot find a route to the destination, silently drop the packet. + replyPayload.Release() + return + } + defer r.Release() + + if !e.protocol.allowICMPReply(header.ICMPv6EchoReply) { + sent.rateLimited.Increment() + replyPayload.Release() + return + } + + replyPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: int(r.MaxHeaderLength()) + header.ICMPv6EchoMinimumSize, + Payload: replyPayload, + }) + defer replyPkt.DecRef() + icmp := header.ICMPv6(replyPkt.TransportHeader().Push(header.ICMPv6EchoMinimumSize)) + replyPkt.TransportProtocolNumber = header.ICMPv6ProtocolNumber + copy(icmp, replyHeader) + icmp.SetType(header.ICMPv6EchoReply) + replyData := replyPkt.Data() + icmp.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmp, + Src: r.LocalAddress(), + Dst: r.RemoteAddress(), + PayloadCsum: replyData.Checksum(), + PayloadLen: replyData.Size(), + })) + replyTClass, _ := ipHdr.TOS() + if err := r.WritePacket(stack.NetworkHeaderParams{ + Protocol: header.ICMPv6ProtocolNumber, + TTL: r.DefaultTTL(), + // Even though RFC 4443 does not mention anything about it, Linux uses the + // TrafficClass of the received echo request when replying. + // https://github.com/torvalds/linux/blob/0280e3c58f9/net/ipv6/icmp.c#L797 + TOS: replyTClass, + }, replyPkt); err != nil { + sent.dropped.Increment() + return + } + sent.echoReply.Increment() +} + // LinkAddressProtocol implements stack.LinkAddressResolver. func (*endpoint) LinkAddressProtocol() tcpip.NetworkProtocolNumber { return header.IPv6ProtocolNumber diff --git a/pkg/tcpip/network/ipv6/ipv6.go b/pkg/tcpip/network/ipv6/ipv6.go index bfc28a5..aa5cec7 100644 --- a/pkg/tcpip/network/ipv6/ipv6.go +++ b/pkg/tcpip/network/ipv6/ipv6.go @@ -674,7 +674,7 @@ func (e *endpoint) Disable() { } func (e *endpoint) disableLocked() { - if !e.Enabled() { + if !e.isEnabled() { return } @@ -730,6 +730,11 @@ func (e *endpoint) MTU() uint32 { return networkMTU } +// EndpointHeaderSize returns the size necessary for the IPv6 header. +func (e *endpoint) EndpointHeaderSize() uint32 { + return header.IPv6MinimumSize +} + // MaxHeaderLength returns the maximum length needed by ipv6 headers (and // underlying protocols). func (e *endpoint) MaxHeaderLength() uint16 { @@ -819,15 +824,23 @@ func (e *endpoint) WritePacket(r *stack.Route, params stack.NetworkHeaderParams, return err } - // iptables filtering. All packets that reach here are locally - // generated. - outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) - if ok := e.protocol.stack.IPTables().CheckOutput(pkt, r, outNicName); !ok { + stk := e.protocol.stack + // iptables filtering. All packets that reach here are locally generated. + outNicName := stk.FindNICNameFromID(e.nic.ID()) + if ok := stk.IPTables().CheckOutput(pkt, r, outNicName); !ok { // iptables is telling us to drop the packet. e.stats.ip.IPTablesOutputDropped.Increment() return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + // TODO: b/486197011 - Add support for NAT re-routing in IPv6. + if !nft.CheckOutput(pkt, r, stack.IP6) { + // nftables is telling us to drop the packet. + return nil + } + } + // If the packet is manipulated as per DNAT Output rules, handle packet // based on destination address and do not send the packet to link // layer. @@ -858,15 +871,23 @@ func (e *endpoint) writePacket(r *stack.Route, pkt *stack.PacketBuffer, protocol return nil } + stk := e.protocol.stack // Postrouting NAT can only change the source address, and does not alter the // route or outgoing interface of the packet. - outNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) - if ok := e.protocol.stack.IPTables().CheckPostrouting(pkt, r, e, outNicName); !ok { + outNicName := stk.FindNICNameFromID(e.nic.ID()) + if ok := stk.IPTables().CheckPostrouting(pkt, r, e, outNicName); !ok { // iptables is telling us to drop the packet. e.stats.ip.IPTablesPostroutingDropped.Increment() return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckPostrouting(pkt, r, stack.IP6) { + // nftables is telling us to drop the packet. + return nil + } + } + stats := e.stats.ip networkMTU, err := calculateNetworkMTU(e.nic.MTU(), uint32(len(pkt.NetworkHeader().Slice()))) if err != nil { @@ -1007,6 +1028,13 @@ func (e *endpoint) forwardUnicastPacket(pkt *stack.PacketBuffer) ip.ForwardingEr return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckForward(pkt, nil, stack.IP6) { + // nftables is telling us to drop the packet. + return nil + } + } + // The packet originally arrived on e so provide its NIC as the input NIC. ep.handleValidatedPacket(h, pkt, e.nic.Name() /* inNICName */) return nil @@ -1048,6 +1076,13 @@ func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketB return nil } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckForward(pkt, route, stack.IP6) { + // nftables is telling us to drop the packet. + return nil + } + } + hopLimit := h.HopLimit() // We need to do a deep copy of the IP packet because @@ -1063,6 +1098,10 @@ func (e *endpoint) forwardPacketWithRoute(route *stack.Route, pkt *stack.PacketB // each node that forwards the packet. newHdr.SetHopLimit(hopLimit - 1) + if route.RequiresTXTransportChecksum() { + newPkt.CalculateTransportChecksum() + } + forwardToEp, ok := e.protocol.getEndpointForNIC(route.NICID()) if !ok { // The interface was removed after we obtained the route. @@ -1123,7 +1162,8 @@ func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { } } - if e.protocol.stack.HandleLocal() { + stk := e.protocol.stack + if stk.HandleLocal() { addressEndpoint := e.AcquireAssignedAddress(header.IPv6(pkt.NetworkHeader().Slice()).SourceAddress(), e.nic.Promiscuous(), stack.CanBePrimaryEndpoint, true /* readOnly */) if addressEndpoint != nil { // The source address is one of our own, so we never should have gotten @@ -1135,12 +1175,19 @@ func (e *endpoint) HandlePacket(pkt *stack.PacketBuffer) { } // Loopback traffic skips the prerouting chain. - inNicName := e.protocol.stack.FindNICNameFromID(e.nic.ID()) - if ok := e.protocol.stack.IPTables().CheckPrerouting(pkt, e, inNicName); !ok { + inNicName := stk.FindNICNameFromID(e.nic.ID()) + if ok := stk.IPTables().CheckPrerouting(pkt, e, inNicName); !ok { // iptables is telling us to drop the packet. stats.IPTablesPreroutingDropped.Increment() return } + + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckPrerouting(pkt, nil, stack.IP6) { + // nftables is telling us to drop the packet. + return + } + } } // CheckPrerouting can modify the backing storage of the packet, so refresh @@ -1386,15 +1433,20 @@ func (e *endpoint) handleValidatedPacket(h header.IPv6, pkt *stack.PacketBuffer, func (e *endpoint) deliverPacketLocally(h header.IPv6, pkt *stack.PacketBuffer, inNICName string) { stats := e.stats.ip - - // iptables filtering. All packets that reach here are intended for - // this machine and need not be forwarded. - if ok := e.protocol.stack.IPTables().CheckInput(pkt, inNICName); !ok { + stk := e.protocol.stack + if ok := stk.IPTables().CheckInput(pkt, inNICName); !ok { // iptables is telling us to drop the packet. stats.IPTablesInputDropped.Increment() return } + if nft := stk.NFTables(); nft != nil && stk.IsNFTablesConfigured() { + if !nft.CheckInput(pkt, nil, stack.IP6) { + // nftables is telling us to drop the packet. + return + } + } + // Any returned error is only useful for terminating execution early, but // we have nothing left to do, so we can drop it. _ = e.processExtensionHeaders(h, pkt, false /* forwarding */) @@ -2692,6 +2744,8 @@ func (p *protocol) SendRejectionError(pkt *stack.PacketBuffer, rejectWith stack. return p.returnError(&icmpReasonPortUnreachable{}, pkt, inputHook) case stack.RejectIPv6WithICMPAdminProhibited: return p.returnError(&icmpReasonAdministrativelyProhibited{}, pkt, inputHook) + case stack.RejectIPv6WithTCPReset: + return ip.RejectWithTCPReset(pkt, ProtocolNumber, p.stack, inputHook) default: panic(fmt.Sprintf("unhandled %[1]T = %[1]d", rejectWith)) } diff --git a/pkg/tcpip/network/ipv6/ipv6_state_autogen.go b/pkg/tcpip/network/ipv6/ipv6_state_autogen.go index 2e97ea4..eb6eb0a 100644 --- a/pkg/tcpip/network/ipv6/ipv6_state_autogen.go +++ b/pkg/tcpip/network/ipv6/ipv6_state_autogen.go @@ -525,7 +525,6 @@ func (t *timer) StateTypeName() string { func (t *timer) StateFields() []string { return []string{ "done", - "timer", } } @@ -535,7 +534,6 @@ func (t *timer) beforeSave() {} func (t *timer) StateSave(stateSinkObject state.Sink) { t.beforeSave() stateSinkObject.Save(0, &t.done) - stateSinkObject.Save(1, &t.timer) } func (t *timer) afterLoad(context.Context) {} @@ -543,7 +541,6 @@ func (t *timer) afterLoad(context.Context) {} // +checklocksignore func (t *timer) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &t.done) - stateSourceObject.Load(1, &t.timer) } func (o *offLinkRoute) StateTypeName() string { diff --git a/pkg/tcpip/network/ipv6/ndp.go b/pkg/tcpip/network/ipv6/ndp.go index a2e3d24..cab60fd 100644 --- a/pkg/tcpip/network/ipv6/ndp.go +++ b/pkg/tcpip/network/ipv6/ndp.go @@ -471,7 +471,7 @@ type timer struct { // done indicates to the timer that the timer was stopped. done *bool - timer tcpip.Timer + timer tcpip.Timer `state:"nosave"` } // +stateify savable diff --git a/pkg/tcpip/socketops.go b/pkg/tcpip/socketops.go index a83d5ad..cbc8494 100644 --- a/pkg/tcpip/socketops.go +++ b/pkg/tcpip/socketops.go @@ -253,13 +253,6 @@ type SocketOptions struct { // receiveBufferSize determines the receive buffer size for this socket. receiveBufferSize atomicbitops.Int64 - // mu protects the access to the below fields. - mu sync.Mutex `state:"nosave"` - - // linger determines the amount of time the socket should linger before - // close. We currently implement this option for TCP socket only. - linger LingerOption - // rcvlowat specifies the minimum number of bytes which should be // received to indicate the socket as readable. rcvlowat atomicbitops.Int32 @@ -267,6 +260,16 @@ type SocketOptions struct { // experimentOptionValue is the value set for the IP option experiment header // if it is not zero. experimentOptionValue atomicbitops.Uint32 + + // mark is the mark value set for the socket. + mark atomicbitops.Uint32 + + // mu protects the access to the below fields. + mu sync.Mutex `state:"nosave"` + + // linger determines the amount of time the socket should linger before + // close. We currently implement this option for TCP socket only. + linger LingerOption } // InitHandler initializes the handler. This must be called before using the @@ -771,3 +774,13 @@ func (so *SocketOptions) SetRcvlowat(rcvlowat int32) Error { func (so *SocketOptions) GetAcceptConn() bool { return so.handler.GetAcceptConn() } + +// GetMark gets value for SO_MARK option. +func (so *SocketOptions) GetMark() uint32 { + return so.mark.Load() +} + +// SetMark sets value for SO_MARK option. +func (so *SocketOptions) SetMark(v uint32) { + so.mark.Store(v) +} diff --git a/pkg/tcpip/stack/address_state_mutex.go b/pkg/tcpip/stack/address_state_mutex.go index eb8c8e0..0d8bb2e 100644 --- a/pkg/tcpip/stack/address_state_mutex.go +++ b/pkg/tcpip/stack/address_state_mutex.go @@ -92,5 +92,5 @@ func addressStateinitLockNames() {} func init() { addressStateinitLockNames() - addressStateprefixIndex = locking.NewMutexClass(reflect.TypeOf(addressStateRWMutex{}), addressStatelockNames) + addressStateprefixIndex = locking.NewMutexClass(reflect.TypeFor[addressStateRWMutex](), addressStatelockNames) } diff --git a/pkg/tcpip/stack/addressable_endpoint_state_mutex.go b/pkg/tcpip/stack/addressable_endpoint_state_mutex.go index df42c81..12d0d37 100644 --- a/pkg/tcpip/stack/addressable_endpoint_state_mutex.go +++ b/pkg/tcpip/stack/addressable_endpoint_state_mutex.go @@ -92,5 +92,5 @@ func addressableEndpointStateinitLockNames() {} func init() { addressableEndpointStateinitLockNames() - addressableEndpointStateprefixIndex = locking.NewMutexClass(reflect.TypeOf(addressableEndpointStateRWMutex{}), addressableEndpointStatelockNames) + addressableEndpointStateprefixIndex = locking.NewMutexClass(reflect.TypeFor[addressableEndpointStateRWMutex](), addressableEndpointStatelockNames) } diff --git a/pkg/tcpip/stack/bridge_mutex.go b/pkg/tcpip/stack/bridge_mutex.go index 4ac0890..4818a81 100644 --- a/pkg/tcpip/stack/bridge_mutex.go +++ b/pkg/tcpip/stack/bridge_mutex.go @@ -92,5 +92,5 @@ func bridgeinitLockNames() {} func init() { bridgeinitLockNames() - bridgeprefixIndex = locking.NewMutexClass(reflect.TypeOf(bridgeRWMutex{}), bridgelockNames) + bridgeprefixIndex = locking.NewMutexClass(reflect.TypeFor[bridgeRWMutex](), bridgelockNames) } diff --git a/pkg/tcpip/stack/bucket_mutex.go b/pkg/tcpip/stack/bucket_mutex.go index fef6e33..fa021f2 100644 --- a/pkg/tcpip/stack/bucket_mutex.go +++ b/pkg/tcpip/stack/bucket_mutex.go @@ -93,5 +93,5 @@ func bucketinitLockNames() { bucketlockNames = []string{"otherTuple"} } func init() { bucketinitLockNames() - bucketprefixIndex = locking.NewMutexClass(reflect.TypeOf(bucketRWMutex{}), bucketlockNames) + bucketprefixIndex = locking.NewMutexClass(reflect.TypeFor[bucketRWMutex](), bucketlockNames) } diff --git a/pkg/tcpip/stack/cleanup_endpoints_mutex.go b/pkg/tcpip/stack/cleanup_endpoints_mutex.go index 67a5bdb..da661a4 100644 --- a/pkg/tcpip/stack/cleanup_endpoints_mutex.go +++ b/pkg/tcpip/stack/cleanup_endpoints_mutex.go @@ -60,5 +60,5 @@ func cleanupEndpointsinitLockNames() {} func init() { cleanupEndpointsinitLockNames() - cleanupEndpointsprefixIndex = locking.NewMutexClass(reflect.TypeOf(cleanupEndpointsMutex{}), cleanupEndpointslockNames) + cleanupEndpointsprefixIndex = locking.NewMutexClass(reflect.TypeFor[cleanupEndpointsMutex](), cleanupEndpointslockNames) } diff --git a/pkg/tcpip/stack/conn_mutex.go b/pkg/tcpip/stack/conn_mutex.go index 16ccbf4..cb0b572 100644 --- a/pkg/tcpip/stack/conn_mutex.go +++ b/pkg/tcpip/stack/conn_mutex.go @@ -92,5 +92,5 @@ func conninitLockNames() {} func init() { conninitLockNames() - connprefixIndex = locking.NewMutexClass(reflect.TypeOf(connRWMutex{}), connlockNames) + connprefixIndex = locking.NewMutexClass(reflect.TypeFor[connRWMutex](), connlockNames) } diff --git a/pkg/tcpip/stack/conn_track_mutex.go b/pkg/tcpip/stack/conn_track_mutex.go index 3e67080..a6aa00a 100644 --- a/pkg/tcpip/stack/conn_track_mutex.go +++ b/pkg/tcpip/stack/conn_track_mutex.go @@ -92,5 +92,5 @@ func connTrackinitLockNames() {} func init() { connTrackinitLockNames() - connTrackprefixIndex = locking.NewMutexClass(reflect.TypeOf(connTrackRWMutex{}), connTracklockNames) + connTrackprefixIndex = locking.NewMutexClass(reflect.TypeFor[connTrackRWMutex](), connTracklockNames) } diff --git a/pkg/tcpip/stack/conntrack.go b/pkg/tcpip/stack/conntrack.go index 7ecf5c9..4b1c667 100644 --- a/pkg/tcpip/stack/conntrack.go +++ b/pkg/tcpip/stack/conntrack.go @@ -18,7 +18,6 @@ import ( "encoding/binary" "fmt" "math" - "math/rand" "sync" "time" @@ -45,6 +44,31 @@ const ( unestablishedTimeout time.Duration = 120 * time.Second ) +// ConnTrackState represents the state of a connection. +type ConnTrackState int + +const ( + // ConnTrackStateInvalid is the invalid connection tracking state. + ConnTrackStateInvalid ConnTrackState = -1 + // ConnTrackStateEstablished represents an established connection. + ConnTrackStateEstablished ConnTrackState = 0 + // ConnTrackStateNew represents a new connection. + ConnTrackStateNew ConnTrackState = 2 + // ConnTrackStateEstablishedReply represents an established connection + // in the reply direction. + ConnTrackStateEstablishedReply ConnTrackState = 3 +) + +// ConnTrackDirection represents the direction of a connection. +type ConnTrackDirection uint8 + +const ( + // ConnTrackDirectionOriginal represents the original direction. + ConnTrackDirectionOriginal ConnTrackDirection = 0 + // ConnTrackDirectionReply represents the reply direction. + ConnTrackDirectionReply ConnTrackDirection = 1 +) + // tuple holds a connection's identifying and manipulating data in one // direction. It is immutable. // @@ -161,6 +185,10 @@ type conn struct { // // +checklocks:stateMu lastUsed tcpip.MonotonicTime + // replySeen indicates whether a packet in the reply direction has been seen. + // + // +checklocks:stateMu + replySeen bool } // timedOut returns whether the connection timed out based on its state. @@ -177,6 +205,27 @@ func (cn *conn) timedOut(now tcpip.MonotonicTime) bool { return now.Sub(cn.lastUsed) > unestablishedTimeout } +// expiresIn returns the duration from now until the connection times out. +func (cn *conn) expiresIn() time.Duration { + var timeout time.Duration + var lastUsed tcpip.MonotonicTime + cn.stateMu.RLock() + state := cn.tcb.State() + lastUsed = cn.lastUsed + cn.stateMu.RUnlock() + if state == tcpconntrack.ResultAlive { + timeout = establishedTimeout + } else { + timeout = unestablishedTimeout + } + now := cn.ct.clock.NowMonotonic() + expires := timeout - now.Sub(lastUsed) + if expires < 0 { + return 0 + } + return expires +} + // update the connection tracking state. func (cn *conn) update(pkt *PacketBuffer, reply bool) { cn.stateMu.Lock() @@ -184,6 +233,9 @@ func (cn *conn) update(pkt *PacketBuffer, reply bool) { // Mark the connection as having been used recently so it isn't reaped. cn.lastUsed = cn.ct.clock.NowMonotonic() + if reply { + cn.replySeen = true + } if pkt.TransportProtocolNumber != header.TCPProtocolNumber { return @@ -206,6 +258,10 @@ func (cn *conn) update(pkt *PacketBuffer, reply bool) { } } +type connTrackRNG interface { + Uint32() uint32 +} + // ConnTrack tracks all connections created for NAT rules. Most users are // expected to only call handlePacket, insertRedirectConn, and maybeInsertNoop. // @@ -225,12 +281,26 @@ type ConnTrack struct { // seed is a one-time random value initialized at stack startup // and is used in the calculation of hash keys for the list of buckets. // It is immutable. + // + // TODO(gvisor.dev/issue/4595): When Stack.tables becomes savable and + // ConnTrack flows into checkpoint state, this seed must be redrawn + // from secureRNG during restore AND the entries in buckets must be + // rehashed under the new seed. bucket_index = jenkins.Sum32(seed) % + // len(buckets) couples the seed value to bucket layout; redrawing the + // seed without rehashing leaves restored entries unreachable by + // Lookup. Persisting the pre-checkpoint seed extends the brute-force + // window across save boundaries. seed uint32 + // nftIDSeed is a one-time random value initialized at stack startup + // and is used in the calculation of tuple IDs for nftables. + // It is immutable. + nftIDSeed uint32 + // clock provides timing used to determine conntrack reapings. clock tcpip.Clock // TODO(b/341946753): Restore when netstack is savable. - rand *rand.Rand `state:"nosave"` + rng connTrackRNG `state:"nosave"` mu connTrackRWMutex `state:"nosave"` // mu protects the buckets slice, but not buckets' contents. Only take @@ -271,99 +341,6 @@ func v6NetAndTransHdr(icmpPayload []byte, minTransHdrLen int) (header.Network, [ return netHdr, transHdr[:minTransHdrLen] } -func getEmbeddedNetAndTransHeaders(pkt *PacketBuffer, netHdrLength int, getNetAndTransHdr netAndTransHeadersFunc, transProto tcpip.TransportProtocolNumber) (header.Network, header.ChecksummableTransport, bool) { - switch transProto { - case header.TCPProtocolNumber: - if netAndTransHeader, ok := pkt.Data().PullUp(netHdrLength + header.TCPMinimumSize); ok { - netHeader, transHeaderBytes := getNetAndTransHdr(netAndTransHeader, header.TCPMinimumSize) - return netHeader, header.TCP(transHeaderBytes), true - } - case header.UDPProtocolNumber: - if netAndTransHeader, ok := pkt.Data().PullUp(netHdrLength + header.UDPMinimumSize); ok { - netHeader, transHeaderBytes := getNetAndTransHdr(netAndTransHeader, header.UDPMinimumSize) - return netHeader, header.UDP(transHeaderBytes), true - } - } - return nil, nil, false -} - -func getHeaders(pkt *PacketBuffer) (netHdr header.Network, transHdr header.Transport, isICMPError bool, ok bool) { - switch pkt.TransportProtocolNumber { - case header.TCPProtocolNumber: - if tcpHeader := header.TCP(pkt.TransportHeader().Slice()); len(tcpHeader) >= header.TCPMinimumSize { - return pkt.Network(), tcpHeader, false, true - } - return nil, nil, false, false - case header.UDPProtocolNumber: - if udpHeader := header.UDP(pkt.TransportHeader().Slice()); len(udpHeader) >= header.UDPMinimumSize { - return pkt.Network(), udpHeader, false, true - } - return nil, nil, false, false - case header.ICMPv4ProtocolNumber: - icmpHeader := header.ICMPv4(pkt.TransportHeader().Slice()) - if len(icmpHeader) < header.ICMPv4MinimumSize { - return nil, nil, false, false - } - - switch icmpType := icmpHeader.Type(); icmpType { - case header.ICMPv4Echo, header.ICMPv4EchoReply: - return pkt.Network(), icmpHeader, false, true - case header.ICMPv4DstUnreachable, header.ICMPv4TimeExceeded, header.ICMPv4ParamProblem: - default: - panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) - } - - h, ok := pkt.Data().PullUp(header.IPv4MinimumSize) - if !ok { - panic(fmt.Sprintf("should have a valid IPv4 packet; only have %d bytes, want at least %d bytes", pkt.Data().Size(), header.IPv4MinimumSize)) - } - - if header.IPv4(h).HeaderLength() > header.IPv4MinimumSize { - // TODO(https://gvisor.dev/issue/6765): Handle IPv4 options. - panic("should have dropped packets with IPv4 options") - } - - if netHdr, transHdr, ok := getEmbeddedNetAndTransHeaders(pkt, header.IPv4MinimumSize, v4NetAndTransHdr, pkt.tuple.tupleID.transProto); ok { - return netHdr, transHdr, true, true - } - return nil, nil, false, false - case header.ICMPv6ProtocolNumber: - icmpHeader := header.ICMPv6(pkt.TransportHeader().Slice()) - if len(icmpHeader) < header.ICMPv6MinimumSize { - return nil, nil, false, false - } - - switch icmpType := icmpHeader.Type(); icmpType { - case header.ICMPv6EchoRequest, header.ICMPv6EchoReply: - return pkt.Network(), icmpHeader, false, true - case header.ICMPv6DstUnreachable, header.ICMPv6PacketTooBig, header.ICMPv6TimeExceeded, header.ICMPv6ParamProblem: - default: - panic(fmt.Sprintf("unexpected ICMPv6 type = %d", icmpType)) - } - - h, ok := pkt.Data().PullUp(header.IPv6MinimumSize) - if !ok { - panic(fmt.Sprintf("should have a valid IPv6 packet; only have %d bytes, want at least %d bytes", pkt.Data().Size(), header.IPv6MinimumSize)) - } - - // We do not support extension headers in ICMP errors so the next header - // in the IPv6 packet should be a tracked protocol if we reach this point. - // - // TODO(https://gvisor.dev/issue/6789): Support extension headers. - transProto := pkt.tuple.tupleID.transProto - if got := header.IPv6(h).TransportProtocol(); got != transProto { - panic(fmt.Sprintf("got TransportProtocol() = %d, want = %d", got, transProto)) - } - - if netHdr, transHdr, ok := getEmbeddedNetAndTransHeaders(pkt, header.IPv6MinimumSize, v6NetAndTransHdr, transProto); ok { - return netHdr, transHdr, true, true - } - return nil, nil, false, false - default: - panic(fmt.Sprintf("unexpected transport protocol = %d", pkt.TransportProtocolNumber)) - } -} - func getTupleIDForRegularPacket(netHdr header.Network, netProto tcpip.NetworkProtocolNumber, transHdr header.Transport, transProto tcpip.TransportProtocolNumber) tupleID { return tupleID{ srcAddr: netHdr.SourceAddress(), @@ -376,7 +353,7 @@ func getTupleIDForRegularPacket(netHdr header.Network, netProto tcpip.NetworkPro } func getTupleIDForPacketInICMPError(pkt *PacketBuffer, getNetAndTransHdr netAndTransHeadersFunc, netProto tcpip.NetworkProtocolNumber, netLen int, transProto tcpip.TransportProtocolNumber) (tupleID, bool) { - if netHdr, transHdr, ok := getEmbeddedNetAndTransHeaders(pkt, netLen, getNetAndTransHdr, transProto); ok { + if netHdr, transHdr, ok := pkt.GetEmbeddedNetAndTransHeaders(netLen, getNetAndTransHdr, transProto); ok { return tupleID{ srcAddr: netHdr.DestinationAddress(), srcPortOrEchoRequestIdent: transHdr.DestinationPort(), @@ -602,6 +579,11 @@ func (ct *ConnTrack) getConnAndUpdate(pkt *PacketBuffer, skipChecksumValidation return t } +// GetConnAndUpdatePkt gets the connection for the packet and also sets the packet's tuple. +func (ct *ConnTrack) GetConnAndUpdatePkt(pkt *PacketBuffer, skipChecksumValidation bool) { + pkt.tuple = ct.getConnAndUpdate(pkt, skipChecksumValidation) +} + func (ct *ConnTrack) connForTID(tid tupleID) *tuple { ct.mu.RLock() bkt := &ct.buckets[ct.bucket(tid)] @@ -610,6 +592,114 @@ func (ct *ConnTrack) connForTID(tid tupleID) *tuple { return bkt.connForTID(tid, ct.clock.NowMonotonic()) } +// ConnTrackInfo holds connection tracking information for a packet. +type ConnTrackInfo struct { + State ConnTrackState + Direction ConnTrackDirection + SrcAddr tcpip.Address + DstAddr tcpip.Address + SrcPort uint16 + DstPort uint16 + NetProto tcpip.NetworkProtocolNumber + TransProto tcpip.TransportProtocolNumber + Expiration time.Duration + PseudoID uint32 + Bytes uint64 + Packets uint64 +} + +// ConnTrackInfoOpts holds options for GetConnTrackInfo. +type ConnTrackInfoOpts struct { + FillState bool + UseReplyDir bool + FillPseudoID bool + FillExpiration bool +} + +// getTCPConnTrackState converts the TCB state to ConnTrackState. +func (cn *conn) getTCPConnTrackState(useReplyDir bool) ConnTrackState { + state := ConnTrackStateInvalid + cn.stateMu.RLock() + tcbState := cn.tcb.State() + cn.stateMu.RUnlock() + switch tcbState { + case tcpconntrack.ResultConnecting: + state = ConnTrackStateNew + + case tcpconntrack.ResultAlive, tcpconntrack.ResultReset, + tcpconntrack.ResultClosedByOriginator, tcpconntrack.ResultClosedByResponder: + + if useReplyDir { + state = ConnTrackStateEstablishedReply + } else { + state = ConnTrackStateEstablished + } + case tcpconntrack.ResultDrop: + state = ConnTrackStateInvalid + } + return state +} + +// getConnTrackState returns the connection tracking state for the connection. +func (cn *conn) getConnTrackState(useReplyDir bool) ConnTrackState { + state := ConnTrackStateInvalid + // TCP connections have their own state machine in the TCB. + if cn.original.tupleID.transProto == header.TCPProtocolNumber { + return cn.getTCPConnTrackState(useReplyDir) + } + // For non-TCP connections, fill the info based on the reply. + cn.stateMu.RLock() + replySeen := cn.replySeen + cn.stateMu.RUnlock() + if useReplyDir { + state = ConnTrackStateEstablishedReply + } else if replySeen { + state = ConnTrackStateEstablished + } else { + state = ConnTrackStateNew + } + return state +} + +// FillConnTrackInfo fills connection tracking information for the connection. +func (cn *conn) FillConnTrackInfo(opts ConnTrackInfoOpts, info *ConnTrackInfo) bool { + state := ConnTrackStateInvalid + if opts.FillState { + state = cn.getConnTrackState(opts.UseReplyDir) + } + + dir := ConnTrackDirectionOriginal + t := &cn.original + if opts.UseReplyDir { + t = &cn.reply + dir = ConnTrackDirectionReply + } + tID := t.tupleID + + pID := uint32(0) + if opts.FillPseudoID { + // Generate a pseudo-ID similar to Linux nf_ct_get_id + pID = tupleHash(cn.original.tupleID, cn.ct.nftIDSeed) + } + + var expires time.Duration + if opts.FillExpiration { + expires = cn.expiresIn() + } + + info.State = state + info.Direction = dir + info.SrcAddr = tID.srcAddr + info.DstAddr = tID.dstAddr + info.SrcPort = tID.srcPortOrEchoRequestIdent + info.DstPort = tID.dstPortOrEchoReplyIdent + info.NetProto = tID.netProto + info.TransProto = tID.transProto + info.Expiration = expires + info.PseudoID = pID + return true +} + func (bkt *bucket) connForTID(tid tupleID, now tcpip.MonotonicTime) *tuple { bkt.mu.RLock() defer bkt.mu.RUnlock() @@ -697,325 +787,14 @@ func (cn *conn) finalize() bool { } } -// If NAT has not been configured for this connection, either mark the -// connection as configured for "no-op NAT", in the case of DNAT, or, in the -// case of SNAT, perform source port remapping so that source ports used by -// locally-generated traffic do not conflict with ports occupied by existing NAT -// bindings. -// -// Note that in the typical case this is also a no-op, because `snatAction` -// will do nothing if the original tuple is already unique. -func (cn *conn) maybePerformNoopNAT(pkt *PacketBuffer, hook Hook, r *Route, dnat bool) { - cn.mu.Lock() - var manip *manipType - if dnat { - manip = &cn.destinationManip - } else { - manip = &cn.sourceManip - } - if *manip != manipNotPerformed { - cn.mu.Unlock() - _ = cn.handlePacket(pkt, hook, r) - return - } - if dnat { - *manip = manipPerformedNoop - cn.mu.Unlock() - _ = cn.handlePacket(pkt, hook, r) - return - } - cn.mu.Unlock() - - // At this point, we know that NAT has not yet been performed on this - // connection, and the DNAT case has been handled with a no-op. For SNAT, we - // simply perform source port remapping to ensure that source ports for - // locally generated traffic do not clash with ports used by existing NAT - // bindings. - _, _ = snatAction(pkt, hook, r, 0, tcpip.Address{}, true /* changePort */, false /* changeAddress */) -} - -type portOrIdentRange struct { - start uint16 - size uint32 -} - -// performNAT setups up the connection for the specified NAT and rewrites the -// packet. -// -// If NAT has already been performed on the connection, then the packet will -// be rewritten with the NAT performed on the connection, ignoring the passed -// address and port range. -// -// Generally, only the first packet of a connection reaches this method; other -// packets will be manipulated without needing to modify the connection. -func (cn *conn) performNAT(pkt *PacketBuffer, hook Hook, r *Route, portsOrIdents portOrIdentRange, natAddress tcpip.Address, dnat, changePort, changeAddress bool) { - lastPortOrIdent := func() uint16 { - lastPortOrIdent := uint32(portsOrIdents.start) + portsOrIdents.size - 1 - if lastPortOrIdent > math.MaxUint16 { - panic(fmt.Sprintf("got lastPortOrIdent = %d, want <= MaxUint16(=%d); portsOrIdents=%#v", lastPortOrIdent, math.MaxUint16, portsOrIdents)) - } - return uint16(lastPortOrIdent) - }() - - // Make sure the packet is re-written after performing NAT. - defer func() { - // handlePacket returns true if the packet may skip the NAT table as the - // connection is already NATed, but if we reach this point we must be in the - // NAT table, so the return value is useless for us. - _ = cn.handlePacket(pkt, hook, r) - }() - - cn.mu.Lock() - defer cn.mu.Unlock() - - var manip *manipType - var address *tcpip.Address - var portOrIdent *uint16 - if dnat { - manip = &cn.destinationManip - address = &cn.reply.tupleID.srcAddr - portOrIdent = &cn.reply.tupleID.srcPortOrEchoRequestIdent - } else { - manip = &cn.sourceManip - address = &cn.reply.tupleID.dstAddr - portOrIdent = &cn.reply.tupleID.dstPortOrEchoReplyIdent - } - - if *manip != manipNotPerformed { - return - } - *manip = manipPerformed - if changeAddress { - *address = natAddress - } - - // Everything below here is port-fiddling. - if !changePort { - return - } - - // Does the current port/ident fit in the range? - if portsOrIdents.start <= *portOrIdent && *portOrIdent <= lastPortOrIdent { - // Yes, is the current reply tuple unique? - // - // Or, does the reply tuple refer to the same connection as the current one that - // we are NATing? This would apply, for example, to a self-connected socket, - // where the original and reply tuples are identical. - other := cn.ct.connForTID(cn.reply.tupleID) - if other == nil || other.conn == cn { - // Yes! No need to change the port. - return - } - } - - // Try our best to find a port/ident that results in a unique reply tuple. - // - // We limit the number of attempts to find a unique tuple to not waste a lot - // of time looking for a unique tuple. - // - // Matches linux behaviour introduced in - // https://github.com/torvalds/linux/commit/a504b703bb1da526a01593da0e4be2af9d9f5fa8. - const maxAttemptsForInitialRound uint32 = 128 - const minAttemptsToContinue = 16 - - allowedInitialAttempts := maxAttemptsForInitialRound - if allowedInitialAttempts > portsOrIdents.size { - allowedInitialAttempts = portsOrIdents.size - } - - for maxAttempts := allowedInitialAttempts; ; maxAttempts /= 2 { - // Start reach round with a random initial port/ident offset. - randOffset := cn.ct.rand.Uint32() - - for i := uint32(0); i < maxAttempts; i++ { - newPortOrIdentU32 := uint32(portsOrIdents.start) + (randOffset+i)%portsOrIdents.size - if newPortOrIdentU32 > math.MaxUint16 { - panic(fmt.Sprintf("got newPortOrIdentU32 = %d, want <= MaxUint16(=%d); portsOrIdents=%#v, randOffset=%d", newPortOrIdentU32, math.MaxUint16, portsOrIdents, randOffset)) - } - - *portOrIdent = uint16(newPortOrIdentU32) - - if other := cn.ct.connForTID(cn.reply.tupleID); other == nil { - // We found a unique tuple! - return - } - } - - if maxAttempts == portsOrIdents.size { - // We already tried all the ports/idents in the range so no need to keep - // trying. - return - } - - if maxAttempts < minAttemptsToContinue { - return - } - } - - // We did not find a unique tuple, use the last used port anyways. - // TODO(https://gvisor.dev/issue/6850): Handle not finding a unique tuple - // better (e.g. remove the connection and drop the packet). -} - -// handlePacket attempts to handle a packet and perform NAT if the connection -// has had NAT performed on it. -// -// Returns true if the packet can skip the NAT table. -func (cn *conn) handlePacket(pkt *PacketBuffer, hook Hook, rt *Route) bool { - netHdr, transHdr, isICMPError, ok := getHeaders(pkt) - if !ok { - return false - } - - fullChecksum := false - updatePseudoHeader := false - natDone := &pkt.snatDone - dnat := false - switch hook { - case Prerouting: - // Packet came from outside the stack so it must have a checksum set - // already. - fullChecksum = true - updatePseudoHeader = true - - natDone = &pkt.dnatDone - dnat = true - case Input: - case Forward: - panic("should not handle packet in the forwarding hook") - case Output: - natDone = &pkt.dnatDone - dnat = true - fallthrough - case Postrouting: - if pkt.TransportProtocolNumber == header.TCPProtocolNumber && pkt.GSOOptions.Type != GSONone && pkt.GSOOptions.NeedsCsum { - updatePseudoHeader = true - } else if rt.RequiresTXTransportChecksum() { - fullChecksum = true - updatePseudoHeader = true - } - default: - panic(fmt.Sprintf("unrecognized hook = %d", hook)) - } - - if *natDone { - panic(fmt.Sprintf("packet already had NAT(dnat=%t) performed at hook=%s; pkt=%#v", dnat, hook, pkt)) - } - - // TODO(gvisor.dev/issue/5748): TCP checksums on inbound packets should be - // validated if checksum offloading is off. It may require IP defrag if the - // packets are fragmented. - - reply := pkt.tuple.reply - - tid, manip := func() (tupleID, manipType) { - cn.mu.RLock() - defer cn.mu.RUnlock() - - if reply { - tid := cn.original.tupleID - - if dnat { - return tid, cn.sourceManip - } - return tid, cn.destinationManip - } - - tid := cn.reply.tupleID - if dnat { - return tid, cn.destinationManip - } - return tid, cn.sourceManip - }() - switch manip { - case manipNotPerformed: - return false - case manipPerformedNoop: - *natDone = true - return true - case manipPerformed: - default: - panic(fmt.Sprintf("unhandled manip = %d", manip)) - } - - newPort := tid.dstPortOrEchoReplyIdent - newAddr := tid.dstAddr - if dnat { - newPort = tid.srcPortOrEchoRequestIdent - newAddr = tid.srcAddr - } - - rewritePacket( - netHdr, - transHdr, - !dnat != isICMPError, - fullChecksum, - updatePseudoHeader, - newPort, - newAddr, - ) - - *natDone = true - - if !isICMPError { - return true - } - - // We performed NAT on (erroneous) packet that triggered an ICMP response, but - // not the ICMP packet itself. - switch pkt.TransportProtocolNumber { - case header.ICMPv4ProtocolNumber: - icmp := header.ICMPv4(pkt.TransportHeader().Slice()) - // TODO(https://gvisor.dev/issue/6788): Incrementally update ICMP checksum. - icmp.SetChecksum(0) - icmp.SetChecksum(header.ICMPv4Checksum(icmp, pkt.Data().Checksum())) - - network := header.IPv4(pkt.NetworkHeader().Slice()) - if dnat { - network.SetDestinationAddressWithChecksumUpdate(tid.srcAddr) - } else { - network.SetSourceAddressWithChecksumUpdate(tid.dstAddr) - } - case header.ICMPv6ProtocolNumber: - network := header.IPv6(pkt.NetworkHeader().Slice()) - srcAddr := network.SourceAddress() - dstAddr := network.DestinationAddress() - if dnat { - dstAddr = tid.srcAddr - } else { - srcAddr = tid.dstAddr - } - - icmp := header.ICMPv6(pkt.TransportHeader().Slice()) - // TODO(https://gvisor.dev/issue/6788): Incrementally update ICMP checksum. - icmp.SetChecksum(0) - payload := pkt.Data() - icmp.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ - Header: icmp, - Src: srcAddr, - Dst: dstAddr, - PayloadCsum: payload.Checksum(), - PayloadLen: payload.Size(), - })) - - if dnat { - network.SetDestinationAddress(dstAddr) - } else { - network.SetSourceAddress(srcAddr) - } - } - - return true -} - // bucket gets the conntrack bucket for a tupleID. // +checklocksread:ct.mu func (ct *ConnTrack) bucket(id tupleID) int { return ct.bucketWithTableLength(id, len(ct.buckets)) } -func (ct *ConnTrack) bucketWithTableLength(id tupleID, tableLength int) int { - h := jenkins.Sum32(ct.seed) +func tupleHash(id tupleID, seed uint32) uint32 { + h := jenkins.Sum32(seed) h.Write(id.srcAddr.AsSlice()) h.Write(id.dstAddr.AsSlice()) shortBuf := make([]byte, 2) @@ -1027,7 +806,12 @@ func (ct *ConnTrack) bucketWithTableLength(id tupleID, tableLength int) int { h.Write([]byte(shortBuf)) binary.LittleEndian.PutUint16(shortBuf, uint16(id.netProto)) h.Write([]byte(shortBuf)) - return int(h.Sum32()) % tableLength + return h.Sum32() +} + +func (ct *ConnTrack) bucketWithTableLength(id tupleID, tableLength int) int { + h := tupleHash(id, ct.seed) + return int(h) % tableLength } // reapUnused deletes timed out entries from the conntrack map. The rules for @@ -1167,3 +951,53 @@ func (ct *ConnTrack) originalDst(epID TransportEndpointID, netProto tcpip.Networ id := t.conn.original.tupleID return id.dstAddr, id.dstPortOrEchoReplyIdent, nil } + +// NewConnTrack creates and initializes a new ConnTrack object. +func NewConnTrack(clock tcpip.Clock, rng connTrackRNG, seed *uint32) *ConnTrack { + if seed == nil { + r := rng.Uint32() + seed = &r + } + ct := &ConnTrack{ + clock: clock, + rng: rng, + seed: *seed, + nftIDSeed: rng.Uint32(), + } + ct.init() + return ct +} + +// NewConnTrackWithReaper creates and initializes a new ConnTrack and reaper. +// Reaper garbage collects unused connections. +func NewConnTrackWithReaper(clock tcpip.Clock, rng connTrackRNG, seed *uint32) (*ConnTrack, tcpip.Timer) { + ct := NewConnTrack(clock, rng, seed) + var reaper tcpip.Timer + bucket := 0 + interval := 1 * time.Second + reaper = ct.clock.AfterFunc(interval, func() { + bucket, interval = ct.reapUnused(bucket, interval) + reaper.Reset(interval) + }) + return ct, reaper +} + +// NfConnTrackPriority returns the priority of the conntrack hook. +// Check `ipv4/ipv6_conntrack_ops` in nf_conntrack_proto.c. +func NfConnTrackPriority(hook NFHook) (int, bool) { + switch hook { + case NFPrerouting: + // NF_IP_PRI_CONNTRACK + return -200, true + case NFInput: + // NF_IP_PRI_CONNTRACK_CONFIRM + return math.MaxInt32, true + case NFPostrouting: + // NF_IP_PRI_CONNTRACK_CONFIRM + return math.MaxInt32, true + case NFOutput: + // NF_IP_PRI_CONNTRACK + return -200, true + } + return 0, false +} diff --git a/pkg/tcpip/stack/endpoints_by_nic_mutex.go b/pkg/tcpip/stack/endpoints_by_nic_mutex.go index dee401b..06eaf18 100644 --- a/pkg/tcpip/stack/endpoints_by_nic_mutex.go +++ b/pkg/tcpip/stack/endpoints_by_nic_mutex.go @@ -92,5 +92,5 @@ func endpointsByNICinitLockNames() {} func init() { endpointsByNICinitLockNames() - endpointsByNICprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointsByNICRWMutex{}), endpointsByNIClockNames) + endpointsByNICprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointsByNICRWMutex](), endpointsByNIClockNames) } diff --git a/pkg/tcpip/stack/headertype_string.go b/pkg/tcpip/stack/headertype_string.go index cd80de0..bc320a4 100644 --- a/pkg/tcpip/stack/headertype_string.go +++ b/pkg/tcpip/stack/headertype_string.go @@ -30,7 +30,7 @@ func _() { const _headerType_name = "virtioNetHeaderlinkHeadernetworkHeadertransportHeadernumHeaderType" -var _headerType_index = [...]uint8{0, 10, 23, 38, 51} +var _headerType_index = [...]uint8{0, 15, 25, 38, 53, 66} func (i headerType) String() string { if i < 0 || i >= headerType(len(_headerType_index)-1) { diff --git a/pkg/tcpip/stack/icmp_rate_limit.go b/pkg/tcpip/stack/icmp_rate_limit.go index 1b3c3b1..29a0a56 100644 --- a/pkg/tcpip/stack/icmp_rate_limit.go +++ b/pkg/tcpip/stack/icmp_rate_limit.go @@ -15,6 +15,8 @@ package stack import ( + "context" + "github.com/sagernet/gvisor/pkg/tcpip" "golang.org/x/time/rate" ) @@ -34,9 +36,15 @@ const ( // // +stateify savable type ICMPRateLimiter struct { - // TODO(b/341946753): Restore when netstack is savable. limiter *rate.Limiter `state:"nosave"` clock tcpip.Clock + limit rate.Limit + burst int +} + +// afterLoad is invoked by stateify. +func (l *ICMPRateLimiter) afterLoad(context.Context) { + l.limiter = rate.NewLimiter(l.limit, l.burst) } // NewICMPRateLimiter returns a global rate limiter for controlling the rate @@ -46,11 +54,14 @@ func NewICMPRateLimiter(clock tcpip.Clock) *ICMPRateLimiter { return &ICMPRateLimiter{ clock: clock, limiter: rate.NewLimiter(icmpLimit, icmpBurst), + limit: icmpLimit, + burst: icmpBurst, } } // SetLimit sets a new Limit for the limiter. func (l *ICMPRateLimiter) SetLimit(limit rate.Limit) { + l.limit = limit l.limiter.SetLimitAt(l.clock.Now(), limit) } @@ -61,6 +72,7 @@ func (l *ICMPRateLimiter) Limit() rate.Limit { // SetBurst sets a new burst size for the limiter. func (l *ICMPRateLimiter) SetBurst(burst int) { + l.burst = burst l.limiter.SetBurstAt(l.clock.Now(), burst) } diff --git a/pkg/tcpip/stack/iptables.go b/pkg/tcpip/stack/iptables.go index 884abfd..3a7c503 100644 --- a/pkg/tcpip/stack/iptables.go +++ b/pkg/tcpip/stack/iptables.go @@ -33,6 +33,7 @@ const ( NATID TableID = iota MangleID FilterID + RawID NumTables ) @@ -111,6 +112,27 @@ func DefaultTables(clock tcpip.Clock, rand *rand.Rand) *IPTables { Postrouting: HookUnset, }, }, + RawID: { + Rules: []Rule{ + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &AcceptTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + {Filter: EmptyFilter4(), Target: &ErrorTarget{NetworkProtocol: header.IPv4ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: 0, + Input: HookUnset, + Forward: HookUnset, + Output: 1, + Postrouting: HookUnset, + }, + Underflows: [NumHooks]int{ + Prerouting: 0, + Input: HookUnset, + Forward: HookUnset, + Output: 1, + Postrouting: HookUnset, + }, + }, }, v6Tables: [NumTables]Table{ NATID: { @@ -176,11 +198,32 @@ func DefaultTables(clock tcpip.Clock, rand *rand.Rand) *IPTables { Postrouting: HookUnset, }, }, + RawID: { + Rules: []Rule{ + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &AcceptTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + {Filter: EmptyFilter6(), Target: &ErrorTarget{NetworkProtocol: header.IPv6ProtocolNumber}}, + }, + BuiltinChains: [NumHooks]int{ + Prerouting: 0, + Input: HookUnset, + Forward: HookUnset, + Output: 1, + Postrouting: HookUnset, + }, + Underflows: [NumHooks]int{ + Prerouting: 0, + Input: HookUnset, + Forward: HookUnset, + Output: 1, + Postrouting: HookUnset, + }, + }, }, connections: ConnTrack{ seed: rand.Uint32(), clock: clock, - rand: rand, + rng: rand, }, } } @@ -215,6 +258,24 @@ func EmptyNATTable() Table { } } +// EmptyRawTable returns a Table with no rules and only the Prerouting and +// Output hooks set, matching the Linux raw table's valid hooks. +func EmptyRawTable() Table { + return Table{ + Rules: []Rule{}, + BuiltinChains: [NumHooks]int{ + Input: HookUnset, + Forward: HookUnset, + Postrouting: HookUnset, + }, + Underflows: [NumHooks]int{ + Input: HookUnset, + Forward: HookUnset, + Postrouting: HookUnset, + }, + } +} + // GetTable returns a table with the given id and IP version. It panics when an // invalid id is provided. func (it *IPTables) GetTable(id TableID, ipv6 bool) Table { @@ -338,6 +399,10 @@ func (it *IPTables) shouldSkipOrPopulateTables(tables []checkTable, pkt *PacketB // +checkescape func (it *IPTables) CheckPrerouting(pkt *PacketBuffer, addressEP AddressableEndpoint, inNicName string) bool { tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: check, + tableID: RawID, + }, { fn: check, tableID: MangleID, @@ -448,6 +513,10 @@ func (it *IPTables) CheckForward(pkt *PacketBuffer, inNicName, outNicName string // +checkescape func (it *IPTables) CheckOutput(pkt *PacketBuffer, r *Route, outNicName string) bool { tables := [...]checkTable{ // escapes: on arm this causes an allocation. + { + fn: check, + tableID: RawID, + }, { fn: check, tableID: MangleID, @@ -532,7 +601,7 @@ func checkNAT(it *IPTables, table Table, hook Hook, pkt *PacketBuffer, r *Route, // See check. func (it *IPTables) checkNAT(table Table, hook Hook, pkt *PacketBuffer, r *Route, addressEP AddressableEndpoint, inNicName, outNicName string) bool { t := pkt.tuple - if t != nil && t.conn.handlePacket(pkt, hook, r) { + if t != nil && IPTHandlePacket(pkt, hook, r) { return true } @@ -561,7 +630,7 @@ func (it *IPTables) checkNAT(table Table, hook Hook, pkt *PacketBuffer, r *Route // // If the packet was already NATed, the connection must be NATed. if !natDone { - t.conn.maybePerformNoopNAT(pkt, hook, r, dnat) + IPTMaybePerformNoopNAT(pkt, hook, r, dnat) } return true diff --git a/pkg/tcpip/stack/iptables_mutex.go b/pkg/tcpip/stack/iptables_mutex.go index 5e6a78f..94ace0c 100644 --- a/pkg/tcpip/stack/iptables_mutex.go +++ b/pkg/tcpip/stack/iptables_mutex.go @@ -92,5 +92,5 @@ func ipTablesinitLockNames() {} func init() { ipTablesinitLockNames() - ipTablesprefixIndex = locking.NewMutexClass(reflect.TypeOf(ipTablesRWMutex{}), ipTableslockNames) + ipTablesprefixIndex = locking.NewMutexClass(reflect.TypeFor[ipTablesRWMutex](), ipTableslockNames) } diff --git a/pkg/tcpip/stack/iptables_targets.go b/pkg/tcpip/stack/iptables_targets.go index 575c411..ff89607 100644 --- a/pkg/tcpip/stack/iptables_targets.go +++ b/pkg/tcpip/stack/iptables_targets.go @@ -64,8 +64,11 @@ const ( RejectIPv4WithICMPNetUnreachable RejectIPv4WithICMPHostUnreachable RejectIPv4WithICMPPortUnreachable + RejectIPv4WithICMPProtUnreachable + RejectIPv4WithICMPEchoReply RejectIPv4WithICMPNetProhibited RejectIPv4WithICMPHostProhibited + RejectIPv4WithTCPReset RejectIPv4WithICMPAdminProhibited ) @@ -106,9 +109,14 @@ type RejectIPv6WithICMPType int const ( _ RejectIPv6WithICMPType = iota RejectIPv6WithICMPNoRoute + RejectIPv6WithICMPAdminProhibited + RejectIPv6WithICMPNotNeighbour RejectIPv6WithICMPAddrUnreachable RejectIPv6WithICMPPortUnreachable - RejectIPv6WithICMPAdminProhibited + RejectIPv6WithICMPEchoReply + RejectIPv6WithTCPReset + RejectIPv6WithICMPPolicyFail + RejectIPv6WithICMPRejectRoute ) // RejectIPv6Target drops packets and sends back an error packet in response to the @@ -297,10 +305,10 @@ type SNATTarget struct { } func dnatAction(pkt *PacketBuffer, hook Hook, r *Route, port uint16, address tcpip.Address, changePort, changeAddress bool) (RuleVerdict, int) { - return natAction(pkt, hook, r, portOrIdentRange{start: port, size: 1}, address, true /* dnat */, changePort, changeAddress) + return natAction(pkt, hook, r, PortOrIdentRange{Start: port, Size: 1}, address, true /* dnat */, changePort, changeAddress) } -func targetPortRangeForTCPAndUDP(originalSrcPort uint16) portOrIdentRange { +func targetPortRangeForTCPAndUDP(originalSrcPort uint16) PortOrIdentRange { // As per iptables(8), // // If no port range is specified, then source ports below 512 will be @@ -309,16 +317,16 @@ func targetPortRangeForTCPAndUDP(originalSrcPort uint16) portOrIdentRange { // 1024 or above. switch { case originalSrcPort < 512: - return portOrIdentRange{start: 1, size: 511} + return PortOrIdentRange{Start: 1, Size: 511} case originalSrcPort < 1024: - return portOrIdentRange{start: 1, size: 1023} + return PortOrIdentRange{Start: 1, Size: 1023} default: - return portOrIdentRange{start: 1024, size: math.MaxUint16 - 1023} + return PortOrIdentRange{Start: 1024, Size: math.MaxUint16 - 1023} } } func snatAction(pkt *PacketBuffer, hook Hook, r *Route, port uint16, address tcpip.Address, changePort, changeAddress bool) (RuleVerdict, int) { - portsOrIdents := portOrIdentRange{start: port, size: 1} + portsOrIdents := PortOrIdentRange{Start: port, Size: 1} switch pkt.TransportProtocolNumber { case header.UDPProtocolNumber: @@ -334,20 +342,20 @@ func snatAction(pkt *PacketBuffer, hook Hook, r *Route, port uint16, address tcp // behaviour. // // https://github.com/torvalds/linux/blob/58e1100fdc5990b0cc0d4beaf2562a92e621ac7d/net/netfilter/nf_nat_core.c#L391 - portsOrIdents = portOrIdentRange{start: 0, size: math.MaxUint16 + 1} + portsOrIdents = PortOrIdentRange{Start: 0, Size: math.MaxUint16 + 1} } return natAction(pkt, hook, r, portsOrIdents, address, false /* dnat */, changePort, changeAddress) } -func natAction(pkt *PacketBuffer, hook Hook, r *Route, portsOrIdents portOrIdentRange, address tcpip.Address, dnat, changePort, changeAddress bool) (RuleVerdict, int) { +func natAction(pkt *PacketBuffer, hook Hook, r *Route, portsOrIdents PortOrIdentRange, address tcpip.Address, dnat, changePort, changeAddress bool) (RuleVerdict, int) { // Drop the packet if network and transport header are not set. if len(pkt.NetworkHeader().Slice()) == 0 || len(pkt.TransportHeader().Slice()) == 0 { return RuleDrop, 0 } if t := pkt.tuple; t != nil { - t.conn.performNAT(pkt, hook, r, portsOrIdents, address, dnat, changePort, changeAddress) + IPTPerformNAT(pkt, hook, r, portsOrIdents, address, dnat, changePort, changeAddress) return RuleAccept, 0 } @@ -412,81 +420,22 @@ func (mt *MasqueradeTarget) Action(pkt *PacketBuffer, hook Hook, r *Route, addre return snatAction(pkt, hook, r, 0 /* port */, address, true /* changePort */, true /* changeAddress */) } -func rewritePacket(n header.Network, t header.Transport, updateSRCFields, fullChecksum, updatePseudoHeader bool, newPortOrIdent uint16, newAddr tcpip.Address) { - switch t := t.(type) { - case header.ChecksummableTransport: - if updateSRCFields { - if fullChecksum { - t.SetSourcePortWithChecksumUpdate(newPortOrIdent) - } else { - t.SetSourcePort(newPortOrIdent) - } - } else { - if fullChecksum { - t.SetDestinationPortWithChecksumUpdate(newPortOrIdent) - } else { - t.SetDestinationPort(newPortOrIdent) - } - } +// CTTarget is a no-op implementation of the CT (conntrack) target used in the +// raw table. In Linux, CT --zone sets conntrack zones for connection tracking +// isolation. gVisor's conntrack does not support zones, so this target simply +// accepts the packet, allowing iptables-restore to load rulesets that reference +// CT targets (e.g. Istio with DNS capture enabled). +// +// +stateify savable +type CTTarget struct { + // NetworkProtocol is the network protocol the target is used with. + NetworkProtocol tcpip.NetworkProtocolNumber - if updatePseudoHeader { - var oldAddr tcpip.Address - if updateSRCFields { - oldAddr = n.SourceAddress() - } else { - oldAddr = n.DestinationAddress() - } - - t.UpdateChecksumPseudoHeaderAddress(oldAddr, newAddr, fullChecksum) - } - case header.ICMPv4: - switch icmpType := t.Type(); icmpType { - case header.ICMPv4Echo: - if updateSRCFields { - t.SetIdentWithChecksumUpdate(newPortOrIdent) - } - case header.ICMPv4EchoReply: - if !updateSRCFields { - t.SetIdentWithChecksumUpdate(newPortOrIdent) - } - default: - panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) - } - case header.ICMPv6: - switch icmpType := t.Type(); icmpType { - case header.ICMPv6EchoRequest: - if updateSRCFields { - t.SetIdentWithChecksumUpdate(newPortOrIdent) - } - case header.ICMPv6EchoReply: - if !updateSRCFields { - t.SetIdentWithChecksumUpdate(newPortOrIdent) - } - default: - panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) - } - - var oldAddr tcpip.Address - if updateSRCFields { - oldAddr = n.SourceAddress() - } else { - oldAddr = n.DestinationAddress() - } - - t.UpdateChecksumPseudoHeaderAddress(oldAddr, newAddr) - default: - panic(fmt.Sprintf("unhandled transport = %#v", t)) - } - - if checksummableNetHeader, ok := n.(header.ChecksummableNetwork); ok { - if updateSRCFields { - checksummableNetHeader.SetSourceAddressWithChecksumUpdate(newAddr) - } else { - checksummableNetHeader.SetDestinationAddressWithChecksumUpdate(newAddr) - } - } else if updateSRCFields { - n.SetSourceAddress(newAddr) - } else { - n.SetDestinationAddress(newAddr) - } + // Zone is the conntrack zone ID. Stored but not acted upon. + Zone uint16 +} + +// Action implements Target.Action. It is a no-op that accepts the packet. +func (*CTTarget) Action(*PacketBuffer, Hook, *Route, AddressableEndpoint) (RuleVerdict, int) { + return RuleAccept, 0 } diff --git a/pkg/tcpip/stack/iptables_types.go b/pkg/tcpip/stack/iptables_types.go index 500d218..fd7970e 100644 --- a/pkg/tcpip/stack/iptables_types.go +++ b/pkg/tcpip/stack/iptables_types.go @@ -82,7 +82,7 @@ const ( type IPTables struct { connections ConnTrack - reaper tcpip.Timer + reaper tcpip.Timer `state:"nosave"` mu ipTablesRWMutex `state:"nosave"` // v4Tables and v6tables map tableIDs to tables. They hold builtin @@ -283,7 +283,9 @@ func (fl IPHeaderFilter) match(pkt *PacketBuffer, hook Hook, inNicName, outNicNa case header.IPv6ProtocolNumber: hdr := header.IPv6(pkt.NetworkHeader().Slice()) - transProto = hdr.TransportProtocol() + // The transport protocol may be preceded by IPv6 extension headers, so + // use the protocol from parsing (see IPv6.TransportProtocol). + transProto = pkt.TransportProtocolNumber dstAddr = hdr.DestinationAddress() srcAddr = hdr.SourceAddress() diff --git a/pkg/tcpip/stack/multi_port_endpoint_mutex.go b/pkg/tcpip/stack/multi_port_endpoint_mutex.go index 9e73cfb..77a1492 100644 --- a/pkg/tcpip/stack/multi_port_endpoint_mutex.go +++ b/pkg/tcpip/stack/multi_port_endpoint_mutex.go @@ -92,5 +92,5 @@ func multiPortEndpointinitLockNames() {} func init() { multiPortEndpointinitLockNames() - multiPortEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(multiPortEndpointRWMutex{}), multiPortEndpointlockNames) + multiPortEndpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[multiPortEndpointRWMutex](), multiPortEndpointlockNames) } diff --git a/pkg/tcpip/stack/nat.go b/pkg/tcpip/stack/nat.go new file mode 100644 index 0000000..61f8866 --- /dev/null +++ b/pkg/tcpip/stack/nat.go @@ -0,0 +1,581 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stack + +import ( + "fmt" + "math" + + "github.com/sagernet/gvisor/pkg/log" + "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/header" +) + +// NATType represents the type of NAT. +type NATType int + +const ( + // SNAT is source NAT. + SNAT NATType = iota + // DNAT is destination NAT. + DNAT + // NATUnknown is unknown NAT type. + NATUnknown +) + +// ToNATType converts a uint8 to a NATType. +func ToNATType(t uint8) NATType { + switch t { + case 0: + return SNAT + case 1: + return DNAT + } + return NATUnknown +} + +func (natType NATType) String() string { + switch natType { + case SNAT: + return "SNAT" + case DNAT: + return "DNAT" + default: + return "NATUnknown" + } +} + +// NfNATPriority returns the priority of the NAT hook. +// Check `ipv4/ipv6_nat_ops` in nf_nat_proto.c. +func NfNATPriority(hook NFHook) (int, bool) { + switch hook { + case NFPrerouting: + // NF_IP_PRI_NAT_DST + return -100, true + case NFPostrouting: + // NF_IP_PRI_NAT_SRC + return 100, true + case NFOutput: + // NF_IP_PRI_NAT_DST + return -100, true + case NFInput: + // NF_IP_PRI_NAT_SRC + return 100, true + } + // NAT is not supported for other hooks. + return 0, false +} + +// NfHookToNATType returns the applicable NAT type +// for the given netfilter hook. +func NfHookToNATType(hook NFHook) NATType { + switch hook { + case NFPrerouting, NFOutput: + return DNAT + case NFInput, NFPostrouting: + return SNAT + } + return NATUnknown +} + +// handlePacketOpts contains the options for handlePacket. +type handlePacketOpts struct { + fullChecksum bool + updatePseudoHeader bool + natType NATType +} + +// handlePacket attempts to handle a packet and perform NAT if the connection +// has had NAT performed on it. +// +// Returns true if the packet can skip the NAT table. +func handlePacket(pkt *PacketBuffer, opts *handlePacketOpts) bool { + if opts == nil || opts.natType == NATUnknown { + return false + } + netHdr, transHdr, isICMPError, ok := pkt.GetHeaders() + if !ok { + return false + } + + natDone := &pkt.snatDone + dnat := false + if opts.natType == DNAT { + natDone = &pkt.dnatDone + dnat = true + } + + if *natDone { + panic(fmt.Sprintf("packet already had NAT: %s performed; pkt=%#v", opts.natType, pkt)) + } + + // TODO(gvisor.dev/issue/5748): TCP checksums on inbound packets should be + // validated if checksum offloading is off. It may require IP defrag if the + // packets are fragmented. + + reply := pkt.tuple.reply + cn := pkt.tuple.conn + + tid, manip := func() (tupleID, manipType) { + cn.mu.RLock() + defer cn.mu.RUnlock() + + if reply { + tid := cn.original.tupleID + + if dnat { + return tid, cn.sourceManip + } + return tid, cn.destinationManip + } + + tid := cn.reply.tupleID + if dnat { + return tid, cn.destinationManip + } + return tid, cn.sourceManip + }() + switch manip { + case manipNotPerformed: + return false + case manipPerformedNoop: + *natDone = true + return true + case manipPerformed: + default: + panic(fmt.Sprintf("unhandled manip = %d", manip)) + } + + newPort := tid.dstPortOrEchoReplyIdent + newAddr := tid.dstAddr + if dnat { + newPort = tid.srcPortOrEchoRequestIdent + newAddr = tid.srcAddr + } + + UpdateHeaders( + netHdr, + transHdr, + !dnat != isICMPError, + opts.fullChecksum, + opts.updatePseudoHeader, + newPort, + newAddr, + ) + + *natDone = true + + if !isICMPError { + return true + } + + // We performed NAT on (erroneous) packet that triggered an ICMP response, but + // not the ICMP packet itself. + switch pkt.TransportProtocolNumber { + case header.ICMPv4ProtocolNumber: + icmp := header.ICMPv4(pkt.TransportHeader().Slice()) + // TODO(https://gvisor.dev/issue/6788): Incrementally update ICMP checksum. + icmp.SetChecksum(0) + icmp.SetChecksum(header.ICMPv4Checksum(icmp, pkt.Data().Checksum())) + + network := header.IPv4(pkt.NetworkHeader().Slice()) + if dnat { + network.SetDestinationAddressWithChecksumUpdate(tid.srcAddr) + } else { + network.SetSourceAddressWithChecksumUpdate(tid.dstAddr) + } + case header.ICMPv6ProtocolNumber: + network := header.IPv6(pkt.NetworkHeader().Slice()) + srcAddr := network.SourceAddress() + dstAddr := network.DestinationAddress() + if dnat { + dstAddr = tid.srcAddr + } else { + srcAddr = tid.dstAddr + } + + icmp := header.ICMPv6(pkt.TransportHeader().Slice()) + // TODO(https://gvisor.dev/issue/6788): Incrementally update ICMP checksum. + icmp.SetChecksum(0) + payload := pkt.Data() + icmp.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmp, + Src: srcAddr, + Dst: dstAddr, + PayloadCsum: payload.Checksum(), + PayloadLen: payload.Size(), + })) + + if dnat { + network.SetDestinationAddress(dstAddr) + } else { + network.SetSourceAddress(srcAddr) + } + } + + return true +} + +// IPTHandlePacket handles and applies NAT to the packet if required. +func IPTHandlePacket(pkt *PacketBuffer, hook Hook, r *Route) bool { + opts := handlePacketOpts{ + fullChecksum: false, + updatePseudoHeader: false, + natType: SNAT, + } + requiresTXTransportChecksum := false + if r != nil { + requiresTXTransportChecksum = r.RequiresTXTransportChecksum() + } + switch hook { + case Prerouting: + opts.fullChecksum = true + opts.updatePseudoHeader = true + opts.natType = DNAT + case Input: + case Forward: + panic("should not handle packet in the forwarding hook") + case Output: + opts.natType = DNAT + fallthrough + case Postrouting: + if pkt.TransportProtocolNumber == header.TCPProtocolNumber && pkt.GSOOptions.Type != GSONone && pkt.GSOOptions.NeedsCsum { + opts.updatePseudoHeader = true + } else if requiresTXTransportChecksum { + opts.fullChecksum = true + opts.updatePseudoHeader = true + } + default: + panic(fmt.Sprintf("unrecognized hook = %d", hook)) + } + + return handlePacket(pkt, &opts) +} + +// PortOrIdentRange represents a range of ports or idents +// range to use for NAT. +type PortOrIdentRange struct { + Start uint16 + Size uint32 +} + +// ConfigureNAT setups up the connection for the specified NAT and rewrites the +// packet. +// +// If NAT has already been performed on the connection, then the packet will +// be rewritten with the NAT performed on the connection, ignoring the passed +// address and port range. +// +// Generally, only the first packet of a connection reaches this method; other +// packets will be manipulated without needing to modify the connection. +// +// Returns whether the NAT was configured or not. +func (cn *conn) ConfigureNAT(portsOrIdents PortOrIdentRange, natAddress tcpip.Address, natType NATType, changePort, changeAddress bool) bool { + lastPortOrIdentU32 := uint32(portsOrIdents.Start) + portsOrIdents.Size - 1 + if lastPortOrIdentU32 > math.MaxUint16 { + log.Warningf("got lastPortOrIdent = %d, want <= MaxUint16(=%d); portsOrIdents=%#v", lastPortOrIdentU32, math.MaxUint16, portsOrIdents) + return false + } + lastPortOrIdent := uint16(lastPortOrIdentU32) + + cn.mu.Lock() + defer cn.mu.Unlock() + + var manip *manipType + var address *tcpip.Address + var portOrIdent *uint16 + if natType == DNAT { + manip = &cn.destinationManip + address = &cn.reply.tupleID.srcAddr + portOrIdent = &cn.reply.tupleID.srcPortOrEchoRequestIdent + } else { + manip = &cn.sourceManip + address = &cn.reply.tupleID.dstAddr + portOrIdent = &cn.reply.tupleID.dstPortOrEchoReplyIdent + } + + if *manip != manipNotPerformed { + return true + } + *manip = manipPerformed + if changeAddress { + *address = natAddress + } + + // Everything below here is port-fiddling. + if !changePort { + return true + } + + // Does the current port/ident fit in the range? + if portsOrIdents.Start <= *portOrIdent && *portOrIdent <= lastPortOrIdent { + // Yes, is the current reply tuple unique? + // + // Or, does the reply tuple refer to the same connection as the current one that + // we are NATing? This would apply, for example, to a self-connected socket, + // where the original and reply tuples are identical. + other := cn.ct.connForTID(cn.reply.tupleID) + if other == nil || other.conn == cn { + // Yes! No need to change the port. + return true + } + } + + // Try our best to find a port/ident that results in a unique reply tuple. + // + // We limit the number of attempts to find a unique tuple to not waste a lot + // of time looking for a unique tuple. + // + // Matches linux behaviour introduced in + // https://github.com/torvalds/linux/commit/a504b703bb1da526a01593da0e4be2af9d9f5fa8. + const maxAttemptsForInitialRound uint32 = 128 + const minAttemptsToContinue = 16 + + allowedInitialAttempts := maxAttemptsForInitialRound + if allowedInitialAttempts > portsOrIdents.Size { + allowedInitialAttempts = portsOrIdents.Size + } + + for maxAttempts := allowedInitialAttempts; ; maxAttempts /= 2 { + // Start reach round with a random initial port/ident offset. + randOffset := cn.ct.rng.Uint32() + + for i := uint32(0); i < maxAttempts; i++ { + newPortOrIdentU32 := uint32(portsOrIdents.Start) + (randOffset+i)%portsOrIdents.Size + if newPortOrIdentU32 > math.MaxUint16 { + log.Warningf("got newPortOrIdentU32 = %d, want <= MaxUint16(=%d); portsOrIdents=%#v", newPortOrIdentU32, math.MaxUint16, portsOrIdents) + continue + } + + *portOrIdent = uint16(newPortOrIdentU32) + + if other := cn.ct.connForTID(cn.reply.tupleID); other == nil { + // We found a unique tuple! + return true + } + } + + if maxAttempts == portsOrIdents.Size { + // We already tried all the ports/idents in the range so no need to keep + // trying. + return false + } + + if maxAttempts < minAttemptsToContinue { + return false + } + } + + // We did not find a unique tuple, use the last used port anyways. + // TODO(https://gvisor.dev/issue/6850): Handle not finding a unique tuple + // better (e.g. remove the connection and drop the packet). +} + +// IPTPerformNAT performs NAT on the packet and updates the connection. +// Used by IPTables. +func IPTPerformNAT(pkt *PacketBuffer, hook Hook, r *Route, portsOrIdents PortOrIdentRange, natAddress tcpip.Address, dnat, changePort, changeAddress bool) { + // Make sure the packet is re-written after performing NAT. + defer func() { + // handlePacket returns true if the packet may skip the NAT table as the + // connection is already NATed, but if we reach this point we must be in the + // NAT table, so the return value is useless for us. + _ = IPTHandlePacket(pkt, hook, r) + }() + cn := pkt.tuple.conn + natType := SNAT + if dnat { + natType = DNAT + } + _ = cn.ConfigureNAT(portsOrIdents, natAddress, natType, changePort, changeAddress) +} + +// IPTMaybePerformNoopNAT can apply NAT or configure a no-op NAT. +// If NAT has not been configured for this connection, either mark the +// connection as configured for "no-op NAT", in the case of DNAT, or, in the +// case of SNAT, perform source port remapping so that source ports used by +// locally-generated traffic do not conflict with ports occupied by existing NAT +// bindings. +// +// Note that in the typical case this is also a no-op, because `snatAction` +// will do nothing if the original tuple is already unique. +func IPTMaybePerformNoopNAT(pkt *PacketBuffer, hook Hook, r *Route, dnat bool) { + cn := pkt.tuple.conn + cn.mu.Lock() + var manip *manipType + if dnat { + manip = &cn.destinationManip + } else { + manip = &cn.sourceManip + } + if *manip != manipNotPerformed { + cn.mu.Unlock() + _ = IPTHandlePacket(pkt, hook, r) + return + } + if dnat { + *manip = manipPerformedNoop + cn.mu.Unlock() + _ = IPTHandlePacket(pkt, hook, r) + return + } + cn.mu.Unlock() + + // At this point, we know that NAT has not yet been performed on this + // connection, and the DNAT case has been handled with a no-op. For SNAT, we + // simply perform source port remapping to ensure that source ports for + // locally generated traffic do not clash with ports used by existing NAT + // bindings. + _, _ = snatAction(pkt, hook, r, 0, tcpip.Address{}, true /* changePort */, false /* changeAddress */) +} + +// NFTApplyNAT applies NAT to the packet and updates the connection. +// Similar to IPTHandlePacket but for NFTables hooks. +func NFTApplyNAT(pkt *PacketBuffer, hook NFHook, rt *Route) bool { + requiresTXTransportChecksum := false + if rt != nil { + requiresTXTransportChecksum = rt.RequiresTXTransportChecksum() + } + opts := handlePacketOpts{ + fullChecksum: false, + updatePseudoHeader: false, + natType: SNAT, + } + switch hook { + case NFPrerouting: + opts.fullChecksum = true + opts.updatePseudoHeader = true + opts.natType = DNAT + case NFInput: + case NFForward: + panic("should not handle packet in the forwarding hook") + case NFOutput: + opts.natType = DNAT + fallthrough + case NFPostrouting: + if pkt.TransportProtocolNumber == header.TCPProtocolNumber && pkt.GSOOptions.Type != GSONone && pkt.GSOOptions.NeedsCsum { + opts.updatePseudoHeader = true + } else if requiresTXTransportChecksum { + opts.fullChecksum = true + opts.updatePseudoHeader = true + } + default: + panic(fmt.Sprintf("unrecognized hook = %d", hook)) + } + + return handlePacket(pkt, &opts) +} + +// IsNATConfigured returns whether NAT has been configured for the given NAT type. +func (cn *conn) IsNATConfigured(natType NATType) bool { + cn.mu.RLock() + defer cn.mu.RUnlock() + switch natType { + case SNAT: + return cn.sourceManip != manipNotPerformed + case DNAT: + return cn.destinationManip != manipNotPerformed + } + return false +} + +// ConfigureNoopNAT configures the connection for no-op NAT. +// Similar to the func `IPTMaybePerformNoopNAT` except that this one only configures NO-OP NAT and is independent of IPTables. +func (cn *conn) ConfigureNoopNAT(pkt *PacketBuffer, natType NATType) bool { + cn.mu.Lock() + var manip *manipType + if natType == DNAT { + manip = &cn.destinationManip + } else { + manip = &cn.sourceManip + } + + if *manip != manipNotPerformed { + cn.mu.Unlock() + return true + } + + if natType == DNAT { + *manip = manipPerformedNoop + cn.mu.Unlock() + return true + } + cn.mu.Unlock() + + // At this point, we know that NAT has not yet been performed on this + // connection, and the DNAT case has been handled with a no-op. For SNAT, we + // simply perform source port remapping to ensure that source ports for + // locally generated traffic do not clash with ports used by existing NAT + // bindings. + + portsOrIdents := PortOrIdentRange{Start: 0, Size: math.MaxUint16 + 1} + + // However, we need to extract the port from packet. + var port uint16 + switch pkt.TransportProtocolNumber { + case header.UDPProtocolNumber: + port = header.UDP(pkt.TransportHeader().Slice()).SourcePort() + case header.TCPProtocolNumber: + port = header.TCP(pkt.TransportHeader().Slice()).SourcePort() + } + + if port != 0 { + portsOrIdents = targetPortRangeForTCPAndUDP(port) + } + + return cn.ConfigureNAT(portsOrIdents, tcpip.Address{}, natType, true /* changePort */, false /* changeAddress */) +} + +// ConfigureMasquerade configures the connection for masquerade. +func (cn *conn) configureMasquerade(pkt *PacketBuffer, route *Route, stk *Stack, ports PortOrIdentRange, changePort bool) bool { + srcAddr := pkt.Network().SourceAddress() + if srcAddr == header.IPv4Any || srcAddr == header.IPv6Any { + return false + } + // Masquerade is only supported for postrouting. + if route == nil { + return false + } + + // Get the network endpoint for the outgoing interface to find its primary address. + netEP, err := stk.GetNetworkEndpoint(route.NICID(), route.NetProto()) + if err != nil { + return false + } + + addressEP, ok := netEP.(AddressableEndpoint) + if !ok { + return false + } + + // Ref: net/netfilter/nf_nat_masquerade.c:nf_nat_masquerade_ipv[4|6]() + // Use the next hop address as the destination address if it is set. + nh := route.NextHop() + if nh.Len() == 0 { + nh = pkt.Network().DestinationAddress() + } + + // addressEP is expected to be set for the postrouting hook. + // Find the outgoing primary address for the destination address. + ep := addressEP.AcquireOutgoingPrimaryAddress(nh, tcpip.Address{} /* srcHint */, false /* allowExpired */) + if ep == nil { + // No address exists that we can use as a source address. + return false + } + address := ep.AddressWithPrefix().Address + ep.DecRef() + + // Configure NAT for the packet to change the source address. + return cn.ConfigureNAT(ports, address, SNAT, changePort, true /* changeAddress */) +} diff --git a/pkg/tcpip/stack/neighbor_cache_mutex.go b/pkg/tcpip/stack/neighbor_cache_mutex.go index 8d5a9e4..451e135 100644 --- a/pkg/tcpip/stack/neighbor_cache_mutex.go +++ b/pkg/tcpip/stack/neighbor_cache_mutex.go @@ -92,5 +92,5 @@ func neighborCacheinitLockNames() {} func init() { neighborCacheinitLockNames() - neighborCacheprefixIndex = locking.NewMutexClass(reflect.TypeOf(neighborCacheRWMutex{}), neighborCachelockNames) + neighborCacheprefixIndex = locking.NewMutexClass(reflect.TypeFor[neighborCacheRWMutex](), neighborCachelockNames) } diff --git a/pkg/tcpip/stack/neighbor_entry.go b/pkg/tcpip/stack/neighbor_entry.go index 9a34bf6..689d771 100644 --- a/pkg/tcpip/stack/neighbor_entry.go +++ b/pkg/tcpip/stack/neighbor_entry.go @@ -578,7 +578,7 @@ func (e *neighborEntry) handleConfirmationLocked(linkAddr tcpip.LinkAddress, fla // here. ep := e.cache.nic.getNetworkEndpoint(header.IPv6ProtocolNumber) if ep == nil { - panic(fmt.Sprintf("have a neighbor entry for an IPv6 router but no IPv6 network endpoint")) + panic("have a neighbor entry for an IPv6 router but no IPv6 network endpoint") } if ndpEP, ok := ep.(NDPEndpoint); ok { diff --git a/pkg/tcpip/stack/neighbor_entry_mutex.go b/pkg/tcpip/stack/neighbor_entry_mutex.go index 3f28901..276a4ca 100644 --- a/pkg/tcpip/stack/neighbor_entry_mutex.go +++ b/pkg/tcpip/stack/neighbor_entry_mutex.go @@ -92,5 +92,5 @@ func neighborEntryinitLockNames() {} func init() { neighborEntryinitLockNames() - neighborEntryprefixIndex = locking.NewMutexClass(reflect.TypeOf(neighborEntryRWMutex{}), neighborEntrylockNames) + neighborEntryprefixIndex = locking.NewMutexClass(reflect.TypeFor[neighborEntryRWMutex](), neighborEntrylockNames) } diff --git a/pkg/tcpip/stack/nftables_types.go b/pkg/tcpip/stack/nftables_types.go index 113d8f1..f75732a 100644 --- a/pkg/tcpip/stack/nftables_types.go +++ b/pkg/tcpip/stack/nftables_types.go @@ -20,13 +20,13 @@ import ( // NFTablesInterface is an interface for evaluating chains. type NFTablesInterface interface { - CheckPrerouting(pkt *PacketBuffer, af AddressFamily) bool - CheckInput(pkt *PacketBuffer, af AddressFamily) bool - CheckForward(pkt *PacketBuffer, af AddressFamily) bool - CheckOutput(pkt *PacketBuffer, af AddressFamily) bool - CheckPostrouting(pkt *PacketBuffer, af AddressFamily) bool - CheckIngress(pkt *PacketBuffer, af AddressFamily) bool - CheckEgress(pkt *PacketBuffer, af AddressFamily) bool + CheckPrerouting(pkt *PacketBuffer, route *Route, af AddressFamily) bool + CheckInput(pkt *PacketBuffer, route *Route, af AddressFamily) bool + CheckForward(pkt *PacketBuffer, route *Route, af AddressFamily) bool + CheckOutput(pkt *PacketBuffer, route *Route, af AddressFamily) bool + CheckPostrouting(pkt *PacketBuffer, route *Route, af AddressFamily) bool + CheckIngress(pkt *PacketBuffer, route *Route, af AddressFamily) bool + CheckEgress(pkt *PacketBuffer, route *Route, af AddressFamily) bool } // NFHook describes specific points in the pipeline where chains can be attached. @@ -147,24 +147,3 @@ func (f AddressFamily) String() string { } panic(fmt.Sprintf("invalid address family: %d", int(f))) } - -// -// Verdict Implementation. -// There are two types of verdicts: -// 1. Netfilter (External) Verdicts: Drop, Accept, Stolen, Queue, Repeat, Stop -// These are terminal verdicts that are returned to the kernel. -// 2. Nftable (Internal) Verdicts:, Continue, Break, Jump, Goto, Return -// These are internal verdicts that only exist within the nftables library. -// Both share the same numeric space (uint32 Verdict Code). -// - -// NFVerdict represents the result of evaluating a packet against a rule or chain. -type NFVerdict struct { - // Code is the numeric code that represents the verdict issued. - Code uint32 - - // ChainName is the name of the chain to continue evaluation if the verdict is - // Jump or Goto. - // Note: the chain must be in the same table as the current chain. - ChainName string -} diff --git a/pkg/tcpip/stack/nic.go b/pkg/tcpip/stack/nic.go index 63e3c51..838eb83 100644 --- a/pkg/tcpip/stack/nic.go +++ b/pkg/tcpip/stack/nic.go @@ -316,7 +316,7 @@ func (n *nic) enable() tcpip.Error { // resources. This guarantees no packets between this NIC and the network // stack. // -// It returns an action that has to be excuted after releasing the Stack lock +// It returns an action that has to be executed after releasing the Stack lock // and any error encountered. func (n *nic) remove(closeLinkEndpoint bool) (func(), tcpip.Error) { n.enableDisableMu.Lock() @@ -843,23 +843,34 @@ func (n *nic) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *Packe // DeliverTransportPacket delivers the packets to the appropriate transport // protocol endpoint. func (n *nic) DeliverTransportPacket(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) TransportPacketDisposition { + res, _ := n.deliverTransportPacket(protocol, pkt) + return res +} + +// DeliverTransportPacketWithDefaultHandlerResult implements +// TransportDispatcherWithDefaultHandlerResult.DeliverTransportPacketWithDefaultHandlerResult. +func (n *nic) DeliverTransportPacketWithDefaultHandlerResult(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) (TransportPacketDisposition, bool) { + return n.deliverTransportPacket(protocol, pkt) +} + +func (n *nic) deliverTransportPacket(protocol tcpip.TransportProtocolNumber, pkt *PacketBuffer) (TransportPacketDisposition, bool) { state, ok := n.stack.transportProtocols[protocol] if !ok { n.stats.unknownL4ProtocolRcvdPacketCounts.Increment(uint64(protocol)) - return TransportPacketProtocolUnreachable + return TransportPacketProtocolUnreachable, false } transProto := state.proto if len(pkt.TransportHeader().Slice()) == 0 { n.stats.malformedL4RcvdPackets.Increment() - return TransportPacketHandled + return TransportPacketHandled, false } srcPort, dstPort, err := transProto.ParsePorts(pkt.TransportHeader().Slice()) if err != nil { n.stats.malformedL4RcvdPackets.Increment() - return TransportPacketHandled + return TransportPacketHandled, false } netProto, ok := n.stack.networkProtocols[pkt.NetworkProtocolNumber] @@ -875,13 +886,13 @@ func (n *nic) DeliverTransportPacket(protocol tcpip.TransportProtocolNumber, pkt RemoteAddress: src, } if n.stack.demux.deliverPacket(protocol, pkt, id) { - return TransportPacketHandled + return TransportPacketHandled, false } // Try to deliver to per-stack default handler. if state.defaultHandler != nil { if state.defaultHandler(id, pkt) { - return TransportPacketHandled + return TransportPacketHandled, true } } @@ -891,11 +902,11 @@ func (n *nic) DeliverTransportPacket(protocol tcpip.TransportProtocolNumber, pkt switch res := transProto.HandleUnknownDestinationPacket(id, pkt); res { case UnknownDestinationPacketMalformed: n.stats.malformedL4RcvdPackets.Increment() - return TransportPacketHandled + return TransportPacketHandled, false case UnknownDestinationPacketUnhandled: - return TransportPacketDestinationPortUnreachable + return TransportPacketDestinationPortUnreachable, false case UnknownDestinationPacketHandled: - return TransportPacketHandled + return TransportPacketHandled, false default: panic(fmt.Sprintf("unrecognized result from HandleUnknownDestinationPacket = %d", res)) } diff --git a/pkg/tcpip/stack/nic_mutex.go b/pkg/tcpip/stack/nic_mutex.go index 43d8325..d35d529 100644 --- a/pkg/tcpip/stack/nic_mutex.go +++ b/pkg/tcpip/stack/nic_mutex.go @@ -92,5 +92,5 @@ func nicinitLockNames() {} func init() { nicinitLockNames() - nicprefixIndex = locking.NewMutexClass(reflect.TypeOf(nicRWMutex{}), niclockNames) + nicprefixIndex = locking.NewMutexClass(reflect.TypeFor[nicRWMutex](), niclockNames) } diff --git a/pkg/tcpip/stack/packet_buffer.go b/pkg/tcpip/stack/packet_buffer.go index 53d8de5..af8b388 100644 --- a/pkg/tcpip/stack/packet_buffer.go +++ b/pkg/tcpip/stack/packet_buffer.go @@ -20,6 +20,7 @@ import ( "github.com/sagernet/gvisor/pkg/buffer" "github.com/sagernet/gvisor/pkg/sync" "github.com/sagernet/gvisor/pkg/tcpip" + "github.com/sagernet/gvisor/pkg/tcpip/checksum" "github.com/sagernet/gvisor/pkg/tcpip/header" ) @@ -56,6 +57,9 @@ type PacketBufferOptions struct { // OnRelease is a function to be run when the packet buffer is no longer // referenced (released back to the pool). OnRelease func() + + // Mark is the mark value of this packet. + Mark uint32 } // A PacketBuffer contains all the data of a network packet. @@ -154,6 +158,10 @@ type PacketBuffer struct { // NICID is the ID of the last interface the network packet was handled at. NICID tcpip.NICID + // InputNICID is the ID of the interface that the network packet + // was received on. + InputNICID tcpip.NICID + // RXChecksumValidated indicates that checksum verification may be // safely skipped. RXChecksumValidated bool @@ -161,6 +169,9 @@ type PacketBuffer struct { // NetworkPacketInfo holds an incoming packet's network-layer information. NetworkPacketInfo NetworkPacketInfo + // Mark is the mark value of this packet. + Mark uint32 + tuple *tuple // onRelease is a function to be run when the packet buffer is no longer @@ -182,6 +193,7 @@ func NewPacketBuffer(opts PacketBufferOptions) *PacketBuffer { } pk.NetworkPacketInfo.IsForwardedPacket = opts.IsForwardedPacket pk.onRelease = opts.OnRelease + pk.Mark = opts.Mark pk.InitRefs() return pk } @@ -380,6 +392,7 @@ func (pk *PacketBuffer) Clone() *PacketBuffer { newPk.headers = pk.headers newPk.Hash = pk.Hash newPk.Owner = pk.Owner + newPk.Mark = pk.Mark newPk.GSOOptions = pk.GSOOptions newPk.EgressRoute = pk.EgressRoute newPk.NetworkProtocolNumber = pk.NetworkProtocolNumber @@ -388,6 +401,7 @@ func (pk *PacketBuffer) Clone() *PacketBuffer { newPk.TransportProtocolNumber = pk.TransportProtocolNumber newPk.PktType = pk.PktType newPk.NICID = pk.NICID + newPk.InputNICID = pk.InputNICID newPk.RXChecksumValidated = pk.RXChecksumValidated newPk.NetworkPacketInfo = pk.NetworkPacketInfo newPk.tuple = pk.tuple @@ -431,6 +445,7 @@ func (pk *PacketBuffer) CloneToInbound() *PacketBuffer { newPk.InitRefs() // Treat unfilled header portion as reserved. newPk.reserved = pk.AvailableHeaderBytes() + newPk.Mark = pk.Mark newPk.tuple = pk.tuple return newPk } @@ -466,10 +481,78 @@ func (pk *PacketBuffer) DeepCopyForForwarding(reservedHeaderBytes int) *PacketBu } newPk.tuple = pk.tuple + newPk.Mark = pk.Mark + newPk.InputNICID = pk.InputNICID return newPk } +// IsConnTrackConfigured returns whether connection tracking is configured for this packet. +func (pk *PacketBuffer) IsConnTrackConfigured() bool { + return pk.tuple != nil && pk.tuple.conn != nil +} + +// FillConnTrackInfo fills connection tracking information for the packet. +func (pk *PacketBuffer) FillConnTrackInfo(opts ConnTrackInfoOpts, info *ConnTrackInfo) bool { + t := pk.tuple + if t == nil || t.conn == nil { + return false + } + return t.conn.FillConnTrackInfo(opts, info) +} + +// IsReplyPacket returns whether the packet is a reply packet. +func (pk *PacketBuffer) IsReplyPacket() bool { + t := pk.tuple + if t == nil { + return false + } + return t.reply +} + +// IsNATConfigured returns whether NAT is configured for this packet. +func (pk *PacketBuffer) IsNATConfigured(nt NATType) bool { + if !pk.IsConnTrackConfigured() { + return false + } + return pk.tuple.conn.IsNATConfigured(nt) +} + +// ConfigureNoopNAT configures a no-op NAT for the packet. +// Called if no NAT rules are configured for this packet. +func (pk *PacketBuffer) ConfigureNoopNAT(natType NATType) bool { + if !pk.IsConnTrackConfigured() { + return false + } + return pk.tuple.conn.ConfigureNoopNAT(pk, natType) +} + +// ConfigureNAT configures NAT for the packet. +// Called if NAT rules are configured for this packet. +// Returns whether NAT was configured or not. +func (pk *PacketBuffer) ConfigureNAT(portsOrIdents PortOrIdentRange, natAddress tcpip.Address, natType NATType, changePort, changeAddress bool) bool { + if !pk.IsConnTrackConfigured() { + return false + } + return pk.tuple.conn.ConfigureNAT(portsOrIdents, natAddress, natType, changePort, changeAddress) +} + +// ConfigureMasquerade configures NAT masquerade for the packet. +func (pk *PacketBuffer) ConfigureMasquerade(portsOrIdents PortOrIdentRange, route *Route, stk *Stack, changePort bool) bool { + if !pk.IsConnTrackConfigured() { + return false + } + return pk.tuple.conn.configureMasquerade(pk, route, stk, portsOrIdents, changePort) +} + +// FinalizeConnTrack finalizes the connection tracking state for the packet. +func (pk *PacketBuffer) FinalizeConnTrack() bool { + if pk.tuple == nil || pk.tuple.conn == nil { + return true + } + return pk.tuple.conn.finalize() +} + // headerInfo stores metadata about a header in a packet. // // +stateify savable @@ -768,3 +851,312 @@ func BufferSince(h PacketHeader) buffer.Buffer { clone.TrimFront(int64(offset)) return clone } + +// ExperimentOptionValue returns the experiment option value from the packet +// and a bool indicating whether an experiment option value was found. +func (pk *PacketBuffer) ExperimentOptionValue() (uint16, bool) { + switch pk.NetworkProtocolNumber { + case header.IPv4ProtocolNumber: + h := header.IPv4(pk.NetworkHeader().Slice()) + opts := h.Options() + iter := opts.MakeIterator() + for { + opt, done, err := iter.Next() + if err != nil { + return 0, false + } + if done { + return 0, false + } + if opt.Type() == header.IPv4OptionExperimentType { + return opt.(*header.IPv4OptionExperiment).Value(), true + } + } + case header.IPv6ProtocolNumber: + h := header.IPv6(pk.NetworkHeader().Slice()) + v := pk.NetworkHeader().View() + if v != nil { + v.TrimFront(header.IPv6MinimumSize) + } + buf := buffer.MakeWithView(v) + buf.Append(pk.TransportHeader().View()) + dataBuf := pk.Data().ToBuffer() + buf.Merge(&dataBuf) + it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(h.NextHeader()), buf) + + for { + hdr, done, err := it.Next() + if done || err != nil { + break + } + if h, ok := hdr.(header.IPv6ExperimentExtHdr); ok { + hdr.Release() + return h.Value, true + } + hdr.Release() + } + default: + panic(fmt.Sprintf("Unexpected network protocol number %d", pk.NetworkProtocolNumber)) + } + return 0, false +} + +// GetEmbeddedNetAndTransHeaders returns the network and transport headers of the +// packet. +func (pk *PacketBuffer) GetEmbeddedNetAndTransHeaders(netHdrLength int, getNetAndTransHdr netAndTransHeadersFunc, transProto tcpip.TransportProtocolNumber) (header.Network, header.ChecksummableTransport, bool) { + switch transProto { + case header.TCPProtocolNumber: + if netAndTransHeader, ok := pk.Data().PullUp(netHdrLength + header.TCPMinimumSize); ok { + netHeader, transHeaderBytes := getNetAndTransHdr(netAndTransHeader, header.TCPMinimumSize) + return netHeader, header.TCP(transHeaderBytes), true + } + case header.UDPProtocolNumber: + if netAndTransHeader, ok := pk.Data().PullUp(netHdrLength + header.UDPMinimumSize); ok { + netHeader, transHeaderBytes := getNetAndTransHdr(netAndTransHeader, header.UDPMinimumSize) + return netHeader, header.UDP(transHeaderBytes), true + } + } + return nil, nil, false +} + +// GetHeaders returns the network and transport headers of the packet. +func (pk *PacketBuffer) GetHeaders() (netHdr header.Network, transHdr header.Transport, isICMPError bool, ok bool) { + switch pk.TransportProtocolNumber { + case header.TCPProtocolNumber: + if tcpHeader := header.TCP(pk.TransportHeader().Slice()); len(tcpHeader) >= header.TCPMinimumSize { + return pk.Network(), tcpHeader, false, true + } + return nil, nil, false, false + case header.UDPProtocolNumber: + if udpHeader := header.UDP(pk.TransportHeader().Slice()); len(udpHeader) >= header.UDPMinimumSize { + return pk.Network(), udpHeader, false, true + } + return nil, nil, false, false + case header.ICMPv4ProtocolNumber: + icmpHeader := header.ICMPv4(pk.TransportHeader().Slice()) + if len(icmpHeader) < header.ICMPv4MinimumSize { + return nil, nil, false, false + } + + switch icmpType := icmpHeader.Type(); icmpType { + case header.ICMPv4Echo, header.ICMPv4EchoReply: + return pk.Network(), icmpHeader, false, true + case header.ICMPv4DstUnreachable, header.ICMPv4TimeExceeded, header.ICMPv4ParamProblem: + default: + return nil, nil, false, false + } + + h, ok := pk.Data().PullUp(header.IPv4MinimumSize) + if !ok { + return nil, nil, false, false + } + + hdrLength := int(header.IPv4(h).HeaderLength()) + // Pull up the full IPv4 header which might include options. + if hdrLength > header.IPv4MinimumSize { + // TODO(https://gvisor.dev/issue/6765): Handle IPv4 + // options. + h, ok = pk.Data().PullUp(hdrLength) + if !ok { + return nil, nil, false, false + } + } + + if netHdr, transHdr, ok := pk.GetEmbeddedNetAndTransHeaders(hdrLength, v4NetAndTransHdr, tcpip.TransportProtocolNumber(header.IPv4(h).Protocol())); ok { + return netHdr, transHdr, true, true + } + return nil, nil, false, false + case header.ICMPv6ProtocolNumber: + icmpHeader := header.ICMPv6(pk.TransportHeader().Slice()) + if len(icmpHeader) < header.ICMPv6MinimumSize { + return nil, nil, false, false + } + + switch icmpType := icmpHeader.Type(); icmpType { + case header.ICMPv6EchoRequest, header.ICMPv6EchoReply: + return pk.Network(), icmpHeader, false, true + case header.ICMPv6DstUnreachable, header.ICMPv6PacketTooBig, header.ICMPv6TimeExceeded, header.ICMPv6ParamProblem: + default: + return nil, nil, false, false + } + + h, ok := pk.Data().PullUp(header.IPv6MinimumSize) + if !ok { + return nil, nil, false, false + } + + // We do not support extension headers in ICMP errors so the next header + // in the IPv6 packet should be a tracked protocol if we reach this point. + // + // TODO(https://gvisor.dev/issue/6789): Support extension headers. + transProto, _ := header.IPv6(h).TryParseTransportProtocol() + if netHdr, transHdr, ok := pk.GetEmbeddedNetAndTransHeaders(header.IPv6MinimumSize, v6NetAndTransHdr, transProto); ok { + return netHdr, transHdr, true, true + } + return nil, nil, false, false + default: + return nil, nil, false, false + } +} + +// UpdateHeaders updates the headers of the packet with the new port and address. +func UpdateHeaders(n header.Network, t header.Transport, updateSRCFields, fullChecksum, updatePseudoHeader bool, newPortOrIdent uint16, newAddr tcpip.Address) { + switch t := t.(type) { + case header.ChecksummableTransport: + if updateSRCFields { + if fullChecksum { + t.SetSourcePortWithChecksumUpdate(newPortOrIdent) + } else { + t.SetSourcePort(newPortOrIdent) + } + } else { + if fullChecksum { + t.SetDestinationPortWithChecksumUpdate(newPortOrIdent) + } else { + t.SetDestinationPort(newPortOrIdent) + } + } + + if updatePseudoHeader { + var oldAddr tcpip.Address + if updateSRCFields { + oldAddr = n.SourceAddress() + } else { + oldAddr = n.DestinationAddress() + } + + t.UpdateChecksumPseudoHeaderAddress(oldAddr, newAddr, fullChecksum) + } + case header.ICMPv4: + switch icmpType := t.Type(); icmpType { + case header.ICMPv4Echo: + if updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + case header.ICMPv4EchoReply: + if !updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + default: + panic(fmt.Sprintf("unexpected ICMPv4 type = %d", icmpType)) + } + case header.ICMPv6: + switch icmpType := t.Type(); icmpType { + case header.ICMPv6EchoRequest: + if updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + case header.ICMPv6EchoReply: + if !updateSRCFields { + t.SetIdentWithChecksumUpdate(newPortOrIdent) + } + default: + panic(fmt.Sprintf("unexpected ICMPv6 type = %d", icmpType)) + } + + var oldAddr tcpip.Address + if updateSRCFields { + oldAddr = n.SourceAddress() + } else { + oldAddr = n.DestinationAddress() + } + + t.UpdateChecksumPseudoHeaderAddress(oldAddr, newAddr) + default: + panic(fmt.Sprintf("unhandled transport = %#v", t)) + } + + if checksummableNetHeader, ok := n.(header.ChecksummableNetwork); ok { + if updateSRCFields { + checksummableNetHeader.SetSourceAddressWithChecksumUpdate(newAddr) + } else { + checksummableNetHeader.SetDestinationAddressWithChecksumUpdate(newAddr) + } + } else if updateSRCFields { + n.SetSourceAddress(newAddr) + } else { + n.SetDestinationAddress(newAddr) + } +} + +// CalculateTransportChecksum calculates the transport-layer checksum of the +// packet. +// TODO: b/521901282 - Verify with GSO. +func (pk *PacketBuffer) CalculateTransportChecksum() { + netHdr, transHdr, isICMPError, ok := pk.GetHeaders() + if isICMPError { + // Skip ICMP errors because GetHeaders() returns inner headers, but pk.Data() + // contains the outer payload (including inner IP header), which would + // corrupt the checksum calculation if used as the transport payload. + // Inner headers are already incrementally updated by NAT if needed. + // This aligns with Linux, which also relies on incremental updates for + // inner headers and does not perform full recalculation from scratch. + return + } + if !ok { + // Try to parse headers from Data if not set (e.g., forwarded packet). + if pk.NetworkProtocolNumber == 0 { + return + } + netHdr = pk.Network() + transProto := netHdr.TransportProtocol() + + var headerSize int + switch transProto { + case header.TCPProtocolNumber: + // Peek at minimum TCP header to find data offset (which includes options). + b, ok := pk.Data().PullUp(header.TCPMinimumSize) + if !ok { + return + } + tcp := header.TCP(b) + headerSize = int(tcp.DataOffset()) + if headerSize < header.TCPMinimumSize { + return + } + case header.UDPProtocolNumber: + headerSize = header.UDPMinimumSize + default: + return + } + + // Consume the transport header. + if _, ok := pk.TransportHeader().Consume(headerSize); !ok { + return + } + pk.TransportProtocolNumber = transProto + + // Refresh headers. + netHdr, transHdr, isICMPError, ok = pk.GetHeaders() + if !ok || isICMPError { + return + } + } + + var xsum uint16 + switch t := transHdr.(type) { + case header.TCP: + src := netHdr.SourceAddress() + dst := netHdr.DestinationAddress() + proto := netHdr.TransportProtocol() + totalLen := uint16(len(t) + pk.Data().Size()) + xsum = header.PseudoHeaderChecksum(proto, src, dst, totalLen) + xsum = checksum.Combine(xsum, pk.Data().Checksum()) + t.SetChecksum(0) + t.SetChecksum(^t.CalculateChecksum(xsum)) + case header.UDP: + src := netHdr.SourceAddress() + dst := netHdr.DestinationAddress() + proto := netHdr.TransportProtocol() + totalLen := uint16(len(t) + pk.Data().Size()) + xsum = header.PseudoHeaderChecksum(proto, src, dst, totalLen) + xsum = checksum.Combine(xsum, pk.Data().Checksum()) + t.SetChecksum(0) + csum := ^t.CalculateChecksum(xsum) + // udp csum RFC 768. + if csum == 0 { + csum = 0xFFFF + } + t.SetChecksum(csum) + } +} diff --git a/pkg/tcpip/stack/packet_endpoint_list_mutex.go b/pkg/tcpip/stack/packet_endpoint_list_mutex.go index 0844e79..a8279ca 100644 --- a/pkg/tcpip/stack/packet_endpoint_list_mutex.go +++ b/pkg/tcpip/stack/packet_endpoint_list_mutex.go @@ -92,5 +92,5 @@ func packetEndpointListinitLockNames() {} func init() { packetEndpointListinitLockNames() - packetEndpointListprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetEndpointListRWMutex{}), packetEndpointListlockNames) + packetEndpointListprefixIndex = locking.NewMutexClass(reflect.TypeFor[packetEndpointListRWMutex](), packetEndpointListlockNames) } diff --git a/pkg/tcpip/stack/packet_eps_mutex.go b/pkg/tcpip/stack/packet_eps_mutex.go index 679ba48..b2eda6c 100644 --- a/pkg/tcpip/stack/packet_eps_mutex.go +++ b/pkg/tcpip/stack/packet_eps_mutex.go @@ -92,5 +92,5 @@ func packetEPsinitLockNames() {} func init() { packetEPsinitLockNames() - packetEPsprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetEPsRWMutex{}), packetEPslockNames) + packetEPsprefixIndex = locking.NewMutexClass(reflect.TypeFor[packetEPsRWMutex](), packetEPslockNames) } diff --git a/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go b/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go index 1a71bfb..4c3cc1f 100644 --- a/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go +++ b/pkg/tcpip/stack/packets_pending_link_resolution_mutex.go @@ -60,5 +60,5 @@ func packetsPendingLinkResolutioninitLockNames() {} func init() { packetsPendingLinkResolutioninitLockNames() - packetsPendingLinkResolutionprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetsPendingLinkResolutionMutex{}), packetsPendingLinkResolutionlockNames) + packetsPendingLinkResolutionprefixIndex = locking.NewMutexClass(reflect.TypeFor[packetsPendingLinkResolutionMutex](), packetsPendingLinkResolutionlockNames) } diff --git a/pkg/tcpip/stack/registration.go b/pkg/tcpip/stack/registration.go index 1c1ef64..142fd2b 100644 --- a/pkg/tcpip/stack/registration.go +++ b/pkg/tcpip/stack/registration.go @@ -365,6 +365,21 @@ type TransportDispatcher interface { DeliverRawPacket(tcpip.TransportProtocolNumber, *PacketBuffer) } +// TransportDispatcherWithDefaultHandlerResult extends TransportDispatcher with +// default-handler-specific delivery metadata. +type TransportDispatcherWithDefaultHandlerResult interface { + TransportDispatcher + + // DeliverTransportPacketWithDefaultHandlerResult delivers packets to the + // appropriate transport protocol endpoint and reports whether the packet was + // specifically handled by the per-stack default transport protocol handler. + // + // pkt.NetworkHeader must be set before calling this method. + // + // DeliverTransportPacketWithDefaultHandlerResult may modify the packet. + DeliverTransportPacketWithDefaultHandlerResult(tcpip.TransportProtocolNumber, *PacketBuffer) (TransportPacketDisposition, bool) +} + // PacketLooping specifies where an outbound packet should be sent. type PacketLooping byte @@ -872,6 +887,9 @@ type NetworkEndpoint interface { // minus the network endpoint max header length. MTU() uint32 + // EndpointHeaderSize returns the size of this endpoint header. + EndpointHeaderSize() uint32 + // MaxHeaderLength returns the maximum size the network (and lower // level layers combined) headers can have. Higher levels use this // information to reserve space in the front of the packets they're @@ -1135,7 +1153,6 @@ const ( CapabilityRXChecksumOffload CapabilityResolutionRequired CapabilitySaveRestore - CapabilityDisconnectOk CapabilityLoopback ) diff --git a/pkg/tcpip/stack/route.go b/pkg/tcpip/stack/route.go index ed2bc28..c32960e 100644 --- a/pkg/tcpip/stack/route.go +++ b/pkg/tcpip/stack/route.go @@ -180,7 +180,7 @@ func constructAndValidateRoute(netProto tcpip.NetworkProtocolNumber, addressEndp // AssignableAddressEndpoint. func makeRoute(netProto tcpip.NetworkProtocolNumber, gateway, localAddr, remoteAddr tcpip.Address, outgoingNIC, localAddressNIC *nic, localAddressEndpoint AssignableAddressEndpoint, handleLocal, multicastLoop bool, mtu uint32) *Route { if localAddressNIC.stack != outgoingNIC.stack { - panic(fmt.Sprintf("cannot create a route with NICs from different stacks")) + panic("cannot create a route with NICs from different stacks") } if localAddr.BitLen() == 0 { @@ -245,6 +245,14 @@ func makeRoute(netProto tcpip.NetworkProtocolNumber, gateway, localAddr, remoteA } func makeRouteInner(netProto tcpip.NetworkProtocolNumber, localAddr, remoteAddr tcpip.Address, outgoingNIC, localAddressNIC *nic, localAddressEndpoint AssignableAddressEndpoint, loop PacketLooping, mtu uint32) *Route { + if mtu != 0 { + adjusted := mtu - outgoingNIC.getNetworkEndpoint(netProto).EndpointHeaderSize() + if adjusted > mtu { + mtu = 0 + } else { + mtu = adjusted + } + } r := &Route{ routeInfo: routeInfo{ NetProto: netProto, @@ -339,11 +347,6 @@ func (r *Route) HasSaveRestoreCapability() bool { return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilitySaveRestore != 0 } -// HasDisconnectOkCapability returns true if the route supports disconnecting. -func (r *Route) HasDisconnectOkCapability() bool { - return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityDisconnectOk != 0 -} - // GSOMaxSize returns the maximum GSO packet size. func (r *Route) GSOMaxSize() uint32 { if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok { @@ -525,6 +528,7 @@ func (r *Route) DefaultTTL() uint8 { // MTU returns the MTU of the route if present, otherwise the MTU of the underlying network endpoint. func (r *Route) MTU() uint32 { if r.mtu > 0 { + // r.mtu is already adjusted to account for IP headers. See makeRouteInner. return r.mtu } return r.outgoingNIC.getNetworkEndpoint(r.NetProto()).MTU() diff --git a/pkg/tcpip/stack/route_mutex.go b/pkg/tcpip/stack/route_mutex.go index ca06e5c..ccf5600 100644 --- a/pkg/tcpip/stack/route_mutex.go +++ b/pkg/tcpip/stack/route_mutex.go @@ -92,5 +92,5 @@ func routeinitLockNames() {} func init() { routeinitLockNames() - routeprefixIndex = locking.NewMutexClass(reflect.TypeOf(routeRWMutex{}), routelockNames) + routeprefixIndex = locking.NewMutexClass(reflect.TypeFor[routeRWMutex](), routelockNames) } diff --git a/pkg/tcpip/stack/route_stack_mutex.go b/pkg/tcpip/stack/route_stack_mutex.go index bfa849f..d879240 100644 --- a/pkg/tcpip/stack/route_stack_mutex.go +++ b/pkg/tcpip/stack/route_stack_mutex.go @@ -92,5 +92,5 @@ func routeStackinitLockNames() {} func init() { routeStackinitLockNames() - routeStackprefixIndex = locking.NewMutexClass(reflect.TypeOf(routeStackRWMutex{}), routeStacklockNames) + routeStackprefixIndex = locking.NewMutexClass(reflect.TypeFor[routeStackRWMutex](), routeStacklockNames) } diff --git a/pkg/tcpip/stack/save_restore.go b/pkg/tcpip/stack/save_restore.go index 5b38acd..222e07a 100644 --- a/pkg/tcpip/stack/save_restore.go +++ b/pkg/tcpip/stack/save_restore.go @@ -22,6 +22,31 @@ import ( cryptorand "github.com/sagernet/gvisor/pkg/rand" ) +// beforeSave is invoked by stateify. +func (s *Stack) beforeSave() { + // removeConf will be set only in case of save/restore. + s.mu.Lock() + if !s.removeConf { + s.mu.Unlock() + return + } + + // Remove all the NICs and routes from the stack as they will be + // created again during restore based on the new network config. + deferActs := make([]func(), 0) + for id := range s.nics { + act, _ := s.removeNICLocked(id, true /* closeLinkEndpoint */) + if act != nil { + deferActs = append(deferActs, act) + } + } + s.mu.Unlock() + + for _, act := range deferActs { + act() + } +} + // afterLoad is invoked by stateify. func (s *Stack) afterLoad(context.Context) { s.insecureRNG = rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/pkg/tcpip/stack/stack.go b/pkg/tcpip/stack/stack.go index 0d1d738..259f366 100644 --- a/pkg/tcpip/stack/stack.go +++ b/pkg/tcpip/stack/stack.go @@ -20,17 +20,18 @@ package stack import ( - "context" "encoding/binary" "fmt" "io" "math/rand" + "sync/atomic" "time" "github.com/sagernet/gvisor/pkg/atomicbitops" "github.com/sagernet/gvisor/pkg/buffer" "github.com/sagernet/gvisor/pkg/log" cryptorand "github.com/sagernet/gvisor/pkg/rand" + "github.com/sagernet/gvisor/pkg/sync" "github.com/sagernet/gvisor/pkg/tcpip" "github.com/sagernet/gvisor/pkg/tcpip/header" "github.com/sagernet/gvisor/pkg/tcpip/ports" @@ -96,7 +97,7 @@ type Stack struct { // +checklocks:mu nics map[tcpip.NICID]*nic `state:"nosave"` // +checklocks:mu - loopbackNIC *nic + loopbackNIC *nic `state:"nosave"` // +checklocks:mu defaultForwardingEnabled map[tcpip.NetworkProtocolNumber]struct{} @@ -121,7 +122,15 @@ type Stack struct { tables *IPTables `state:"nosave"` // nftables is the nftables interface for packet filtering and manipulation rules. - nftables NFTablesInterface `state:"nosave"` + // Using atomic.Pointer for RCU lock-free reads. + nftables atomic.Pointer[NFTablesInterface] `state:"nosave"` + + // nftablesUpdateMu serializes concurrent netlink batch modifications to nftables. + nftablesUpdateMu sync.Mutex `state:"nosave"` + + // nftablesConfigured indicates whether NFTables is configured with at + // least one rule on a chain at a network hook. + nftablesConfigured atomicbitops.Bool // restoredEndpoints is a list of endpoints that need to be restored if the // stack is being restored. @@ -179,8 +188,23 @@ type Stack struct { // initialized at stack startup. tsOffsetSecret uint32 - // saveRestoreEnabled indicates whether the stack is saved and restored. - saveRestoreEnabled bool + // removeConf indicates whether to remove NICs and routes and terminate + // active connections before saving. This flag will be set to true only + // when resume is false. + removeConf bool `state:"nosave"` + + // allowLiveTCPMigration allows TCP connection state to be migrated. + // If false, any connected TCP endpoints will be terminated + // during save/restore. + allowLiveTCPMigration bool `state:"nosave"` + + // externalNetworkingDisabled indicates whether external networking is + // disabled. This means all non-loopback NICs are disabled. + externalNetworkingDisabled bool + + // allowConnectedOnSave indicates whether connections should be + // allowed to remain connected during save. + allowConnectedOnSave bool } // NetworkProtocolFactory instantiates a network protocol. @@ -231,6 +255,11 @@ type Options struct { // operations. AllowPacketEndpointWrite bool + // AllowLiveTCPMigration allows TCP connection state to be migrated. + // If false, any connected TCP endpoints will be terminated + // during save/restore. + AllowLiveTCPMigration bool + // RandSource is an optional source to use to generate random // numbers. If omitted it defaults to a Source seeded by the data // returned by the stack secure RNG. @@ -398,7 +427,6 @@ func New(opts Options) *Stack { stats: opts.Stats.FillIn(), handleLocal: opts.HandleLocal, tables: opts.IPTables, - nftables: opts.NFTables, icmpRateLimiter: NewICMPRateLimiter(clock), seed: secureRNG.Uint32(), nudConfigs: opts.NUDConfigs, @@ -415,9 +443,11 @@ func New(opts Options) *Stack { Default: DefaultBufferSize, Max: DefaultMaxBufferSize, }, - tcpInvalidRateLimit: defaultTCPInvalidRateLimit, - tsOffsetSecret: secureRNG.Uint32(), + tcpInvalidRateLimit: defaultTCPInvalidRateLimit, + tsOffsetSecret: secureRNG.Uint32(), + allowLiveTCPMigration: opts.AllowLiveTCPMigration, } + s.SetNFTables(opts.NFTables) // Add specified network protocols. for _, netProtoFactory := range opts.NetworkProtocols { @@ -895,8 +925,8 @@ type NICOptions struct { // GetNICByID return a network device associated with the specified ID. func (s *Stack) GetNICByID(id tcpip.NICID) (*nic, tcpip.Error) { - s.mu.Lock() - defer s.mu.Unlock() + s.mu.RLock() + defer s.mu.RUnlock() n, ok := s.nics[id] if !ok { @@ -1017,7 +1047,7 @@ func (s *Stack) CheckNIC(id tcpip.NICID) bool { // RemoveNIC removes NIC and all related routes from the network stack. func (s *Stack) RemoveNIC(id tcpip.NICID) tcpip.Error { s.mu.Lock() - deferAct, err := s.removeNICLocked(id) + deferAct, err := s.removeNICLocked(id, true /* closeLinkEndpoint */) s.mu.Unlock() if deferAct != nil { deferAct() @@ -1028,7 +1058,7 @@ func (s *Stack) RemoveNIC(id tcpip.NICID) tcpip.Error { // removeNICLocked removes NIC and all related routes from the network stack. // // +checklocks:s.mu -func (s *Stack) removeNICLocked(id tcpip.NICID) (func(), tcpip.Error) { +func (s *Stack) removeNICLocked(id tcpip.NICID, closeLinkEndpoint bool) (func(), tcpip.Error) { nic, ok := s.nics[id] if !ok { return nil, &tcpip.ErrUnknownNICID{} @@ -1056,7 +1086,19 @@ func (s *Stack) removeNICLocked(id tcpip.NICID) (func(), tcpip.Error) { if s.loopbackNIC == nic { s.loopbackNIC = nil } - return nic.remove(true /* closeLinkEndpoint */) + return nic.remove(closeLinkEndpoint) +} + +// GetNICCoordinatorID returns the ID of the coordinator device of a NIC. +func (s *Stack) GetNICCoordinatorID(id tcpip.NICID) (tcpip.NICID, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + if nic, ok := s.nics[id]; ok { + if nic.Primary != nil { + return nic.Primary.id, true + } + } + return 0, false } // SetNICCoordinator sets a coordinator device. @@ -1159,6 +1201,9 @@ type NICInfo struct { // MulticastForwarding holds the forwarding status for each network endpoint // that supports multicast forwarding. MulticastForwarding map[tcpip.NetworkProtocolNumber]bool + + // Primary is the index of the main controlling interface in a bonded setup. + Primary tcpip.NICID } // HasNIC returns true if the NICID is defined in the stack. @@ -1169,65 +1214,87 @@ func (s *Stack) HasNIC(id tcpip.NICID) bool { return ok } +type forwardingFn func(tcpip.NetworkProtocolNumber) (bool, tcpip.Error) + +func forwardingValue(forwardingFn forwardingFn, proto tcpip.NetworkProtocolNumber, nicID tcpip.NICID, fnName string) (forward bool, ok bool) { + switch forwarding, err := forwardingFn(proto); err.(type) { + case nil: + return forwarding, true + case *tcpip.ErrUnknownProtocol: + panic(fmt.Sprintf("expected network protocol %d to be available on NIC %d", proto, nicID)) + case *tcpip.ErrNotSupported: + // Not all network protocols support forwarding. + default: + panic(fmt.Sprintf("nic(id=%d).%s(%d): %s", nicID, fnName, proto, err)) + } + return false, false +} + +// precondition: s.mu is held. +func (s *Stack) nicInfo(nic *nic, id tcpip.NICID) *NICInfo { + flags := NICStateFlags{ + Up: true, // Netstack interfaces are always up. + Running: nic.Enabled(), + Promiscuous: nic.Promiscuous(), + Loopback: nic.IsLoopback(), + } + + netStats := make(map[tcpip.NetworkProtocolNumber]NetworkEndpointStats) + for proto, netEP := range nic.networkEndpoints { + netStats[proto] = netEP.Stats() + } + + info := NICInfo{ + Name: nic.name, + LinkAddress: nic.NetworkLinkEndpoint.LinkAddress(), + ProtocolAddresses: nic.primaryAddresses(), + Flags: flags, + MTU: nic.NetworkLinkEndpoint.MTU(), + Stats: nic.stats.local, + NetworkStats: netStats, + Context: nic.context, + ARPHardwareType: nic.NetworkLinkEndpoint.ARPHardwareType(), + Forwarding: make(map[tcpip.NetworkProtocolNumber]bool), + MulticastForwarding: make(map[tcpip.NetworkProtocolNumber]bool), + } + + for proto := range s.networkProtocols { + if forwarding, ok := forwardingValue(nic.forwarding, proto, id, "forwarding"); ok { + info.Forwarding[proto] = forwarding + } + + if multicastForwarding, ok := forwardingValue(nic.multicastForwarding, proto, id, "multicastForwarding"); ok { + info.MulticastForwarding[proto] = multicastForwarding + } + } + + if nic.Primary != nil { + info.Primary = nic.Primary.id + } + + return &info +} + +// SingleNICInfo returns the NICInfo for the given NICID. +func (s *Stack) SingleNICInfo(id tcpip.NICID) (*NICInfo, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + if nic, ok := s.nics[id]; !ok { + return nil, false + } else { + return s.nicInfo(nic, id), true + } +} + // NICInfo returns a map of NICIDs to their associated information. func (s *Stack) NICInfo() map[tcpip.NICID]NICInfo { s.mu.RLock() defer s.mu.RUnlock() - type forwardingFn func(tcpip.NetworkProtocolNumber) (bool, tcpip.Error) - forwardingValue := func(forwardingFn forwardingFn, proto tcpip.NetworkProtocolNumber, nicID tcpip.NICID, fnName string) (forward bool, ok bool) { - switch forwarding, err := forwardingFn(proto); err.(type) { - case nil: - return forwarding, true - case *tcpip.ErrUnknownProtocol: - panic(fmt.Sprintf("expected network protocol %d to be available on NIC %d", proto, nicID)) - case *tcpip.ErrNotSupported: - // Not all network protocols support forwarding. - default: - panic(fmt.Sprintf("nic(id=%d).%s(%d): %s", nicID, fnName, proto, err)) - } - return false, false - } - nics := make(map[tcpip.NICID]NICInfo) for id, nic := range s.nics { - flags := NICStateFlags{ - Up: true, // Netstack interfaces are always up. - Running: nic.Enabled(), - Promiscuous: nic.Promiscuous(), - Loopback: nic.IsLoopback(), - } - - netStats := make(map[tcpip.NetworkProtocolNumber]NetworkEndpointStats) - for proto, netEP := range nic.networkEndpoints { - netStats[proto] = netEP.Stats() - } - - info := NICInfo{ - Name: nic.name, - LinkAddress: nic.NetworkLinkEndpoint.LinkAddress(), - ProtocolAddresses: nic.primaryAddresses(), - Flags: flags, - MTU: nic.NetworkLinkEndpoint.MTU(), - Stats: nic.stats.local, - NetworkStats: netStats, - Context: nic.context, - ARPHardwareType: nic.NetworkLinkEndpoint.ARPHardwareType(), - Forwarding: make(map[tcpip.NetworkProtocolNumber]bool), - MulticastForwarding: make(map[tcpip.NetworkProtocolNumber]bool), - } - - for proto := range s.networkProtocols { - if forwarding, ok := forwardingValue(nic.forwarding, proto, id, "forwarding"); ok { - info.Forwarding[proto] = forwarding - } - - if multicastForwarding, ok := forwardingValue(nic.multicastForwarding, proto, id, "multicastForwarding"); ok { - info.MulticastForwarding[proto] = multicastForwarding - } - } - - nics[id] = info + nics[id] = *s.nicInfo(nic, id) } return nics } @@ -1991,7 +2058,7 @@ func (s *Stack) Wait() { for id, n := range s.nics { // Remove NIC to ensure that qDisc goroutines are correctly // terminated on stack teardown. - act, _ := s.removeNICLocked(id) + act, _ := s.removeNICLocked(id, true /* closeLinkEndpoint */) n.NetworkLinkEndpoint.Wait() if act != nil { deferActs = append(deferActs, act) @@ -2025,31 +2092,43 @@ func (s *Stack) getNICs() map[tcpip.NICID]*nic { return nics } +// ResetConfig resets the stack's NICs and ID generator. +func (s *Stack) ResetConfig() { + nics := make(map[tcpip.NICID]*nic) + s.mu.Lock() + defer s.mu.Unlock() + s.nics = nics + s.loopbackNIC = nil + s.nicIDGen.Store(0) +} + // ReplaceConfig replaces config in the loaded stack. func (s *Stack) ReplaceConfig(st *Stack) { if st == nil { - panic("stack.Stack cannot be nil when netstack s/r is enabled") + panic("stack.Stack cannot be nil when replacing config") } // Update route table. s.SetRouteTable(st.GetRouteTable()) - // Update NICs. nics := st.getNICs() + s.mu.Lock() defer s.mu.Unlock() - s.nics = make(map[tcpip.NICID]*nic) - s.loopbackNIC = nil + + // Update iptables and nftables. + s.tables = st.IPTables() + s.SetNFTables(st.NFTables()) for id, nic := range nics { nic.stack = s s.nics[id] = nic if nic.IsLoopback() { s.loopbackNIC = nic + } else if s.externalNetworkingDisabled { + nic.disable() } _ = s.NextNICID() } - s.tables = st.tables - s.nftables = st.nftables } // Restore restarts the stack after a restore. This must be called after the @@ -2060,7 +2139,6 @@ func (s *Stack) Restore() { s.mu.Lock() eps := s.restoredEndpoints s.restoredEndpoints = nil - saveRestoreEnabled := s.saveRestoreEnabled s.mu.Unlock() for _, e := range eps { e.Restore(s) @@ -2070,13 +2148,9 @@ func (s *Stack) Restore() { // protocol level background workers. tcpip.AsyncLoading.Wait() - // Now resume any protocol level background workers. + // Now restore any protocol level background workers. for _, p := range s.transportProtocols { - if saveRestoreEnabled { - p.proto.Restore() - } else { - p.proto.Resume() - } + p.proto.Restore() } } @@ -2152,6 +2226,12 @@ func (s *Stack) unregisterPacketEndpointLocked(nicID tcpip.NICID, netProto tcpip // WritePacketToRemote writes a payload on the specified NIC using the provided // network protocol and remote link address. func (s *Stack) WritePacketToRemote(nicID tcpip.NICID, remote tcpip.LinkAddress, netProto tcpip.NetworkProtocolNumber, payload buffer.Buffer) tcpip.Error { + return s.WritePacketToRemoteWithMark(nicID, remote, netProto, payload, 0) +} + +// WritePacketToRemoteWithMark writes a payload on the specified NIC using the +// provided network protocol, remote link address, and packet mark. +func (s *Stack) WritePacketToRemoteWithMark(nicID tcpip.NICID, remote tcpip.LinkAddress, netProto tcpip.NetworkProtocolNumber, payload buffer.Buffer, mark uint32) tcpip.Error { s.mu.Lock() nic, ok := s.nics[nicID] s.mu.Unlock() @@ -2161,6 +2241,7 @@ func (s *Stack) WritePacketToRemote(nicID tcpip.NICID, remote tcpip.LinkAddress, pkt := NewPacketBuffer(PacketBufferOptions{ ReserveHeaderBytes: int(nic.MaxHeaderLength()), Payload: payload, + Mark: mark, }) defer pkt.DecRef() pkt.NetworkProtocolNumber = netProto @@ -2170,6 +2251,12 @@ func (s *Stack) WritePacketToRemote(nicID tcpip.NICID, remote tcpip.LinkAddress, // WriteRawPacket writes data directly to the specified NIC without adding any // headers. func (s *Stack) WriteRawPacket(nicID tcpip.NICID, proto tcpip.NetworkProtocolNumber, payload buffer.Buffer) tcpip.Error { + return s.WriteRawPacketWithMark(nicID, proto, payload, 0) +} + +// WriteRawPacketWithMark writes data directly to the specified NIC without adding any +// headers, setting the specified packet mark. +func (s *Stack) WriteRawPacketWithMark(nicID tcpip.NICID, proto tcpip.NetworkProtocolNumber, payload buffer.Buffer, mark uint32) tcpip.Error { s.mu.RLock() nic, ok := s.nics[nicID] s.mu.RUnlock() @@ -2179,6 +2266,7 @@ func (s *Stack) WriteRawPacket(nicID tcpip.NICID, proto tcpip.NetworkProtocolNum pkt := NewPacketBuffer(PacketBufferOptions{ Payload: payload, + Mark: mark, }) defer pkt.DecRef() pkt.NetworkProtocolNumber = proto @@ -2244,14 +2332,47 @@ func (s *Stack) IPTables() *IPTables { return s.tables } +// SetIPTables sets the stack's iptables. +func (s *Stack) SetIPTables(tables *IPTables) { + s.tables = tables +} + // NFTables returns the stack's nftables. func (s *Stack) NFTables() NFTablesInterface { - return s.nftables + val := s.nftables.Load() + if val == nil { + return nil + } + return *val } // SetNFTables sets the stack's nftables. func (s *Stack) SetNFTables(nft NFTablesInterface) { - s.nftables = nft + if nft == nil { + s.nftables.Store(nil) + } else { + s.nftables.Store(&nft) + } +} + +// LockNFTablesUpdate locks the stack's nftables update mutex for netlink batch modification. +func (s *Stack) LockNFTablesUpdate() { + s.nftablesUpdateMu.Lock() +} + +// UnlockNFTablesUpdate unlocks the stack's nftables update mutex. +func (s *Stack) UnlockNFTablesUpdate() { + s.nftablesUpdateMu.Unlock() +} + +// IsNFTablesConfigured returns true if the stack has nftables configured. +func (s *Stack) IsNFTablesConfigured() bool { + return s.nftablesConfigured.Load() +} + +// SetNFTablesConfigured sets whether the stack has nftables configured. +func (s *Stack) SetNFTablesConfigured(configured bool) { + s.nftablesConfigured.Store(configured) } // ICMPLimit returns the maximum number of ICMP messages that can be sent @@ -2460,12 +2581,11 @@ func (s *Stack) SetNICStack(id tcpip.NICID, peer *Stack) (tcpip.NICID, tcpip.Err s.mu.Unlock() return id, nil } - delete(s.nics, id) - // Remove routes in-place. n tracks the number of routes written. - s.RemoveRoutes(func(r tcpip.Route) bool { return r.NIC == id }) - ne := nic.NetworkLinkEndpoint.(LinkEndpoint) - deferAct, err := nic.remove(false /* closeLinkEndpoint */) + linkEp := nic.NetworkLinkEndpoint.(LinkEndpoint) + name := nic.Name() + + deferAct, err := s.removeNICLocked(id, false /* closeLinkEndpoint */) s.mu.Unlock() if deferAct != nil { deferAct() @@ -2475,34 +2595,71 @@ func (s *Stack) SetNICStack(id tcpip.NICID, peer *Stack) (tcpip.NICID, tcpip.Err } id = tcpip.NICID(peer.NextNICID()) - return id, peer.CreateNICWithOptions(id, ne, NICOptions{Name: nic.Name()}) + return id, peer.CreateNICWithOptions(id, linkEp, NICOptions{Name: name}) } -// EnableSaveRestore marks the saveRestoreEnabled to true. -func (s *Stack) EnableSaveRestore() { +// SetRemoveConf sets the removeConf in stack to the given value. +func (s *Stack) SetRemoveConf(removeConf bool) { s.mu.Lock() defer s.mu.Unlock() - - s.saveRestoreEnabled = true + s.removeConf = removeConf } -// IsSaveRestoreEnabled returns true if save restore is enabled for the stack. -func (s *Stack) IsSaveRestoreEnabled() bool { +// GetRemoveConf gets the removeConf from stack. +func (s *Stack) GetRemoveConf() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.removeConf +} + +// SetAllowConnectedOnSave sets allowConnectedOnSave in stack with the given value. +func (s *Stack) SetAllowConnectedOnSave(allowConnectedOnSave bool) { s.mu.Lock() defer s.mu.Unlock() - - return s.saveRestoreEnabled + s.allowConnectedOnSave = allowConnectedOnSave } -// contextID is this package's type for context.Context.Value keys. -type contextID int - -const ( - // CtxRestoreStack is a Context.Value key for the stack to be used in restore. - CtxRestoreStack contextID = iota -) - -// RestoreStackFromContext returns the stack to be used during restore. -func RestoreStackFromContext(ctx context.Context) *Stack { - return ctx.Value(CtxRestoreStack).(*Stack) +// GetAllowConnectedOnSave gets the allowConnectedOnSave from stack. +func (s *Stack) GetAllowConnectedOnSave() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.allowConnectedOnSave +} + +// AllowLiveTCPMigration returns if TCP connections can be migrated. +func (s *Stack) AllowLiveTCPMigration() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.allowLiveTCPMigration +} + +// SetAllowLiveTCPMigration sets if TCP connections can be migrated. +func (s *Stack) SetAllowLiveTCPMigration(allow bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.allowLiveTCPMigration = allow +} + +// DisableAllNonLoopbackNICs disables all non-loopback NICs in the stack. +func (s *Stack) DisableAllNonLoopbackNICs() { + s.mu.Lock() + defer s.mu.Unlock() + s.externalNetworkingDisabled = true + for _, nic := range s.nics { + if !nic.IsLoopback() { + nic.disable() + } + } +} + +// EnableAllNonLoopbackNICs enables all non-loopback NICs in the stack. +func (s *Stack) EnableAllNonLoopbackNICs() { + s.mu.Lock() + defer s.mu.Unlock() + s.externalNetworkingDisabled = false + for _, nic := range s.nics { + if !nic.IsLoopback() { + nic.enable() + } + } } diff --git a/pkg/tcpip/stack/stack_mutex.go b/pkg/tcpip/stack/stack_mutex.go index 811c819..d16728c 100644 --- a/pkg/tcpip/stack/stack_mutex.go +++ b/pkg/tcpip/stack/stack_mutex.go @@ -92,5 +92,5 @@ func stackinitLockNames() {} func init() { stackinitLockNames() - stackprefixIndex = locking.NewMutexClass(reflect.TypeOf(stackRWMutex{}), stacklockNames) + stackprefixIndex = locking.NewMutexClass(reflect.TypeFor[stackRWMutex](), stacklockNames) } diff --git a/pkg/tcpip/stack/stack_options.go b/pkg/tcpip/stack/stack_options.go index 02dfebe..4f147a6 100644 --- a/pkg/tcpip/stack/stack_options.go +++ b/pkg/tcpip/stack/stack_options.go @@ -37,14 +37,6 @@ const ( defaultTCPInvalidRateLimit = 500 * time.Millisecond ) -// ReceiveBufferSizeOption is used by stack.(Stack*).Option/SetOption to -// get/set the default, min and max receive buffer sizes. -type ReceiveBufferSizeOption struct { - Min int - Default int - Max int -} - // TCPInvalidRateLimitOption is used by stack.(Stack*).Option/SetOption to get/set // stack.tcpInvalidRateLimit. type TCPInvalidRateLimitOption time.Duration diff --git a/pkg/tcpip/stack/stack_state_autogen.go b/pkg/tcpip/stack/stack_state_autogen.go index a9cd134..146fe9f 100644 --- a/pkg/tcpip/stack/stack_state_autogen.go +++ b/pkg/tcpip/stack/stack_state_autogen.go @@ -244,6 +244,7 @@ func (cn *conn) StateFields() []string { "destinationManip", "tcb", "lastUsed", + "replySeen", } } @@ -260,6 +261,7 @@ func (cn *conn) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(5, &cn.destinationManip) stateSinkObject.Save(6, &cn.tcb) stateSinkObject.Save(7, &cn.lastUsed) + stateSinkObject.Save(8, &cn.replySeen) } func (cn *conn) afterLoad(context.Context) {} @@ -274,6 +276,7 @@ func (cn *conn) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(5, &cn.destinationManip) stateSourceObject.Load(6, &cn.tcb) stateSourceObject.Load(7, &cn.lastUsed) + stateSourceObject.Load(8, &cn.replySeen) } func (ct *ConnTrack) StateTypeName() string { @@ -283,6 +286,7 @@ func (ct *ConnTrack) StateTypeName() string { func (ct *ConnTrack) StateFields() []string { return []string{ "seed", + "nftIDSeed", "clock", "buckets", } @@ -294,8 +298,9 @@ func (ct *ConnTrack) beforeSave() {} func (ct *ConnTrack) StateSave(stateSinkObject state.Sink) { ct.beforeSave() stateSinkObject.Save(0, &ct.seed) - stateSinkObject.Save(1, &ct.clock) - stateSinkObject.Save(2, &ct.buckets) + stateSinkObject.Save(1, &ct.nftIDSeed) + stateSinkObject.Save(2, &ct.clock) + stateSinkObject.Save(3, &ct.buckets) } func (ct *ConnTrack) afterLoad(context.Context) {} @@ -303,8 +308,9 @@ func (ct *ConnTrack) afterLoad(context.Context) {} // +checklocksignore func (ct *ConnTrack) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &ct.seed) - stateSourceObject.Load(1, &ct.clock) - stateSourceObject.Load(2, &ct.buckets) + stateSourceObject.Load(1, &ct.nftIDSeed) + stateSourceObject.Load(2, &ct.clock) + stateSourceObject.Load(3, &ct.buckets) } func (bkt *bucket) StateTypeName() string { @@ -339,6 +345,8 @@ func (l *ICMPRateLimiter) StateTypeName() string { func (l *ICMPRateLimiter) StateFields() []string { return []string{ "clock", + "limit", + "burst", } } @@ -348,13 +356,16 @@ func (l *ICMPRateLimiter) beforeSave() {} func (l *ICMPRateLimiter) StateSave(stateSinkObject state.Sink) { l.beforeSave() stateSinkObject.Save(0, &l.clock) + stateSinkObject.Save(1, &l.limit) + stateSinkObject.Save(2, &l.burst) } -func (l *ICMPRateLimiter) afterLoad(context.Context) {} - // +checklocksignore func (l *ICMPRateLimiter) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &l.clock) + stateSourceObject.Load(1, &l.limit) + stateSourceObject.Load(2, &l.burst) + stateSourceObject.AfterLoad(func() { l.afterLoad(ctx) }) } func (a *AcceptTarget) StateTypeName() string { @@ -668,6 +679,34 @@ func (mt *MasqueradeTarget) StateLoad(ctx context.Context, stateSourceObject sta stateSourceObject.Load(0, &mt.NetworkProtocol) } +func (c *CTTarget) StateTypeName() string { + return "pkg/tcpip/stack.CTTarget" +} + +func (c *CTTarget) StateFields() []string { + return []string{ + "NetworkProtocol", + "Zone", + } +} + +func (c *CTTarget) beforeSave() {} + +// +checklocksignore +func (c *CTTarget) StateSave(stateSinkObject state.Sink) { + c.beforeSave() + stateSinkObject.Save(0, &c.NetworkProtocol) + stateSinkObject.Save(1, &c.Zone) +} + +func (c *CTTarget) afterLoad(context.Context) {} + +// +checklocksignore +func (c *CTTarget) StateLoad(ctx context.Context, stateSourceObject state.Source) { + stateSourceObject.Load(0, &c.NetworkProtocol) + stateSourceObject.Load(1, &c.Zone) +} + func (it *IPTables) StateTypeName() string { return "pkg/tcpip/stack.IPTables" } @@ -675,7 +714,6 @@ func (it *IPTables) StateTypeName() string { func (it *IPTables) StateFields() []string { return []string{ "connections", - "reaper", "v4Tables", "v6Tables", "modified", @@ -686,19 +724,17 @@ func (it *IPTables) StateFields() []string { func (it *IPTables) StateSave(stateSinkObject state.Sink) { it.beforeSave() stateSinkObject.Save(0, &it.connections) - stateSinkObject.Save(1, &it.reaper) - stateSinkObject.Save(2, &it.v4Tables) - stateSinkObject.Save(3, &it.v6Tables) - stateSinkObject.Save(4, &it.modified) + stateSinkObject.Save(1, &it.v4Tables) + stateSinkObject.Save(2, &it.v6Tables) + stateSinkObject.Save(3, &it.modified) } // +checklocksignore func (it *IPTables) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &it.connections) - stateSourceObject.Load(1, &it.reaper) - stateSourceObject.Load(2, &it.v4Tables) - stateSourceObject.Load(3, &it.v6Tables) - stateSourceObject.Load(4, &it.modified) + stateSourceObject.Load(1, &it.v4Tables) + stateSourceObject.Load(2, &it.v6Tables) + stateSourceObject.Load(3, &it.modified) stateSourceObject.AfterLoad(func() { it.afterLoad(ctx) }) } @@ -1530,8 +1566,10 @@ func (pk *PacketBuffer) StateFields() []string { "dnatDone", "PktType", "NICID", + "InputNICID", "RXChecksumValidated", "NetworkPacketInfo", + "Mark", "tuple", } } @@ -1557,9 +1595,11 @@ func (pk *PacketBuffer) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(13, &pk.dnatDone) stateSinkObject.Save(14, &pk.PktType) stateSinkObject.Save(15, &pk.NICID) - stateSinkObject.Save(16, &pk.RXChecksumValidated) - stateSinkObject.Save(17, &pk.NetworkPacketInfo) - stateSinkObject.Save(18, &pk.tuple) + stateSinkObject.Save(16, &pk.InputNICID) + stateSinkObject.Save(17, &pk.RXChecksumValidated) + stateSinkObject.Save(18, &pk.NetworkPacketInfo) + stateSinkObject.Save(19, &pk.Mark) + stateSinkObject.Save(20, &pk.tuple) } func (pk *PacketBuffer) afterLoad(context.Context) {} @@ -1582,9 +1622,11 @@ func (pk *PacketBuffer) StateLoad(ctx context.Context, stateSourceObject state.S stateSourceObject.Load(13, &pk.dnatDone) stateSourceObject.Load(14, &pk.PktType) stateSourceObject.Load(15, &pk.NICID) - stateSourceObject.Load(16, &pk.RXChecksumValidated) - stateSourceObject.Load(17, &pk.NetworkPacketInfo) - stateSourceObject.Load(18, &pk.tuple) + stateSourceObject.Load(16, &pk.InputNICID) + stateSourceObject.Load(17, &pk.RXChecksumValidated) + stateSourceObject.Load(18, &pk.NetworkPacketInfo) + stateSourceObject.Load(19, &pk.Mark) + stateSourceObject.Load(20, &pk.tuple) } func (h *headerInfo) StateTypeName() string { @@ -2093,12 +2135,12 @@ func (s *Stack) StateFields() []string { "packetEndpointWriteSupported", "demux", "stats", - "loopbackNIC", "defaultForwardingEnabled", "cleanupEndpoints", "PortManager", "clock", "handleLocal", + "nftablesConfigured", "restoredEndpoints", "resumableEndpoints", "icmpRateLimiter", @@ -2109,12 +2151,11 @@ func (s *Stack) StateFields() []string { "receiveBufferSize", "tcpInvalidRateLimit", "tsOffsetSecret", - "saveRestoreEnabled", + "externalNetworkingDisabled", + "allowConnectedOnSave", } } -func (s *Stack) beforeSave() {} - // +checklocksignore func (s *Stack) StateSave(stateSinkObject state.Sink) { s.beforeSave() @@ -2124,12 +2165,12 @@ func (s *Stack) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(3, &s.packetEndpointWriteSupported) stateSinkObject.Save(4, &s.demux) stateSinkObject.Save(5, &s.stats) - stateSinkObject.Save(6, &s.loopbackNIC) - stateSinkObject.Save(7, &s.defaultForwardingEnabled) - stateSinkObject.Save(8, &s.cleanupEndpoints) - stateSinkObject.Save(9, &s.PortManager) - stateSinkObject.Save(10, &s.clock) - stateSinkObject.Save(11, &s.handleLocal) + stateSinkObject.Save(6, &s.defaultForwardingEnabled) + stateSinkObject.Save(7, &s.cleanupEndpoints) + stateSinkObject.Save(8, &s.PortManager) + stateSinkObject.Save(9, &s.clock) + stateSinkObject.Save(10, &s.handleLocal) + stateSinkObject.Save(11, &s.nftablesConfigured) stateSinkObject.Save(12, &s.restoredEndpoints) stateSinkObject.Save(13, &s.resumableEndpoints) stateSinkObject.Save(14, &s.icmpRateLimiter) @@ -2140,7 +2181,8 @@ func (s *Stack) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(19, &s.receiveBufferSize) stateSinkObject.Save(20, &s.tcpInvalidRateLimit) stateSinkObject.Save(21, &s.tsOffsetSecret) - stateSinkObject.Save(22, &s.saveRestoreEnabled) + stateSinkObject.Save(22, &s.externalNetworkingDisabled) + stateSinkObject.Save(23, &s.allowConnectedOnSave) } // +checklocksignore @@ -2151,12 +2193,12 @@ func (s *Stack) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(3, &s.packetEndpointWriteSupported) stateSourceObject.Load(4, &s.demux) stateSourceObject.Load(5, &s.stats) - stateSourceObject.Load(6, &s.loopbackNIC) - stateSourceObject.Load(7, &s.defaultForwardingEnabled) - stateSourceObject.Load(8, &s.cleanupEndpoints) - stateSourceObject.Load(9, &s.PortManager) - stateSourceObject.Load(10, &s.clock) - stateSourceObject.Load(11, &s.handleLocal) + stateSourceObject.Load(6, &s.defaultForwardingEnabled) + stateSourceObject.Load(7, &s.cleanupEndpoints) + stateSourceObject.Load(8, &s.PortManager) + stateSourceObject.Load(9, &s.clock) + stateSourceObject.Load(10, &s.handleLocal) + stateSourceObject.Load(11, &s.nftablesConfigured) stateSourceObject.Load(12, &s.restoredEndpoints) stateSourceObject.Load(13, &s.resumableEndpoints) stateSourceObject.Load(14, &s.icmpRateLimiter) @@ -2167,7 +2209,8 @@ func (s *Stack) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(19, &s.receiveBufferSize) stateSourceObject.Load(20, &s.tcpInvalidRateLimit) stateSourceObject.Load(21, &s.tsOffsetSecret) - stateSourceObject.Load(22, &s.saveRestoreEnabled) + stateSourceObject.Load(22, &s.externalNetworkingDisabled) + stateSourceObject.Load(23, &s.allowConnectedOnSave) stateSourceObject.AfterLoad(func() { s.afterLoad(ctx) }) } @@ -2442,6 +2485,7 @@ func init() { state.Register((*RedirectTarget)(nil)) state.Register((*SNATTarget)(nil)) state.Register((*MasqueradeTarget)(nil)) + state.Register((*CTTarget)(nil)) state.Register((*IPTables)(nil)) state.Register((*Table)(nil)) state.Register((*Rule)(nil)) diff --git a/pkg/tcpip/stack/state_conn_mutex.go b/pkg/tcpip/stack/state_conn_mutex.go index 62af231..f7d72ad 100644 --- a/pkg/tcpip/stack/state_conn_mutex.go +++ b/pkg/tcpip/stack/state_conn_mutex.go @@ -92,5 +92,5 @@ func stateConninitLockNames() {} func init() { stateConninitLockNames() - stateConnprefixIndex = locking.NewMutexClass(reflect.TypeOf(stateConnRWMutex{}), stateConnlockNames) + stateConnprefixIndex = locking.NewMutexClass(reflect.TypeFor[stateConnRWMutex](), stateConnlockNames) } diff --git a/pkg/tcpip/stack/transport_endpoints_mutex.go b/pkg/tcpip/stack/transport_endpoints_mutex.go index 6083a4a..58f536c 100644 --- a/pkg/tcpip/stack/transport_endpoints_mutex.go +++ b/pkg/tcpip/stack/transport_endpoints_mutex.go @@ -92,5 +92,5 @@ func transportEndpointsinitLockNames() {} func init() { transportEndpointsinitLockNames() - transportEndpointsprefixIndex = locking.NewMutexClass(reflect.TypeOf(transportEndpointsRWMutex{}), transportEndpointslockNames) + transportEndpointsprefixIndex = locking.NewMutexClass(reflect.TypeFor[transportEndpointsRWMutex](), transportEndpointslockNames) } diff --git a/pkg/tcpip/stdclock.go b/pkg/tcpip/stdclock.go index e80e7c4..cc3397c 100644 --- a/pkg/tcpip/stdclock.go +++ b/pkg/tcpip/stdclock.go @@ -91,7 +91,6 @@ func (*stdClock) AfterFunc(d time.Duration, f func()) Timer { } } -// +stateify savable type stdTimer struct { t *time.Timer } diff --git a/pkg/tcpip/tcpip.go b/pkg/tcpip/tcpip.go index f1add0d..9f9d73c 100644 --- a/pkg/tcpip/tcpip.go +++ b/pkg/tcpip/tcpip.go @@ -49,10 +49,8 @@ import ( // Using the header package here would cause an import cycle. const ( - ipv4AddressSize = 4 - ipv4ProtocolNumber = 0x0800 - ipv6AddressSize = 16 - ipv6ProtocolNumber = 0x86dd + ipv4AddressSize = 4 + ipv6AddressSize = 16 ) const ( @@ -714,6 +712,10 @@ type ReadOptions struct { // NeedLinkPacketInfo indicates whether to return the link-layer information, // if supported. NeedLinkPacketInfo bool + + // NeedRecvdExperimentOption indicates whether to return the experiment + // option value from the last received packet, if supported. + NeedReceivedExperimentOption bool } // ReadResult represents result for a successful Endpoint.Read. @@ -734,6 +736,10 @@ type ReadResult struct { // LinkPacketInfo is the link-layer information of the received packet if // ReadOptions.NeedLinkPacketInfo is true. LinkPacketInfo LinkPacketInfo + + // ReceivedExperimentOption is the experiment option value from the last + // received packet if ReadOptions.NeedReceivedExperimentOption is true. + ReceivedExperimentOption uint16 } // Endpoint is the interface implemented by transport protocols (e.g., tcp, udp) @@ -951,9 +957,8 @@ const ( // MTUDiscoverOption is used to set/get the path MTU discovery setting. // - // NOTE: Setting this option to any other value than PMTUDiscoveryDont - // is not supported and will fail as such, and getting this option will - // always return PMTUDiscoveryDont. + // The value controls whether the Don't Fragment (DF) bit is set on + // outgoing IPv4 packets. MTUDiscoverOption // MulticastTTLOption is used by SetSockOptInt/GetSockOptInt to control @@ -1005,6 +1010,10 @@ const ( // PacketMMapReserveOption is used to set the packet mmap reserved space // between the aligned header and the payload. PacketMMapReserveOption + + // IPv6MulticastInterfaceOption is used to set/get the NIC used for + // IPv6 multicast Tx. + IPv6MulticastInterfaceOption ) const ( diff --git a/pkg/tcpip/tcpip_state_autogen.go b/pkg/tcpip/tcpip_state_autogen.go index db7dcd7..bbd937b 100644 --- a/pkg/tcpip/tcpip_state_autogen.go +++ b/pkg/tcpip/tcpip_state_autogen.go @@ -1097,9 +1097,10 @@ func (so *SocketOptions) StateFields() []string { "bindToDevice", "sendBufferSize", "receiveBufferSize", - "linger", "rcvlowat", "experimentOptionValue", + "mark", + "linger", } } @@ -1134,9 +1135,10 @@ func (so *SocketOptions) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(23, &so.bindToDevice) stateSinkObject.Save(24, &so.sendBufferSize) stateSinkObject.Save(25, &so.receiveBufferSize) - stateSinkObject.Save(26, &so.linger) - stateSinkObject.Save(27, &so.rcvlowat) - stateSinkObject.Save(28, &so.experimentOptionValue) + stateSinkObject.Save(26, &so.rcvlowat) + stateSinkObject.Save(27, &so.experimentOptionValue) + stateSinkObject.Save(28, &so.mark) + stateSinkObject.Save(29, &so.linger) } func (so *SocketOptions) afterLoad(context.Context) {} @@ -1169,9 +1171,10 @@ func (so *SocketOptions) StateLoad(ctx context.Context, stateSourceObject state. stateSourceObject.Load(23, &so.bindToDevice) stateSourceObject.Load(24, &so.sendBufferSize) stateSourceObject.Load(25, &so.receiveBufferSize) - stateSourceObject.Load(26, &so.linger) - stateSourceObject.Load(27, &so.rcvlowat) - stateSourceObject.Load(28, &so.experimentOptionValue) + stateSourceObject.Load(26, &so.rcvlowat) + stateSourceObject.Load(27, &so.experimentOptionValue) + stateSourceObject.Load(28, &so.mark) + stateSourceObject.Load(29, &so.linger) } func (l *LocalSockError) StateTypeName() string { @@ -1264,31 +1267,6 @@ func (s *stdClock) StateLoad(ctx context.Context, stateSourceObject state.Source stateSourceObject.AfterLoad(func() { s.afterLoad(ctx) }) } -func (st *stdTimer) StateTypeName() string { - return "pkg/tcpip.stdTimer" -} - -func (st *stdTimer) StateFields() []string { - return []string{ - "t", - } -} - -func (st *stdTimer) beforeSave() {} - -// +checklocksignore -func (st *stdTimer) StateSave(stateSinkObject state.Sink) { - st.beforeSave() - stateSinkObject.Save(0, &st.t) -} - -func (st *stdTimer) afterLoad(context.Context) {} - -// +checklocksignore -func (st *stdTimer) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &st.t) -} - func (mt *MonotonicTime) StateTypeName() string { return "pkg/tcpip.MonotonicTime" } @@ -1505,8 +1483,8 @@ func (c *ReceivableControlMessages) beforeSave() {} // +checklocksignore func (c *ReceivableControlMessages) StateSave(stateSinkObject state.Sink) { c.beforeSave() - var TimestampValue int64 - TimestampValue = c.saveTimestamp() + TimestampValue := c.saveTimestamp() + _ = (int64)(TimestampValue) stateSinkObject.SaveValue(0, TimestampValue) stateSinkObject.Save(1, &c.HasInq) stateSinkObject.Save(2, &c.Inq) @@ -3224,7 +3202,6 @@ func (j *jobInstance) StateTypeName() string { func (j *jobInstance) StateFields() []string { return []string{ - "timer", "earlyReturn", } } @@ -3234,16 +3211,14 @@ func (j *jobInstance) beforeSave() {} // +checklocksignore func (j *jobInstance) StateSave(stateSinkObject state.Sink) { j.beforeSave() - stateSinkObject.Save(0, &j.timer) - stateSinkObject.Save(1, &j.earlyReturn) + stateSinkObject.Save(0, &j.earlyReturn) } func (j *jobInstance) afterLoad(context.Context) {} // +checklocksignore func (j *jobInstance) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &j.timer) - stateSourceObject.Load(1, &j.earlyReturn) + stateSourceObject.Load(0, &j.earlyReturn) } func (j *Job) StateTypeName() string { @@ -3328,7 +3303,6 @@ func init() { state.Register((*LocalSockError)(nil)) state.Register((*SockError)(nil)) state.Register((*stdClock)(nil)) - state.Register((*stdTimer)(nil)) state.Register((*MonotonicTime)(nil)) state.Register((*Address)(nil)) state.Register((*AddressMask)(nil)) diff --git a/pkg/tcpip/timer.go b/pkg/tcpip/timer.go index 91bc001..dd5dde2 100644 --- a/pkg/tcpip/timer.go +++ b/pkg/tcpip/timer.go @@ -59,7 +59,7 @@ import ( // // +stateify savable type jobInstance struct { - timer Timer + timer Timer `state:"nosave"` // Used to inform the timer to early return when it gets stopped while the // lock the timer tries to obtain when fired is held (T1 is a goroutine that diff --git a/pkg/tcpip/transport/icmp/endpoint_state.go b/pkg/tcpip/transport/icmp/endpoint_state.go index 122f9d6..677741f 100644 --- a/pkg/tcpip/transport/icmp/endpoint_state.go +++ b/pkg/tcpip/transport/icmp/endpoint_state.go @@ -16,13 +16,11 @@ package icmp import ( "context" - "fmt" "time" "github.com/sagernet/gvisor/pkg/log" "github.com/sagernet/gvisor/pkg/tcpip" "github.com/sagernet/gvisor/pkg/tcpip/stack" - "github.com/sagernet/gvisor/pkg/tcpip/transport" ) // saveReceivedAt is invoked by stateify. @@ -37,11 +35,7 @@ func (p *icmpPacket) loadReceivedAt(_ context.Context, nsec int64) { // afterLoad is invoked by stateify. func (e *endpoint) afterLoad(ctx context.Context) { - if e.stack.IsSaveRestoreEnabled() { - e.stack.RegisterRestoredEndpoint(e) - } else { - stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) - } + e.stack.RegisterRestoredEndpoint(e) } // beforeSave is invoked by stateify. @@ -59,31 +53,7 @@ func (e *endpoint) Restore(s *stack.Stack) { } e.thaw() - if e.stack.IsSaveRestoreEnabled() { - e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) - return - } - - e.stack = s e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) - - e.mu.Lock() - defer e.mu.Unlock() - - switch state := e.net.State(); state { - case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: - case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: - var err tcpip.Error - info := e.net.Info() - info.ID.LocalPort = e.ident - info.ID, err = e.registerWithStack(info.NetProto, info.ID) - if err != nil { - panic(fmt.Sprintf("e.registerWithStack(%d, %#v): %s", info.NetProto, info.ID, err)) - } - e.ident = info.ID.LocalPort - default: - panic(fmt.Sprintf("unhandled state = %s", state)) - } } // Resume implements tcpip.ResumableEndpoint.Resume. diff --git a/pkg/tcpip/transport/icmp/icmp_state_autogen.go b/pkg/tcpip/transport/icmp/icmp_state_autogen.go index 33af387..7a94ee6 100644 --- a/pkg/tcpip/transport/icmp/icmp_state_autogen.go +++ b/pkg/tcpip/transport/icmp/icmp_state_autogen.go @@ -29,8 +29,8 @@ func (p *icmpPacket) beforeSave() {} // +checklocksignore func (p *icmpPacket) StateSave(stateSinkObject state.Sink) { p.beforeSave() - var receivedAtValue int64 - receivedAtValue = p.saveReceivedAt() + receivedAtValue := p.saveReceivedAt() + _ = (int64)(receivedAtValue) stateSinkObject.SaveValue(4, receivedAtValue) stateSinkObject.Save(0, &p.icmpPacketEntry) stateSinkObject.Save(1, &p.senderAddress) diff --git a/pkg/tcpip/transport/internal/network/endpoint.go b/pkg/tcpip/transport/internal/network/endpoint.go index da60506..676e903 100644 --- a/pkg/tcpip/transport/internal/network/endpoint.go +++ b/pkg/tcpip/transport/internal/network/endpoint.go @@ -66,13 +66,16 @@ type Endpoint struct { // TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6. // +checklocks:mu multicastAddr tcpip.Address - // TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6. // +checklocks:mu multicastNICID tcpip.NICID // +checklocks:mu + ipv6MulticastNICID tcpip.NICID + // +checklocks:mu ipv4TOS uint8 // +checklocks:mu ipv6TClass uint8 + // +checklocks:mu + pmtud tcpip.PMTUDStrategy // Lock ordering: mu > infoMu. infoMu sync.RWMutex `state:"nosave"` @@ -181,7 +184,11 @@ func (e *Endpoint) Close() { } for mem := range e.multicastMemberships { - e.stack.LeaveGroup(e.netProto, mem.nicID, mem.multicastAddr) + proto, err := e.multicastNetProto(mem.multicastAddr) + if err != nil { + panic("non multicast address in an existing membership") + } + e.stack.LeaveGroup(proto, mem.nicID, mem.multicastAddr) } e.multicastMemberships = nil @@ -229,6 +236,7 @@ type WriteContext struct { route *stack.Route ttl uint8 tos uint8 + df bool } func (c *WriteContext) MTU() uint32 { @@ -274,33 +282,33 @@ func (c *WriteContext) TryNewPacketBuffer(reserveHdrBytes int, data buffer.Buffe if !e.hasSendSpaceRLocked() { return nil } - return c.newPacketBufferLocked(reserveHdrBytes, data) + + mark := e.ops.GetMark() + return c.newPacketBufferLocked(reserveHdrBytes, data, mark) } -// TryNewPacketBufferFromPayloader returns a new packet buffer iff the endpoint's send buffer +// TryNewPacketBufferFromPayloader returns a new packet buffer if the endpoint's send buffer // is not full. Otherwise, data from `payloader` isn't read. -// -// If this method returns nil, the caller should wait for the endpoint to become -// writable. -func (c *WriteContext) TryNewPacketBufferFromPayloader(reserveHdrBytes int, payloader tcpip.Payloader) *stack.PacketBuffer { +func (c *WriteContext) TryNewPacketBufferFromPayloader(reserveHdrBytes int, payloader tcpip.Payloader) (*stack.PacketBuffer, tcpip.Error) { e := c.e e.sendBufferSizeInUseMu.Lock() defer e.sendBufferSizeInUseMu.Unlock() if !e.hasSendSpaceRLocked() { - return nil + return nil, &tcpip.ErrWouldBlock{} } var data buffer.Buffer if _, err := data.WriteFromReader(payloader, int64(payloader.Len())); err != nil { data.Release() - return nil + return nil, &tcpip.ErrBadBuffer{} } - return c.newPacketBufferLocked(reserveHdrBytes, data) + mark := e.ops.GetMark() + return c.newPacketBufferLocked(reserveHdrBytes, data, mark), nil } // +checklocks:c.e.sendBufferSizeInUseMu -func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Buffer) *stack.PacketBuffer { +func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Buffer, mark uint32) *stack.PacketBuffer { e := c.e // Note that we allow oversubscription - if there is any space at all in the // send buffer, we accept the full packet which may be larger than the space @@ -323,6 +331,7 @@ func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Bu return stack.NewPacketBuffer(stack.PacketBufferOptions{ ReserveHeaderBytes: reserveHdrBytes, Payload: data, + Mark: mark, OnRelease: func() { e.sendBufferSizeInUseMu.Lock() if got := e.sendBufferSizeInUse; got < pktSize { @@ -360,6 +369,7 @@ func (c *WriteContext) WritePacket(pkt *stack.PacketBuffer, headerIncluded bool) Protocol: c.e.transProto, TTL: c.ttl, TOS: c.tos, + DF: c.df, ExperimentOptionValue: expOptVal, }, pkt) @@ -433,8 +443,8 @@ func (e *Endpoint) AcquireContextForWrite(opts tcpip.WriteOptions) (WriteContext route := e.connectedRoute to := opts.To info := e.Info() - switch { - case to == nil: + switch to { + case nil: // If the user doesn't specify a destination, they should have // connected to another address. if e.State() != transport.DatagramEndpointStateConnected { @@ -566,11 +576,28 @@ func (e *Endpoint) AcquireContextForWrite(opts tcpip.WriteOptions) (WriteContext panic(fmt.Sprintf("invalid protocol number = %d", netProto)) } + // Set the DF (Don't Fragment) bit based on the PMTUD strategy, + // matching TCP behavior in connect.go. + // Note: In gVisor, WANT and DO are treated identically (both set DF). + // Linux kernel differentiates them (WANT allows local fragmentation, + // DO returns EMSGSIZE), but gVisor's IPv4 layer always allows local + // fragmentation for locally-generated packets regardless of DF + // (see gvisor.dev/issue/5919). + // + // PROBE also sets DF, matching Linux ip_dont_fragment(). In Linux, + // PROBE differs from DO only in that it ignores incoming ICMP + // "Fragmentation Needed" messages (i.e. does not update the cached + // route PMTU). Since gVisor does not implement ICMP-based PMTU + // feedback for transport sockets, PROBE and DO are functionally + // equivalent here. + df := e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo || e.pmtud == tcpip.PMTUDiscoveryProbe + return WriteContext{ e: e, route: route, ttl: ttl, tos: tos, + df: df, }, nil } @@ -613,7 +640,7 @@ func (e *Endpoint) connectRouteRLocked(nicID tcpip.NICID, localAddr tcpip.Addres localAddr = tcpip.Address{} } - if header.IsV4MulticastAddress(addr.Addr) || header.IsV6MulticastAddress(addr.Addr) { + if header.IsV4MulticastAddress(addr.Addr) { if nicID == 0 { nicID = e.multicastNICID } @@ -621,6 +648,9 @@ func (e *Endpoint) connectRouteRLocked(nicID tcpip.NICID, localAddr tcpip.Addres localAddr = e.multicastAddr } } + if header.IsV6MulticastAddress(addr.Addr) && nicID == 0 { + nicID = e.ipv6MulticastNICID + } } // Find a route to the desired destination. @@ -835,9 +865,18 @@ func (e *Endpoint) GetRemoteAddress() (tcpip.FullAddress, bool) { func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { switch opt { case tcpip.MTUDiscoverOption: - // Return not supported if the value is not disabling path - // MTU discovery. - if tcpip.PMTUDStrategy(v) != tcpip.PMTUDiscoveryDont { + // Store PMTU discovery settings. The DF bit on outgoing + // packets is set accordingly in AcquireContextForWrite. + // PROBE is accepted alongside DO/WANT/DONT. In Linux, + // PROBE sets DF but ignores ICMP-based PMTU updates; + // since gVisor lacks ICMP PMTU feedback, it behaves + // identically to DO. + switch tcpip.PMTUDStrategy(v) { + case tcpip.PMTUDiscoveryWant, tcpip.PMTUDiscoveryDont, tcpip.PMTUDiscoveryDo, tcpip.PMTUDiscoveryProbe: + e.mu.Lock() + e.pmtud = tcpip.PMTUDStrategy(v) + e.mu.Unlock() + default: return &tcpip.ErrNotSupported{} } @@ -865,6 +904,18 @@ func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { e.mu.Lock() e.ipv6TClass = uint8(v) e.mu.Unlock() + + case tcpip.IPv6MulticastInterfaceOption: + if v != 0 && !e.stack.CheckNIC(tcpip.NICID(v)) { + return &tcpip.ErrUnknownNICID{} + } + e.mu.Lock() + defer e.mu.Unlock() + nic := tcpip.NICID(v) + if info := e.Info(); info.BindNICID != 0 && info.BindNICID != nic { + return &tcpip.ErrInvalidEndpointState{} + } + e.ipv6MulticastNICID = nic } return nil @@ -874,8 +925,10 @@ func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error { func (e *Endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { switch opt { case tcpip.MTUDiscoverOption: - // The only supported setting is path MTU discovery disabled. - return int(tcpip.PMTUDiscoveryDont), nil + e.mu.Lock() + v := int(e.pmtud) + e.mu.Unlock() + return v, nil case tcpip.MulticastTTLOption: e.mu.Lock() @@ -907,11 +960,30 @@ func (e *Endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) { e.mu.RUnlock() return v, nil + case tcpip.IPv6MulticastInterfaceOption: + e.mu.RLock() + v := int(e.ipv6MulticastNICID) + e.mu.RUnlock() + return v, nil + default: return -1, &tcpip.ErrUnknownProtocolOption{} } } +// multicastNetProto returns the network protocol of a given multicast address. +// Returns an error if the address is not a multicast address. +func (e *Endpoint) multicastNetProto(addr tcpip.Address) (tcpip.NetworkProtocolNumber, tcpip.Error) { + switch { + case header.IsV4MulticastAddress(addr): + return header.IPv4ProtocolNumber, nil + case header.IsV6MulticastAddress(addr): + return header.IPv6ProtocolNumber, nil + default: + return 0, &tcpip.ErrInvalidOptionValue{} + } +} + // SetSockOpt sets the socket option. func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { switch v := opt.(type) { @@ -952,21 +1024,23 @@ func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { e.multicastAddr = addr case *tcpip.AddMembershipOption: - if !(header.IsV4MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv4ProtocolNumber) && !(header.IsV6MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv6ProtocolNumber) { - return &tcpip.ErrInvalidOptionValue{} + // Allowing IP_ADD_MEMBERSHIP on an ipv6 socket matches Linux behavior: + // https://github.com/torvalds/linux/blob/cec1e6e5d1a/net/ipv6/ipv6_sockglue.c#L964 + proto, err := e.multicastNetProto(v.MulticastAddr) + if err != nil { + return err } nicID := v.NIC - if v.InterfaceAddr.Unspecified() { if nicID == 0 { - if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, e.netProto, false /* multicastLoop */); err == nil { + if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, proto, false /* multicastLoop */); err == nil { nicID = r.NICID() r.Release() } } } else { - nicID = e.stack.CheckLocalAddress(nicID, e.netProto, v.InterfaceAddr) + nicID = e.stack.CheckLocalAddress(nicID, proto, v.InterfaceAddr) } if nicID == 0 { return &tcpip.ErrUnknownDevice{} @@ -981,27 +1055,28 @@ func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { return &tcpip.ErrPortInUse{} } - if err := e.stack.JoinGroup(e.netProto, nicID, v.MulticastAddr); err != nil { + if err := e.stack.JoinGroup(proto, nicID, v.MulticastAddr); err != nil { return err } e.multicastMemberships[memToInsert] = struct{}{} case *tcpip.RemoveMembershipOption: - if !(header.IsV4MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv4ProtocolNumber) && !(header.IsV6MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv6ProtocolNumber) { - return &tcpip.ErrInvalidOptionValue{} + proto, err := e.multicastNetProto(v.MulticastAddr) + if err != nil { + return err } nicID := v.NIC if v.InterfaceAddr.Unspecified() { if nicID == 0 { - if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, e.netProto, false /* multicastLoop */); err == nil { + if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, proto, false /* multicastLoop */); err == nil { nicID = r.NICID() r.Release() } } } else { - nicID = e.stack.CheckLocalAddress(nicID, e.netProto, v.InterfaceAddr) + nicID = e.stack.CheckLocalAddress(nicID, proto, v.InterfaceAddr) } if nicID == 0 { return &tcpip.ErrUnknownDevice{} @@ -1016,7 +1091,7 @@ func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error { return &tcpip.ErrBadLocalAddress{} } - if err := e.stack.LeaveGroup(e.netProto, nicID, v.MulticastAddr); err != nil { + if err := e.stack.LeaveGroup(proto, nicID, v.MulticastAddr); err != nil { return err } diff --git a/pkg/tcpip/transport/internal/network/endpoint_state.go b/pkg/tcpip/transport/internal/network/endpoint_state.go index c63c8b3..a90ca13 100644 --- a/pkg/tcpip/transport/internal/network/endpoint_state.go +++ b/pkg/tcpip/transport/internal/network/endpoint_state.go @@ -29,8 +29,12 @@ func (e *Endpoint) Resume(s *stack.Stack) error { e.stack = s for m := range e.multicastMemberships { - if err := e.stack.JoinGroup(e.netProto, m.nicID, m.multicastAddr); err != nil { - return fmt.Errorf("e.stack.JoinGroup(%d, %d, %s): %s", e.netProto, m.nicID, m.multicastAddr, err) + proto, err := e.multicastNetProto(m.multicastAddr) + if err != nil { + return fmt.Errorf("non multicast address in an existing membership during Resume: %s", err) + } + if err := e.stack.JoinGroup(proto, m.nicID, m.multicastAddr); err != nil { + return fmt.Errorf("e.stack.JoinGroup(%d, %d, %s): %s", proto, m.nicID, m.multicastAddr, err) } } diff --git a/pkg/tcpip/transport/internal/network/network_state_autogen.go b/pkg/tcpip/transport/internal/network/network_state_autogen.go index 11234a9..25035df 100644 --- a/pkg/tcpip/transport/internal/network/network_state_autogen.go +++ b/pkg/tcpip/transport/internal/network/network_state_autogen.go @@ -29,8 +29,10 @@ func (e *Endpoint) StateFields() []string { "multicastTTL", "multicastAddr", "multicastNICID", + "ipv6MulticastNICID", "ipv4TOS", "ipv6TClass", + "pmtud", "info", "state", } @@ -56,10 +58,12 @@ func (e *Endpoint) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(12, &e.multicastTTL) stateSinkObject.Save(13, &e.multicastAddr) stateSinkObject.Save(14, &e.multicastNICID) - stateSinkObject.Save(15, &e.ipv4TOS) - stateSinkObject.Save(16, &e.ipv6TClass) - stateSinkObject.Save(17, &e.info) - stateSinkObject.Save(18, &e.state) + stateSinkObject.Save(15, &e.ipv6MulticastNICID) + stateSinkObject.Save(16, &e.ipv4TOS) + stateSinkObject.Save(17, &e.ipv6TClass) + stateSinkObject.Save(18, &e.pmtud) + stateSinkObject.Save(19, &e.info) + stateSinkObject.Save(20, &e.state) } func (e *Endpoint) afterLoad(context.Context) {} @@ -81,10 +85,12 @@ func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source stateSourceObject.Load(12, &e.multicastTTL) stateSourceObject.Load(13, &e.multicastAddr) stateSourceObject.Load(14, &e.multicastNICID) - stateSourceObject.Load(15, &e.ipv4TOS) - stateSourceObject.Load(16, &e.ipv6TClass) - stateSourceObject.Load(17, &e.info) - stateSourceObject.Load(18, &e.state) + stateSourceObject.Load(15, &e.ipv6MulticastNICID) + stateSourceObject.Load(16, &e.ipv4TOS) + stateSourceObject.Load(17, &e.ipv6TClass) + stateSourceObject.Load(18, &e.pmtud) + stateSourceObject.Load(19, &e.info) + stateSourceObject.Load(20, &e.state) } func (m *multicastMembership) StateTypeName() string { diff --git a/pkg/tcpip/transport/packet/endpoint.go b/pkg/tcpip/transport/packet/endpoint.go index 1e311d3..6313e07 100644 --- a/pkg/tcpip/transport/packet/endpoint.go +++ b/pkg/tcpip/transport/packet/endpoint.go @@ -267,11 +267,12 @@ func (ep *endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tc } payloadSz := payload.Size() + mark := ep.ops.GetMark() if err := func() tcpip.Error { if ep.cooked { - return ep.stack.WritePacketToRemote(nicID, remote, proto, payload) + return ep.stack.WritePacketToRemoteWithMark(nicID, remote, proto, payload, mark) } - return ep.stack.WriteRawPacket(nicID, proto, payload) + return ep.stack.WriteRawPacketWithMark(nicID, proto, payload, mark) }(); err != nil { return 0, err } @@ -456,14 +457,14 @@ func (ep *endpoint) UpdateLastError(err tcpip.Error) { // GetSockOpt implements tcpip.Endpoint.GetSockOpt. func (ep *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error { - switch opt.(type) { + switch opt := opt.(type) { case *tcpip.TpacketStats: ep.packetMmapMu.RLock() defer ep.packetMmapMu.RUnlock() if ep.packetMMapEp == nil { return nil } - *(opt.(*tcpip.TpacketStats)) = ep.packetMMapEp.Stats() + *opt = ep.packetMMapEp.Stats() return nil default: return &tcpip.ErrUnknownProtocolOption{} diff --git a/pkg/tcpip/transport/packet/endpoint_mutex.go b/pkg/tcpip/transport/packet/endpoint_mutex.go index 685f878..40cfcde 100644 --- a/pkg/tcpip/transport/packet/endpoint_mutex.go +++ b/pkg/tcpip/transport/packet/endpoint_mutex.go @@ -92,5 +92,5 @@ func endpointinitLockNames() {} func init() { endpointinitLockNames() - endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames) + endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames) } diff --git a/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go b/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go index c34ffc3..970656f 100644 --- a/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go +++ b/pkg/tcpip/transport/packet/endpoint_rcv_mutex.go @@ -60,5 +60,5 @@ func rcvinitLockNames() {} func init() { rcvinitLockNames() - rcvprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvMutex{}), rcvlockNames) + rcvprefixIndex = locking.NewMutexClass(reflect.TypeFor[rcvMutex](), rcvlockNames) } diff --git a/pkg/tcpip/transport/packet/endpoint_state.go b/pkg/tcpip/transport/packet/endpoint_state.go index 40751c8..3d6d32f 100644 --- a/pkg/tcpip/transport/packet/endpoint_state.go +++ b/pkg/tcpip/transport/packet/endpoint_state.go @@ -16,9 +16,9 @@ package packet import ( "context" - "fmt" "time" + "github.com/sagernet/gvisor/pkg/log" "github.com/sagernet/gvisor/pkg/tcpip" "github.com/sagernet/gvisor/pkg/tcpip/stack" ) @@ -43,11 +43,6 @@ func (ep *endpoint) beforeSave() { // afterLoad is invoked by stateify. func (ep *endpoint) afterLoad(ctx context.Context) { - if !ep.stack.IsSaveRestoreEnabled() { - ep.mu.Lock() - ep.stack = stack.RestoreStackFromContext(ctx) - ep.mu.Unlock() - } ep.stack.RegisterRestoredEndpoint(ep) } @@ -58,7 +53,8 @@ func (ep *endpoint) Restore(_ *stack.Stack) { ep.ops.InitHandler(ep, ep.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) if err := ep.stack.RegisterPacketEndpoint(ep.boundNIC, ep.boundNetProto, ep); err != nil { - panic(fmt.Sprintf("RegisterPacketEndpoint(%d, %d, _): %s", ep.boundNIC, ep.boundNetProto, err)) + log.Warningf("RegisterPacketEndpoint(%d, %d, _) failed during restore with error: %s", ep.boundNIC, ep.boundNetProto, err) + return } ep.rcvMu.Lock() diff --git a/pkg/tcpip/transport/packet/last_error_mutex.go b/pkg/tcpip/transport/packet/last_error_mutex.go index 241cc2d..ac872ed 100644 --- a/pkg/tcpip/transport/packet/last_error_mutex.go +++ b/pkg/tcpip/transport/packet/last_error_mutex.go @@ -60,5 +60,5 @@ func lastErrorinitLockNames() {} func init() { lastErrorinitLockNames() - lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeOf(lastErrorMutex{}), lastErrorlockNames) + lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeFor[lastErrorMutex](), lastErrorlockNames) } diff --git a/pkg/tcpip/transport/packet/packet_mmap_mutex.go b/pkg/tcpip/transport/packet/packet_mmap_mutex.go index 853ed73..fa42ce6 100644 --- a/pkg/tcpip/transport/packet/packet_mmap_mutex.go +++ b/pkg/tcpip/transport/packet/packet_mmap_mutex.go @@ -92,5 +92,5 @@ func packetMmapinitLockNames() {} func init() { packetMmapinitLockNames() - packetMmapprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetMmapRWMutex{}), packetMmaplockNames) + packetMmapprefixIndex = locking.NewMutexClass(reflect.TypeFor[packetMmapRWMutex](), packetMmaplockNames) } diff --git a/pkg/tcpip/transport/packet/packet_state_autogen.go b/pkg/tcpip/transport/packet/packet_state_autogen.go index 0d62c0a..a37caed 100644 --- a/pkg/tcpip/transport/packet/packet_state_autogen.go +++ b/pkg/tcpip/transport/packet/packet_state_autogen.go @@ -27,8 +27,8 @@ func (p *packet) beforeSave() {} // +checklocksignore func (p *packet) StateSave(stateSinkObject state.Sink) { p.beforeSave() - var receivedAtValue int64 - receivedAtValue = p.saveReceivedAt() + receivedAtValue := p.saveReceivedAt() + _ = (int64)(receivedAtValue) stateSinkObject.SaveValue(2, receivedAtValue) stateSinkObject.Save(0, &p.packetEntry) stateSinkObject.Save(1, &p.data) diff --git a/pkg/tcpip/transport/raw/endpoint_state.go b/pkg/tcpip/transport/raw/endpoint_state.go index 6ba6003..f1d5fad 100644 --- a/pkg/tcpip/transport/raw/endpoint_state.go +++ b/pkg/tcpip/transport/raw/endpoint_state.go @@ -35,11 +35,7 @@ func (p *rawPacket) loadReceivedAt(_ context.Context, nsec int64) { // afterLoad is invoked by stateify. func (e *endpoint) afterLoad(ctx context.Context) { - if e.stack.IsSaveRestoreEnabled() { - e.stack.RegisterRestoredEndpoint(e) - } else { - stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) - } + e.stack.RegisterRestoredEndpoint(e) } // beforeSave is invoked by stateify. @@ -56,20 +52,7 @@ func (e *endpoint) Restore(s *stack.Stack) { return } e.setReceiveDisabled(false) - if e.stack.IsSaveRestoreEnabled() { - e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) - return - } - - e.stack = s e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) - - if e.associated { - netProto := e.net.NetProto() - if err := e.stack.RegisterRawTransportEndpoint(netProto, e.transProto, e); err != nil { - panic("RegisterRawTransportEndpoint failed during restore") - } - } } // Resume implements tcpip.ResumableEndpoint.Resume. diff --git a/pkg/tcpip/transport/raw/raw_state_autogen.go b/pkg/tcpip/transport/raw/raw_state_autogen.go index 7aee9f1..a5174e5 100644 --- a/pkg/tcpip/transport/raw/raw_state_autogen.go +++ b/pkg/tcpip/transport/raw/raw_state_autogen.go @@ -29,8 +29,8 @@ func (p *rawPacket) beforeSave() {} // +checklocksignore func (p *rawPacket) StateSave(stateSinkObject state.Sink) { p.beforeSave() - var receivedAtValue int64 - receivedAtValue = p.saveReceivedAt() + receivedAtValue := p.saveReceivedAt() + _ = (int64)(receivedAtValue) stateSinkObject.SaveValue(2, receivedAtValue) stateSinkObject.Save(0, &p.rawPacketEntry) stateSinkObject.Save(1, &p.data) diff --git a/pkg/tcpip/transport/tcp/accept.go b/pkg/tcpip/transport/tcp/accept.go index 376665d..baf2b93 100644 --- a/pkg/tcpip/transport/tcp/accept.go +++ b/pkg/tcpip/transport/tcp/accept.go @@ -255,7 +255,7 @@ func (l *listenContext) startHandshake(s *segment, opts header.TCPSynOptions, qu // Propagate any inheritable options from the listening endpoint // to the newly created endpoint. - l.listenEP.propagateInheritableOptionsLocked(ep) // +checklocksforce + l.listenEP.propagateInheritableOptionsLocked(ep) // +checklocksforce:ep.mu if !ep.reserveTupleLocked() { ep.mu.Unlock() @@ -359,6 +359,7 @@ func (e *Endpoint) propagateInheritableOptionsLocked(n *Endpoint) { n.boundBindToDevice = e.boundBindToDevice n.boundPortFlags = e.boundPortFlags n.userMSS = e.userMSS + n.ops.SetMark(e.ops.GetMark()) } // reserveTupleLocked reserves an accepted endpoint's tuple. @@ -529,6 +530,7 @@ func (e *Endpoint) handleListenSegment(ctx *listenContext, s *segment) tcpip.Err seq: cookie, ack: s.sequenceNumber + 1, rcvWnd: ctx.rcvWnd, + df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo || e.pmtud == tcpip.PMTUDiscoveryProbe, expOptVal: e.getExperimentOptionValue(route), } if err := e.sendSynTCP(route, fields, synOpts); err != nil { diff --git a/pkg/tcpip/transport/tcp/accept_mutex.go b/pkg/tcpip/transport/tcp/accept_mutex.go index a8147bb..d2cf259 100644 --- a/pkg/tcpip/transport/tcp/accept_mutex.go +++ b/pkg/tcpip/transport/tcp/accept_mutex.go @@ -60,5 +60,5 @@ func acceptinitLockNames() {} func init() { acceptinitLockNames() - acceptprefixIndex = locking.NewMutexClass(reflect.TypeOf(acceptMutex{}), acceptlockNames) + acceptprefixIndex = locking.NewMutexClass(reflect.TypeFor[acceptMutex](), acceptlockNames) } diff --git a/pkg/tcpip/transport/tcp/connect.go b/pkg/tcpip/transport/tcp/connect.go index b6bbd49..a735d9d 100644 --- a/pkg/tcpip/transport/tcp/connect.go +++ b/pkg/tcpip/transport/tcp/connect.go @@ -371,6 +371,7 @@ func (h *handshake) synSentState(s *segment) tcpip.Error { seq: h.iss, ack: h.ackNum, rcvWnd: h.rcvWnd, + df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe, expOptVal: h.ep.getExperimentOptionValue(h.ep.route), }, synOpts) return nil @@ -458,6 +459,7 @@ func (h *handshake) synRcvdState(s *segment) tcpip.Error { seq: h.iss, ack: h.ackNum, rcvWnd: h.rcvWnd, + df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe, expOptVal: h.ep.getExperimentOptionValue(h.ep.route), }, synOpts) return nil @@ -555,6 +557,7 @@ func (h *handshake) processSegments() tcpip.Error { // start sends the first SYN/SYN-ACK. It does not block, even if link address // resolution is required. +// +checklocks:h.ep.mu func (h *handshake) start() { h.startTime = h.ep.stack.Clock().NowMonotonic() h.ep.amss = calculateAdvertisedMSS(h.ep.userMSS, h.ep.route) @@ -595,6 +598,7 @@ func (h *handshake) start() { seq: h.iss, ack: h.ackNum, rcvWnd: h.rcvWnd, + df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe, expOptVal: h.ep.getExperimentOptionValue(h.ep.route), }, synOpts) } @@ -632,6 +636,7 @@ func (h *handshake) retransmitHandlerLocked() tcpip.Error { seq: h.iss, ack: h.ackNum, rcvWnd: h.rcvWnd, + df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe, expOptVal: e.getExperimentOptionValue(e.route), }, h.sendSYNOpts) // If we have ever retransmitted the SYN-ACK or @@ -642,11 +647,10 @@ func (h *handshake) retransmitHandlerLocked() tcpip.Error { return nil } -// transitionToStateEstablisedLocked transitions the endpoint of the handshake +// transitionToStateEstablishedLocked transitions the endpoint of the handshake // to an established state given the last segment received from peer. It also // initializes sender/receiver. // +checklocks:h.ep.mu -// +checklocksalias:h.ep.snd.ep.mu=h.ep.mu func (h *handshake) transitionToStateEstablishedLocked(s *segment) { // Stop the SYN retransmissions now that handshake is complete. if h.retransmitTimer != nil { @@ -656,16 +660,20 @@ func (h *handshake) transitionToStateEstablishedLocked(s *segment) { // Transfer handshake state to TCP connection. We disable // receive window scaling if the peer doesn't support it // (indicated by a negative send window scale). - h.ep.snd = newSender(h.ep, h.iss, h.ackNum-1, h.sndWnd, h.mss, h.sndWndScale) + initSender(h.ep, h.iss, h.ackNum-1, h.sndWnd, h.mss, h.sndWndScale) - now := h.ep.stack.Clock().NowMonotonic() + // Use the final handshake ACK's ingress time (s.rcvdTime) rather than the + // current clock to seed the initial RTT/RTO. If the ACK was delayed inside + // the stack before processing, the processing-time clock would inflate the + // initial RTO, which then persists for several RTTs. + rcvd := s.rcvdTime var rtt time.Duration if h.ep.SendTSOk && s.parsedOptions.TSEcr != 0 { - rtt = h.ep.elapsed(now, s.parsedOptions.TSEcr) + rtt = h.ep.elapsed(rcvd, s.parsedOptions.TSEcr) } if !h.sampleRTTWithTSOnly && rtt == 0 { - rtt = now.Sub(h.startTime) + rtt = rcvd.Sub(h.startTime) } if rtt > 0 { @@ -824,7 +832,10 @@ func (e *Endpoint) sendSynTCP(r *stack.Route, tf tcpFields, opts header.TCPSynOp if r.NetProto() == header.IPv6ProtocolNumber && tf.expOptVal != 0 { hdrSize += header.IPv6ExperimentHdrLength } - p := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize}) + p := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: hdrSize, + Mark: e.ops.GetMark(), + }) defer p.DecRef() if err := e.sendTCP(r, tf, p, stack.GSO{}); err != nil { e.stats.SendErrors.SynSendToNetworkFailed.Increment() @@ -900,7 +911,10 @@ func sendTCPBatch(r *stack.Route, tf tcpFields, pkt *stack.PacketBuffer, gso sta // Reserve extra bytes for the experiment option. hdrSize += header.IPv6ExperimentHdrLength } - splitPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize}) + splitPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + ReserveHeaderBytes: hdrSize, + Mark: pkt.Mark, + }) splitPkt.Data().ReadFromPacketData(pkt.Data(), packetSize) pkt = splitPkt } @@ -1007,9 +1021,10 @@ func (e *Endpoint) makeOptions(sackBlocks []header.SACKBlock) []byte { // sendEmptyRaw sends a TCP segment with no payload to the endpoint's peer. // // +checklocks:e.mu -// +checklocksalias:e.snd.ep.mu=e.mu func (e *Endpoint) sendEmptyRaw(flags header.TCPFlags, seq, ack seqnum.Value, rcvWnd seqnum.Size) tcpip.Error { - pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{}) + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Mark: e.ops.GetMark(), + }) defer pkt.DecRef() return e.sendRaw(pkt, flags, seq, ack, rcvWnd) } @@ -1018,7 +1033,6 @@ func (e *Endpoint) sendEmptyRaw(flags header.TCPFlags, seq, ack seqnum.Value, rc // ownership of pkt. pkt must not have any headers set. // // +checklocks:e.mu -// +checklocksalias:e.snd.ep.mu=e.mu func (e *Endpoint) sendRaw(pkt *stack.PacketBuffer, flags header.TCPFlags, seq, ack seqnum.Value, rcvWnd seqnum.Size) tcpip.Error { var sackBlocks []header.SACKBlock if e.EndpointState() == StateEstablished && e.rcv.pendingRcvdSegments.Len() > 0 && (flags&header.TCPFlagAck != 0) { @@ -1033,21 +1047,21 @@ func (e *Endpoint) sendRaw(pkt *stack.PacketBuffer, flags header.TCPFlags, seq, } pkt.ReserveHeaderBytes(hdrSize) return e.sendTCP(e.route, tcpFields{ - id: e.TransportEndpointInfo.ID, - ttl: calculateTTL(e.route, e.ipv4TTL, e.ipv6HopLimit), - tos: e.sendTOS, - flags: flags, - seq: seq, - ack: ack, - rcvWnd: rcvWnd, - opts: options, - df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo, + id: e.TransportEndpointInfo.ID, + ttl: calculateTTL(e.route, e.ipv4TTL, e.ipv6HopLimit), + tos: e.sendTOS, + flags: flags, + seq: seq, + ack: ack, + rcvWnd: rcvWnd, + opts: options, + // PROBE sets DF like DO; see network/endpoint.go for details. + df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo || e.pmtud == tcpip.PMTUDiscoveryProbe, expOptVal: expOptVal, }, pkt, e.gso) } // +checklocks:e.mu -// +checklocksalias:e.snd.ep.mu=e.mu func (e *Endpoint) sendData(next *segment) { // Initialize the next segment to write if it's currently nil. if e.snd.writeNext == nil { @@ -1065,7 +1079,6 @@ func (e *Endpoint) sendData(next *segment) { // error code and sends a RST if and only if the error is not ErrConnectionReset // indicating that the connection is being reset due to receiving a RST. // +checklocks:e.mu -// +checklocksalias:e.snd.ep.mu=e.mu func (e *Endpoint) resetConnectionLocked(err tcpip.Error) { // Only send a reset if the connection is being aborted for a reason // other than receiving a reset. @@ -1080,12 +1093,26 @@ func (e *Endpoint) resetConnectionLocked(err tcpip.Error) { // // See: https://www.snellman.net/blog/archive/2016-02-01-tcp-rst/ for more // information. - sndWndEnd := e.snd.SndUna.Add(e.snd.SndWnd) - resetSeqNum := sndWndEnd - if !sndWndEnd.LessThan(e.snd.SndNxt) || e.snd.SndNxt.Size(sndWndEnd) < (1< 0 { // If the segment doesn't include the seqnum we're expecting to @@ -265,7 +262,7 @@ func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum case StateEstablished: r.ep.setEndpointState(StateCloseWait) case StateFinWait1: - if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt { + if s.flags.Contains(header.TCPFlagAck) && r.ep.snd.finSent && s.ackNumber == r.ep.snd.SndNxt { // FIN-ACK, transition to TIME-WAIT. r.ep.setEndpointState(StateTimeWait) } else { @@ -299,8 +296,9 @@ func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum } // Handle ACK (not FIN-ACK, which we handled above) during one of the - // shutdown states. - if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt { + // shutdown states. These completions require that our FIN was sent; + // without finSent a data ACK would be mistaken for a FIN ACK. + if s.flags.Contains(header.TCPFlagAck) && r.ep.snd.finSent && s.ackNumber == r.ep.snd.SndNxt { switch r.ep.EndpointState() { case StateFinWait1: r.ep.setEndpointState(StateFinWait2) @@ -320,9 +318,12 @@ func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum return true } -// updateRTT updates the receiver RTT measurement based on the sequence number -// of the received segment. -func (r *receiver) updateRTT() { +// updateRTT estimates a receiver-side RTT for receive-buffer autotuning, based +// on the sequence number of the received segment. rcvdTime is the ingress +// timestamp of the segment that triggered this measurement; it is used instead +// of the current clock so that an internal processing delay does not inflate the +// estimate (which would size the receive buffer too large). +func (r *receiver) updateRTT(rcvdTime tcpip.MonotonicTime) { // From: https://public.lanl.gov/radiant/pubs/drs/sc2001-poster.pdf // // A system that is only transmitting acknowledgements can still @@ -332,7 +333,7 @@ func (r *receiver) updateRTT() { r.ep.rcvQueueMu.Lock() if r.ep.RcvAutoParams.RTTMeasureTime == (tcpip.MonotonicTime{}) { // New measurement. - r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic() + r.ep.RcvAutoParams.RTTMeasureTime = rcvdTime r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd) r.ep.rcvQueueMu.Unlock() return @@ -341,20 +342,19 @@ func (r *receiver) updateRTT() { r.ep.rcvQueueMu.Unlock() return } - rtt := r.ep.stack.Clock().NowMonotonic().Sub(r.ep.RcvAutoParams.RTTMeasureTime) + rtt := rcvdTime.Sub(r.ep.RcvAutoParams.RTTMeasureTime) // We only store the minimum observed RTT here as this is only used in // absence of a SRTT available from either timestamps or a sender // measurement of RTT. if r.ep.RcvAutoParams.RTT == 0 || rtt < r.ep.RcvAutoParams.RTT { r.ep.RcvAutoParams.RTT = rtt } - r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic() + r.ep.RcvAutoParams.RTTMeasureTime = rcvdTime r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd) r.ep.rcvQueueMu.Unlock() } // +checklocks:r.ep.mu -// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu func (r *receiver) handleRcvdSegmentClosing(s *segment, state EndpointState, closed bool) (drop bool, err tcpip.Error) { r.ep.rcvQueueMu.Lock() rcvClosed := r.ep.RcvClosed || r.closed @@ -452,7 +452,6 @@ func (r *receiver) handleRcvdSegmentClosing(s *segment, state EndpointState, clo // handleRcvdSegment handles TCP segments directed at the connection managed by // r as they arrive. It is called by the protocol main loop. // +checklocks:r.ep.mu -// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) { state := r.ep.EndpointState() closed := r.ep.closed @@ -475,8 +474,10 @@ func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) { } } - // Store the time of the last ack. - r.lastRcvdAckTime = r.ep.stack.Clock().NowMonotonic() + // Store the time of the last ack. Use the segment's ingress time rather than + // the current clock so a segment delayed inside the stack before processing + // records when it actually arrived (consumed by the user-timeout check). + r.lastRcvdAckTime = s.rcvdTime // Defer segment processing if it can't be consumed now. if !r.consumeSegment(s, segSeq, segLen) { @@ -519,7 +520,7 @@ func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) { // Since we consumed a segment update the receiver's RTT estimate // if required. if segLen > 0 { - r.updateRTT() + r.updateRTT(s.rcvdTime) } // By consuming the current segment, we may have filled a gap in the @@ -548,7 +549,6 @@ func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) { // handleTimeWaitSegment handles inbound segments received when the endpoint // has entered the TIME_WAIT state. // +checklocks:r.ep.mu -// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu func (r *receiver) handleTimeWaitSegment(s *segment) (resetTimeWait bool, newSyn bool) { segSeq := s.sequenceNumber segLen := seqnum.Size(s.payloadSize()) diff --git a/pkg/tcpip/transport/tcp/rcv_queue_mutex.go b/pkg/tcpip/transport/tcp/rcv_queue_mutex.go index e683804..1c8554d 100644 --- a/pkg/tcpip/transport/tcp/rcv_queue_mutex.go +++ b/pkg/tcpip/transport/tcp/rcv_queue_mutex.go @@ -60,5 +60,5 @@ func rcvQueueinitLockNames() {} func init() { rcvQueueinitLockNames() - rcvQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvQueueMutex{}), rcvQueuelockNames) + rcvQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[rcvQueueMutex](), rcvQueuelockNames) } diff --git a/pkg/tcpip/transport/tcp/reno.go b/pkg/tcpip/transport/tcp/reno.go index 4345376..1e60d34 100644 --- a/pkg/tcpip/transport/tcp/reno.go +++ b/pkg/tcpip/transport/tcp/reno.go @@ -16,6 +16,8 @@ package tcp import ( "time" + + "github.com/sagernet/gvisor/pkg/tcpip" ) // renoState stores the variables related to TCP New Reno congestion @@ -80,7 +82,7 @@ func (r *renoState) reduceSlowStartThreshold() { // Update implements congestionControl.Update. // // +checklocks:r.s.ep.mu -func (r *renoState) Update(packetsAcked int, _ time.Duration) { +func (r *renoState) Update(packetsAcked int, _ time.Duration, _ tcpip.MonotonicTime) { if r.s.SndCwnd < r.s.Ssthresh { packetsAcked = r.updateSlowStart(packetsAcked) if packetsAcked == 0 { diff --git a/pkg/tcpip/transport/tcp/rtt_mutex.go b/pkg/tcpip/transport/tcp/rtt_mutex.go index 4ce9ee8..ce3da98 100644 --- a/pkg/tcpip/transport/tcp/rtt_mutex.go +++ b/pkg/tcpip/transport/tcp/rtt_mutex.go @@ -60,5 +60,5 @@ func rttinitLockNames() {} func init() { rttinitLockNames() - rttprefixIndex = locking.NewMutexClass(reflect.TypeOf(rttMutex{}), rttlockNames) + rttprefixIndex = locking.NewMutexClass(reflect.TypeFor[rttMutex](), rttlockNames) } diff --git a/pkg/tcpip/transport/tcp/sack_scoreboard.go b/pkg/tcpip/transport/tcp/sack_scoreboard.go index fa306a7..6002ee6 100644 --- a/pkg/tcpip/transport/tcp/sack_scoreboard.go +++ b/pkg/tcpip/transport/tcp/sack_scoreboard.go @@ -34,6 +34,12 @@ const ( defaultBtreeDegree = 2 ) +// sackBlockLess is the comparison function for BTreeG, replacing the +// btree.Item interface method. +func sackBlockLess(a, b header.SACKBlock) bool { + return a.Start.LessThan(b.Start) +} + // SACKScoreboard stores a set of disjoint SACK ranges. // // +stateify savable @@ -47,22 +53,22 @@ type SACKScoreboard struct { // the TCP/IP headers and options. smss uint16 maxSACKED seqnum.Value - sacked seqnum.Size `state:"nosave"` - ranges *btree.BTree `state:"nosave"` + sacked seqnum.Size `state:"nosave"` + ranges *btree.BTreeG[header.SACKBlock] `state:"nosave"` } // NewSACKScoreboard returns a new SACK Scoreboard. func NewSACKScoreboard(smss uint16, iss seqnum.Value) *SACKScoreboard { return &SACKScoreboard{ smss: smss, - ranges: btree.New(defaultBtreeDegree), + ranges: btree.NewG[header.SACKBlock](defaultBtreeDegree, sackBlockLess), maxSACKED: iss, } } // Reset erases all known range information from the SACK scoreboard. func (s *SACKScoreboard) Reset() { - s.ranges = btree.New(defaultBtreeDegree) + s.ranges = btree.NewG[header.SACKBlock](defaultBtreeDegree, sackBlockLess) s.sacked = 0 } @@ -73,15 +79,14 @@ func (s *SACKScoreboard) Insert(r header.SACKBlock) { } // Check if we can merge the new range with a range before or after it. - var toDelete []btree.Item + var toDelete []header.SACKBlock if s.maxSACKED.LessThan(r.End - 1) { s.maxSACKED = r.End - 1 } - s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool { - if i == r { + s.ranges.AscendGreaterOrEqual(r, func(sacked header.SACKBlock) bool { + if sacked == r { return true } - sacked := i.(header.SACKBlock) // There is a hole between these two SACK blocks, so we can't // merge anymore. if r.End.LessThan(sacked.Start) { @@ -96,21 +101,20 @@ func (s *SACKScoreboard) Insert(r header.SACKBlock) { if sacked.End.LessThan(r.End) { // sacked is contained in the newly inserted range. // Delete this block. - toDelete = append(toDelete, i) + toDelete = append(toDelete, sacked) return true } // sacked covers a range past end of the newly inserted // block. r.End = sacked.End - toDelete = append(toDelete, i) + toDelete = append(toDelete, sacked) return true }) - s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { - if i == r { + s.ranges.DescendLessOrEqual(r, func(sacked header.SACKBlock) bool { + if sacked == r { return true } - sacked := i.(header.SACKBlock) // sA------sE // rA----rE if sacked.End.LessThan(r.Start) { @@ -126,18 +130,17 @@ func (s *SACKScoreboard) Insert(r header.SACKBlock) { if r.End.LessThan(sacked.End) { r.End = sacked.End } - toDelete = append(toDelete, i) + toDelete = append(toDelete, sacked) return true }) - for _, i := range toDelete { - if sb := s.ranges.Delete(i); sb != nil { - sb := i.(header.SACKBlock) + for _, sb := range toDelete { + if _, ok := s.ranges.Delete(sb); ok { s.sacked -= sb.Start.Size(sb.End) } } - replaced := s.ranges.ReplaceOrInsert(r) - if replaced == nil { + _, replaced := s.ranges.ReplaceOrInsert(r) + if !replaced { s.sacked += r.Start.Size(r.End) } } @@ -150,8 +153,7 @@ func (s *SACKScoreboard) IsSACKED(r header.SACKBlock) bool { } found := false - s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { - sacked := i.(header.SACKBlock) + s.ranges.DescendLessOrEqual(r, func(sacked header.SACKBlock) bool { if sacked.End.LessThan(r.Start) { return false } @@ -168,8 +170,8 @@ func (s *SACKScoreboard) IsSACKED(r header.SACKBlock) bool { func (s *SACKScoreboard) String() string { var str strings.Builder str.WriteString("SACKScoreboard: {") - s.ranges.Ascend(func(i btree.Item) bool { - str.WriteString(fmt.Sprintf("%v,", i)) + s.ranges.Ascend(func(sb header.SACKBlock) bool { + fmt.Fprintf(&str, "%v,", sb) return true }) str.WriteString("}\n") @@ -181,15 +183,14 @@ func (s *SACKScoreboard) Delete(seq seqnum.Value) { if s.Empty() { return } - toDelete := []btree.Item{} - toInsert := []btree.Item{} + var toDelete []header.SACKBlock + var toInsert []header.SACKBlock r := header.SACKBlock{seq, seq.Add(1)} - s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { - if i == r { + s.ranges.DescendLessOrEqual(r, func(sb header.SACKBlock) bool { + if sb == r { return true } - sb := i.(header.SACKBlock) - toDelete = append(toDelete, i) + toDelete = append(toDelete, sb) if sb.End.LessThanEq(seq) { s.sacked -= sb.Start.Size(sb.End) } else { @@ -209,8 +210,8 @@ func (s *SACKScoreboard) Delete(seq seqnum.Value) { // Copy provides a copy of the SACK scoreboard. func (s *SACKScoreboard) Copy() (sackBlocks []header.SACKBlock, maxSACKED seqnum.Value) { - s.ranges.Ascend(func(i btree.Item) bool { - sackBlocks = append(sackBlocks, i.(header.SACKBlock)) + s.ranges.Ascend(func(sb header.SACKBlock) bool { + sackBlocks = append(sackBlocks, sb) return true }) return sackBlocks, s.maxSACKED @@ -232,8 +233,7 @@ func (s *SACKScoreboard) IsRangeLost(r header.SACKBlock) bool { // We need to check if the immediate lower (if any) sacked // range contains or partially overlaps with r. searchMore := true - s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool { - sacked := i.(header.SACKBlock) + s.ranges.DescendLessOrEqual(r, func(sacked header.SACKBlock) bool { if sacked.Contains(r) { searchMore = false return false @@ -256,8 +256,7 @@ func (s *SACKScoreboard) IsRangeLost(r header.SACKBlock) bool { return isLost } - s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool { - sacked := i.(header.SACKBlock) + s.ranges.AscendGreaterOrEqual(r, func(sacked header.SACKBlock) bool { if sacked.Contains(r) { return false } diff --git a/pkg/tcpip/transport/tcp/segment.go b/pkg/tcpip/transport/tcp/segment.go index 863b1e3..ad31f2b 100644 --- a/pkg/tcpip/transport/tcp/segment.go +++ b/pkg/tcpip/transport/tcp/segment.go @@ -139,11 +139,14 @@ func newIncomingSegment(id stack.TransportEndpointID, clock tcpip.Clock, pkt *st return s, nil } -func newOutgoingSegment(id stack.TransportEndpointID, clock tcpip.Clock, buf buffer.Buffer) *segment { +func newOutgoingSegment(id stack.TransportEndpointID, clock tcpip.Clock, buf buffer.Buffer, mark uint32) *segment { s := newSegment() s.id = id s.rcvdTime = clock.NowMonotonic() - s.pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf}) + s.pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buf, + Mark: mark, + }) s.dataMemSize = s.pkt.MemSize() return s } diff --git a/pkg/tcpip/transport/tcp/segment_queue_mutex.go b/pkg/tcpip/transport/tcp/segment_queue_mutex.go index e1e5dd4..c456b35 100644 --- a/pkg/tcpip/transport/tcp/segment_queue_mutex.go +++ b/pkg/tcpip/transport/tcp/segment_queue_mutex.go @@ -60,5 +60,5 @@ func segmentQueueinitLockNames() {} func init() { segmentQueueinitLockNames() - segmentQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(segmentQueueMutex{}), segmentQueuelockNames) + segmentQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[segmentQueueMutex](), segmentQueuelockNames) } diff --git a/pkg/tcpip/transport/tcp/snd.go b/pkg/tcpip/transport/tcp/snd.go index ee29b87..fbf7693 100644 --- a/pkg/tcpip/transport/tcp/snd.go +++ b/pkg/tcpip/transport/tcp/snd.go @@ -74,8 +74,11 @@ type congestionControl interface { // Update is invoked when processing inbound acks. It's passed the // number of packet's that were acked by the most recent cumulative // acknowledgement. rtt is the round-trip time, or is set to unknownRTT - // (above) to indicate the time is unknown. - Update(packetsAcked int, rtt time.Duration) + // (above) to indicate the time is unknown. ackTime is the time the + // processed ACK arrived at the stack (its ingress timestamp), used for + // arrival-anchored timing such as CUBIC HyStart's ACK-train detection so + // that an ACK delayed inside the stack does not distort it. + Update(packetsAcked int, rtt time.Duration, ackTime tcpip.MonotonicTime) // PostRecovery is invoked when the sender is exiting a fast retransmit/ // recovery phase. This provides congestion control algorithms a way @@ -96,6 +99,7 @@ type lossRecovery interface { // sender holds the state necessary to send TCP segments. // +// +checklocksalias:rc.snd.ep.mu=ep.mu // +stateify savable type sender struct { // +checklocks:ep.mu @@ -103,6 +107,11 @@ type sender struct { ep *Endpoint + // finSent is set when the FIN segment is actually transmitted. + // The endpoint may be in FIN_WAIT1/LAST_ACK while the FIN is still + // queued behind blocked data; finSent guards closing ACK completions. + finSent bool + // lr is the loss recovery algorithm used by the sender. lr lossRecovery @@ -242,13 +251,13 @@ type rtt struct { } // +checklocks:ep.mu -func newSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int) *sender { +func initSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int) { // The sender MUST reduce the TCP data length to account for any IP or // TCP options that it is including in the packets that it sends. // See: https://tools.ietf.org/html/rfc6691#section-2 maxPayloadSize := int(mss) - ep.maxOptionSize() - s := &sender{ + ep.snd = &sender{ ep: ep, TCPSenderState: TCPSenderState{ SndWnd: sndWnd, @@ -271,59 +280,50 @@ func newSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint set: make(map[*segment]struct{}), }, } - return newSenderHelper(ep, iss, irs, sndWnd, mss, sndWndScale, maxPayloadSize, s) -} -// newSenderHelper exists to sate checklocks. -// -// +checklocks:ep.mu -// +checklocksalias:s.ep.mu=ep.mu -func newSenderHelper(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int, maxPayloadSize int, s *sender) *sender { - if s.gso { - s.ep.gso.MSS = uint16(maxPayloadSize) + if ep.snd.gso { + ep.snd.ep.gso.MSS = uint16(maxPayloadSize) } - s.cc = s.initCongestionControl(ep.cc) - s.lr = s.initLossRecovery() - s.rc.init(s, iss) + ep.snd.cc = ep.snd.initCongestionControl(ep.cc) + ep.snd.lr = ep.snd.initLossRecovery() + ep.snd.rc.init(ep.snd, iss) // A negative sndWndScale means that no scaling is in use, otherwise we // store the scaling value. if sndWndScale > 0 { - s.SndWndScale = uint8(sndWndScale) + ep.snd.SndWndScale = uint8(sndWndScale) } - s.resendTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.retransmitTimerExpired)) - s.reorderTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.rc.reorderTimerExpired)) - s.probeTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.probeTimerExpired)) - s.corkTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.corkTimerExpired)) + ep.snd.resendTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.retransmitTimerExpired)) + ep.snd.reorderTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.rc.reorderTimerExpired)) + ep.snd.probeTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.probeTimerExpired)) + ep.snd.corkTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.corkTimerExpired)) - s.updateMaxPayloadSize(int(ep.route.MTU()), 0) + ep.snd.updateMaxPayloadSize(int(ep.snd.ep.route.MTU()), 0) // Initialize SACK Scoreboard after updating max payload size as we use // the maxPayloadSize as the smss when determining if a segment is lost // etc. - s.ep.scoreboard = NewSACKScoreboard(uint16(s.MaxPayloadSize), iss) + ep.snd.ep.scoreboard = NewSACKScoreboard(uint16(ep.snd.MaxPayloadSize), iss) // Get Stack wide config. var minRTO tcpip.TCPMinRTOOption - if err := ep.stack.TransportProtocolOption(ProtocolNumber, &minRTO); err != nil { + if err := ep.snd.ep.stack.TransportProtocolOption(ProtocolNumber, &minRTO); err != nil { panic(fmt.Sprintf("unable to get minRTO from stack: %s", err)) } - s.minRTO = time.Duration(minRTO) + ep.snd.minRTO = time.Duration(minRTO) var maxRTO tcpip.TCPMaxRTOOption - if err := ep.stack.TransportProtocolOption(ProtocolNumber, &maxRTO); err != nil { + if err := ep.snd.ep.stack.TransportProtocolOption(ProtocolNumber, &maxRTO); err != nil { panic(fmt.Sprintf("unable to get maxRTO from stack: %s", err)) } - s.maxRTO = time.Duration(maxRTO) + ep.snd.maxRTO = time.Duration(maxRTO) var maxRetries tcpip.TCPMaxRetriesOption - if err := ep.stack.TransportProtocolOption(ProtocolNumber, &maxRetries); err != nil { + if err := ep.snd.ep.stack.TransportProtocolOption(ProtocolNumber, &maxRetries); err != nil { panic(fmt.Sprintf("unable to get maxRetries from stack: %s", err)) } - s.maxRetries = uint32(maxRetries) - - return s + ep.snd.maxRetries = uint32(maxRetries) } // initCongestionControl initializes the specified congestion control module and @@ -434,6 +434,13 @@ func (s *sender) sendAck() { // // +checklocks:s.ep.mu func (s *sender) updateRTO(rtt time.Duration) { + // A negative RTT sample is nonsensical and would skew SRTT/RTTVar (and thus + // RTO). RTT samples are now anchored to a segment's ingress time, which is + // monotonic and never after the corresponding send time, so this should not + // occur; guard defensively rather than corrupt the estimator. + if rtt < 0 { + return + } s.rtt.Lock() if !s.rtt.TCPRTTState.SRTTInited { s.rtt.TCPRTTState.RTTVar = rtt / 2 @@ -902,15 +909,9 @@ func (s *sender) maybeSendSegment(seg *segment, limit int, end seqnum.Value) (se } seg.flags = header.TCPFlagAck | header.TCPFlagFin segEnd = seg.sequenceNumber.Add(1) - // Update the state to reflect that we have now - // queued a FIN. - s.ep.updateConnDirectionState(connDirectionStateSndClosed) - switch s.ep.EndpointState() { - case StateCloseWait: - s.ep.setEndpointState(StateLastAck) - default: - s.ep.setEndpointState(StateFinWait1) - } + // FIN is now being transmitted; mark it so the receiver can tell + // a data-only ACK apart from one that acknowledges our FIN. + s.finSent = true } else { // We're sending a non-FIN segment. if seg.flags&header.TCPFlagFin != 0 { @@ -1013,6 +1014,7 @@ func (s *sender) sendZeroWindowProbe() { // we re-send an ACKed byte to goad the receiver into responding. pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ Payload: buffer.MakeWithData(zeroProbeJunk), + Mark: s.ep.ops.GetMark(), }) defer pkt.DecRef() s.sendSegmentFromPacketBuffer(pkt, header.TCPFlagAck, s.SndUna-1) @@ -1518,13 +1520,16 @@ func (s *sender) inRecovery() bool { // handleRcvdSegment is called when a segment is received; it is responsible for // updating the send-related state. // +checklocks:s.ep.mu -// +checklocksalias:s.rc.snd.ep.mu=s.ep.mu func (s *sender) handleRcvdSegment(rcvdSeg *segment) { bestRTT := unknownRTT - // Check if we can extract an RTT measurement from this ack. + // Check if we can extract an RTT measurement from this ack. Measure against + // the ACK's ingress time (rcvdSeg.rcvdTime), not the current clock: if the + // ACK was delayed inside the stack before being processed (e.g. queued while + // the application held the endpoint lock during a Write), using the + // processing time would inflate the RTT sample and thus SRTT/RTO. if !rcvdSeg.parsedOptions.TS && s.RTTMeasureSeqNum.LessThan(rcvdSeg.ackNumber) { - bestRTT = s.ep.stack.Clock().NowMonotonic().Sub(s.RTTMeasureTime) + bestRTT = rcvdSeg.rcvdTime.Sub(s.RTTMeasureTime) s.updateRTO(bestRTT) s.RTTMeasureSeqNum = s.SndNxt } @@ -1627,7 +1632,10 @@ func (s *sender) handleRcvdSegment(rcvdSeg *segment) { // some new data, i.e., only if it advances the left edge of // the send window. if s.ep.SendTSOk && rcvdSeg.parsedOptions.TSEcr != 0 { - tsRTT := s.ep.elapsed(s.ep.stack.Clock().NowMonotonic(), rcvdSeg.parsedOptions.TSEcr) + // Compute elapsed time from the ACK's ingress time, not the current + // clock, so an ACK delayed inside the stack before processing does + // not inflate the timestamp-based RTT sample (and thus SRTT/RTO). + tsRTT := s.ep.elapsed(rcvdSeg.rcvdTime, rcvdSeg.parsedOptions.TSEcr) s.updateRTO(tsRTT) // Following Linux, prefer RTT computed from ACKs to TSEcr because, // "broken middle-boxes or peers may corrupt TS-ECR fields" @@ -1707,7 +1715,7 @@ func (s *sender) handleRcvdSegment(rcvdSeg *segment) { // If we are not in fast recovery then update the congestion // window based on the number of acknowledged packets. if !s.FastRecovery.Active { - s.cc.Update(originalOutstanding-s.Outstanding, bestRTT) + s.cc.Update(originalOutstanding-s.Outstanding, bestRTT, rcvdSeg.rcvdTime) if s.FastRecovery.Last.LessThan(s.SndUna) { s.state = tcpip.Open // Update RACK when we are exiting fast or RTO @@ -1822,8 +1830,6 @@ func (s *sender) sendSegment(seg *segment) tcpip.Error { // sendSegmentFromPacketBuffer sends a new segment containing the given payload, // flags and sequence number. // +checklocks:s.ep.mu -// +checklocksalias:s.ep.rcv.ep.mu=s.ep.mu -// +checklocksalias:s.ep.rcv.ep.snd.ep.mu=s.ep.mu func (s *sender) sendSegmentFromPacketBuffer(pkt *stack.PacketBuffer, flags header.TCPFlags, seq seqnum.Value) tcpip.Error { s.LastSendTime = s.ep.stack.Clock().NowMonotonic() if seq == s.RTTMeasureSeqNum { @@ -1845,9 +1851,6 @@ func (s *sender) sendSegmentFromPacketBuffer(pkt *stack.PacketBuffer, flags head // sendEmptySegment sends a new empty segment, flags and sequence number. // +checklocks:s.ep.mu -// +checklocksalias:s.ep.rcv.ep.snd.ep.mu=s.ep.mu -// +checklocksalias:s.ep.rcv.ep.mu=s.ep.mu -// +checklocksalias:s.ep.snd.ep.mu=s.ep.mu func (s *sender) sendEmptySegment(flags header.TCPFlags, seq seqnum.Value) tcpip.Error { s.LastSendTime = s.ep.stack.Clock().NowMonotonic() if seq == s.RTTMeasureSeqNum { diff --git a/pkg/tcpip/transport/tcp/snd_queue_mutex.go b/pkg/tcpip/transport/tcp/snd_queue_mutex.go index 7984134..5616928 100644 --- a/pkg/tcpip/transport/tcp/snd_queue_mutex.go +++ b/pkg/tcpip/transport/tcp/snd_queue_mutex.go @@ -60,5 +60,5 @@ func sndQueueinitLockNames() {} func init() { sndQueueinitLockNames() - sndQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(sndQueueMutex{}), sndQueuelockNames) + sndQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[sndQueueMutex](), sndQueuelockNames) } diff --git a/pkg/tcpip/transport/tcp/tcp_state_autogen.go b/pkg/tcpip/transport/tcp/tcp_state_autogen.go index 2dda131..334c298 100644 --- a/pkg/tcpip/transport/tcp/tcp_state_autogen.go +++ b/pkg/tcpip/transport/tcp/tcp_state_autogen.go @@ -25,8 +25,8 @@ func (a *acceptQueue) beforeSave() {} // +checklocksignore func (a *acceptQueue) StateSave(stateSinkObject state.Sink) { a.beforeSave() - var endpointsValue []*Endpoint - endpointsValue = a.saveEndpoints() + endpointsValue := a.saveEndpoints() + _ = ([]*Endpoint)(endpointsValue) stateSinkObject.SaveValue(0, endpointsValue) stateSinkObject.Save(1, &a.pendingEndpoints) stateSinkObject.Save(2, &a.capacity) @@ -150,9 +150,7 @@ func (q *epQueue) StateTypeName() string { } func (q *epQueue) StateFields() []string { - return []string{ - "list", - } + return []string{} } func (q *epQueue) beforeSave() {} @@ -160,14 +158,12 @@ func (q *epQueue) beforeSave() {} // +checklocksignore func (q *epQueue) StateSave(stateSinkObject state.Sink) { q.beforeSave() - stateSinkObject.Save(0, &q.list) } func (q *epQueue) afterLoad(context.Context) {} // +checklocksignore func (q *epQueue) StateLoad(ctx context.Context, stateSourceObject state.Source) { - stateSourceObject.Load(0, &q.list) } func (p *processor) StateTypeName() string { @@ -496,14 +492,15 @@ func (e *Endpoint) StateFields() []string { "lastOutOfWindowAckTime", "pmtud", "alsoBindToV4", + "terminateAtRestore", } } // +checklocksignore func (e *Endpoint) StateSave(stateSinkObject state.Sink) { e.beforeSave() - var stateValue EndpointState - stateValue = e.saveState() + stateValue := e.saveState() + _ = (EndpointState)(stateValue) stateSinkObject.SaveValue(12, stateValue) stateSinkObject.Save(0, &e.TCPEndpointStateInner) stateSinkObject.Save(1, &e.TransportEndpointInfo) @@ -561,6 +558,7 @@ func (e *Endpoint) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(54, &e.lastOutOfWindowAckTime) stateSinkObject.Save(55, &e.pmtud) stateSinkObject.Save(56, &e.alsoBindToV4) + stateSinkObject.Save(57, &e.terminateAtRestore) } // +checklocksignore @@ -621,6 +619,7 @@ func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source stateSourceObject.Load(54, &e.lastOutOfWindowAckTime) stateSourceObject.Load(55, &e.pmtud) stateSourceObject.Load(56, &e.alsoBindToV4) + stateSourceObject.Load(57, &e.terminateAtRestore) stateSourceObject.LoadValue(12, new(EndpointState), func(y any) { e.loadState(ctx, y.(EndpointState)) }) stateSourceObject.AfterLoad(func() { e.afterLoad(ctx) }) } @@ -683,8 +682,6 @@ func (p *protocol) StateFields() []string { "maxRetries", "synRetries", "dispatcher", - "seqnumSecret", - "tsOffsetSecret", } } @@ -711,12 +708,8 @@ func (p *protocol) StateSave(stateSinkObject state.Sink) { stateSinkObject.Save(15, &p.maxRetries) stateSinkObject.Save(16, &p.synRetries) stateSinkObject.Save(17, &p.dispatcher) - stateSinkObject.Save(18, &p.seqnumSecret) - stateSinkObject.Save(19, &p.tsOffsetSecret) } -func (p *protocol) afterLoad(context.Context) {} - // +checklocksignore func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &p.stack) @@ -737,8 +730,7 @@ func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source stateSourceObject.Load(15, &p.maxRetries) stateSourceObject.Load(16, &p.synRetries) stateSourceObject.Load(17, &p.dispatcher) - stateSourceObject.Load(18, &p.seqnumSecret) - stateSourceObject.Load(19, &p.tsOffsetSecret) + stateSourceObject.AfterLoad(func() { p.afterLoad(ctx) }) } func (rc *rackControl) StateTypeName() string { @@ -964,8 +956,8 @@ func (s *segment) beforeSave() {} // +checklocksignore func (s *segment) StateSave(stateSinkObject state.Sink) { s.beforeSave() - var optionsValue []byte - optionsValue = s.saveOptions() + optionsValue := s.saveOptions() + _ = ([]byte)(optionsValue) stateSinkObject.SaveValue(12, optionsValue) stateSinkObject.Save(0, &s.segmentEntry) stateSinkObject.Save(1, &s.segmentRefs) @@ -1053,6 +1045,7 @@ func (s *sender) StateFields() []string { return []string{ "TCPSenderState", "ep", + "finSent", "lr", "firstRetransmittedSegXmitTime", "writeNext", @@ -1078,21 +1071,22 @@ func (s *sender) StateSave(stateSinkObject state.Sink) { s.beforeSave() stateSinkObject.Save(0, &s.TCPSenderState) stateSinkObject.Save(1, &s.ep) - stateSinkObject.Save(2, &s.lr) - stateSinkObject.Save(3, &s.firstRetransmittedSegXmitTime) - stateSinkObject.Save(4, &s.writeNext) - stateSinkObject.Save(5, &s.writeList) - stateSinkObject.Save(6, &s.rtt) - stateSinkObject.Save(7, &s.minRTO) - stateSinkObject.Save(8, &s.maxRTO) - stateSinkObject.Save(9, &s.maxRetries) - stateSinkObject.Save(10, &s.gso) - stateSinkObject.Save(11, &s.state) - stateSinkObject.Save(12, &s.cc) - stateSinkObject.Save(13, &s.rc) - stateSinkObject.Save(14, &s.spuriousRecovery) - stateSinkObject.Save(15, &s.retransmitTS) - stateSinkObject.Save(16, &s.startCork) + stateSinkObject.Save(2, &s.finSent) + stateSinkObject.Save(3, &s.lr) + stateSinkObject.Save(4, &s.firstRetransmittedSegXmitTime) + stateSinkObject.Save(5, &s.writeNext) + stateSinkObject.Save(6, &s.writeList) + stateSinkObject.Save(7, &s.rtt) + stateSinkObject.Save(8, &s.minRTO) + stateSinkObject.Save(9, &s.maxRTO) + stateSinkObject.Save(10, &s.maxRetries) + stateSinkObject.Save(11, &s.gso) + stateSinkObject.Save(12, &s.state) + stateSinkObject.Save(13, &s.cc) + stateSinkObject.Save(14, &s.rc) + stateSinkObject.Save(15, &s.spuriousRecovery) + stateSinkObject.Save(16, &s.retransmitTS) + stateSinkObject.Save(17, &s.startCork) } func (s *sender) afterLoad(context.Context) {} @@ -1101,21 +1095,22 @@ func (s *sender) afterLoad(context.Context) {} func (s *sender) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(0, &s.TCPSenderState) stateSourceObject.Load(1, &s.ep) - stateSourceObject.Load(2, &s.lr) - stateSourceObject.Load(3, &s.firstRetransmittedSegXmitTime) - stateSourceObject.Load(4, &s.writeNext) - stateSourceObject.Load(5, &s.writeList) - stateSourceObject.Load(6, &s.rtt) - stateSourceObject.Load(7, &s.minRTO) - stateSourceObject.Load(8, &s.maxRTO) - stateSourceObject.Load(9, &s.maxRetries) - stateSourceObject.Load(10, &s.gso) - stateSourceObject.Load(11, &s.state) - stateSourceObject.Load(12, &s.cc) - stateSourceObject.Load(13, &s.rc) - stateSourceObject.Load(14, &s.spuriousRecovery) - stateSourceObject.Load(15, &s.retransmitTS) - stateSourceObject.Load(16, &s.startCork) + stateSourceObject.Load(2, &s.finSent) + stateSourceObject.Load(3, &s.lr) + stateSourceObject.Load(4, &s.firstRetransmittedSegXmitTime) + stateSourceObject.Load(5, &s.writeNext) + stateSourceObject.Load(6, &s.writeList) + stateSourceObject.Load(7, &s.rtt) + stateSourceObject.Load(8, &s.minRTO) + stateSourceObject.Load(9, &s.maxRTO) + stateSourceObject.Load(10, &s.maxRetries) + stateSourceObject.Load(11, &s.gso) + stateSourceObject.Load(12, &s.state) + stateSourceObject.Load(13, &s.cc) + stateSourceObject.Load(14, &s.rc) + stateSourceObject.Load(15, &s.spuriousRecovery) + stateSourceObject.Load(16, &s.retransmitTS) + stateSourceObject.Load(17, &s.startCork) } func (wl *protectedWriteList) StateTypeName() string { diff --git a/pkg/tcpip/transport/udp/endpoint.go b/pkg/tcpip/transport/udp/endpoint.go index 6f6a64c..b53bed8 100644 --- a/pkg/tcpip/transport/udp/endpoint.go +++ b/pkg/tcpip/transport/udp/endpoint.go @@ -294,6 +294,10 @@ func (e *endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult if opts.NeedRemoteAddr { res.RemoteAddr = p.senderAddress } + if opts.NeedReceivedExperimentOption { + expOptVal, _ := p.pkt.ExperimentOptionValue() + res.ReceivedExperimentOption = expOptVal + } n, err := p.pkt.Data().ReadTo(dst, opts.Peek) if n == 0 && err != nil { @@ -463,9 +467,9 @@ func (e *endpoint) write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcp dataSz := p.Len() pktInfo := udpInfo.ctx.PacketInfo() - pkt := udpInfo.ctx.TryNewPacketBufferFromPayloader(header.UDPMinimumSize+int(pktInfo.MaxHeaderLength), p) - if pkt == nil { - return 0, &tcpip.ErrWouldBlock{} + pkt, err := udpInfo.ctx.TryNewPacketBufferFromPayloader(header.UDPMinimumSize+int(pktInfo.MaxHeaderLength), p) + if err != nil { + return 0, err } defer pkt.DecRef() diff --git a/pkg/tcpip/transport/udp/endpoint_state.go b/pkg/tcpip/transport/udp/endpoint_state.go index 66ed425..095c2aa 100644 --- a/pkg/tcpip/transport/udp/endpoint_state.go +++ b/pkg/tcpip/transport/udp/endpoint_state.go @@ -21,7 +21,6 @@ import ( "github.com/sagernet/gvisor/pkg/log" "github.com/sagernet/gvisor/pkg/tcpip" "github.com/sagernet/gvisor/pkg/tcpip/stack" - "github.com/sagernet/gvisor/pkg/tcpip/transport" ) // saveReceivedAt is invoked by stateify. @@ -36,11 +35,7 @@ func (p *udpPacket) loadReceivedAt(_ context.Context, nsec int64) { // afterLoad is invoked by stateify. func (e *endpoint) afterLoad(ctx context.Context) { - if e.stack.IsSaveRestoreEnabled() { - e.stack.RegisterRestoredEndpoint(e) - } else { - stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e) - } + e.stack.RegisterRestoredEndpoint(e) } // beforeSave is invoked by stateify. @@ -62,32 +57,7 @@ func (e *endpoint) Restore(s *stack.Stack) { // Unfreeze the endpoint to handle packets. e.frozen = false - if e.stack.IsSaveRestoreEnabled() { - e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) - return - } - e.stack = s e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits) - - switch state := e.net.State(); state { - case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed: - case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected: - // Our saved state had a port, but we don't actually have a - // reservation. We need to remove the port from our state, but still - // pass it to the reservation machinery. - var err tcpip.Error - id := e.net.Info().ID - id.LocalPort = e.localPort - id.RemotePort = e.remotePort - id, e.boundBindToDevice, err = e.registerWithStack(e.effectiveNetProtos, id) - if err != nil { - panic("registering udp endpoint with the stack failed during restore") - } - e.localPort = id.LocalPort - e.remotePort = id.RemotePort - default: - panic("unhandled state") - } } // Resume implements tcpip.ResumableEndpoint.Resume. diff --git a/pkg/tcpip/transport/udp/forwarder.go b/pkg/tcpip/transport/udp/forwarder.go index 3205b1a..8418e02 100644 --- a/pkg/tcpip/transport/udp/forwarder.go +++ b/pkg/tcpip/transport/udp/forwarder.go @@ -65,6 +65,15 @@ type ForwarderRequest struct { pkt *stack.PacketBuffer } +// NewForwarderRequest creates a new ForwarderRequest. +func NewForwarderRequest(stack *stack.Stack, id stack.TransportEndpointID, pkt *stack.PacketBuffer) *ForwarderRequest { + return &ForwarderRequest{ + stack: stack, + id: id, + pkt: pkt, + } +} + // ID returns the 4-tuple (src address, src port, dst address, dst port) that // represents the session request. func (r *ForwarderRequest) ID() stack.TransportEndpointID { @@ -106,7 +115,3 @@ func (r *ForwarderRequest) CreateEndpoint(queue *waiter.Queue) (tcpip.Endpoint, return ep, nil } - -func (r *ForwarderRequest) Packet() *stack.PacketBuffer { - return r.pkt -} diff --git a/pkg/tcpip/transport/udp/udp_state_autogen.go b/pkg/tcpip/transport/udp/udp_state_autogen.go index 4b868be..430dea3 100644 --- a/pkg/tcpip/transport/udp/udp_state_autogen.go +++ b/pkg/tcpip/transport/udp/udp_state_autogen.go @@ -31,8 +31,8 @@ func (p *udpPacket) beforeSave() {} // +checklocksignore func (p *udpPacket) StateSave(stateSinkObject state.Sink) { p.beforeSave() - var receivedAtValue int64 - receivedAtValue = p.saveReceivedAt() + receivedAtValue := p.saveReceivedAt() + _ = (int64)(receivedAtValue) stateSinkObject.SaveValue(6, receivedAtValue) stateSinkObject.Save(0, &p.udpPacketEntry) stateSinkObject.Save(1, &p.netProto) diff --git a/pkg/waiter/waiter.go b/pkg/waiter/waiter.go index 71b29d4..fb20468 100644 --- a/pkg/waiter/waiter.go +++ b/pkg/waiter/waiter.go @@ -154,6 +154,15 @@ func (e *Entry) Init(eventListener EventListener, mask EventMask) { e.mask = mask } +// SetQueuedMask changes the entry mask. +// +// Preconditions: The Entry must be registered to the given Queue. +func (e *Entry) SetQueuedMask(q *Queue, mask EventMask) { + q.mu.Lock() + e.mask = mask + q.mu.Unlock() +} + // Mask returns the entry mask. func (e *Entry) Mask() EventMask { return e.mask @@ -201,6 +210,14 @@ func NewFunctionEntry(mask EventMask, fn func(EventMask)) (e Entry) { return e } +// NoopListener is an EventListener that does nothing. +// +// +stateify savable +type NoopListener struct{} + +// NotifyEvent implements EventListener.NotifyEvent. +func (NoopListener) NotifyEvent(mask EventMask) {} + // Queue represents the wait queue where waiters can be added and // notifiers can notify them when events happen. // diff --git a/pkg/waiter/waiter_state_autogen.go b/pkg/waiter/waiter_state_autogen.go index 073c9e0..7f0fd2b 100644 --- a/pkg/waiter/waiter_state_autogen.go +++ b/pkg/waiter/waiter_state_autogen.go @@ -39,6 +39,27 @@ func (e *Entry) StateLoad(ctx context.Context, stateSourceObject state.Source) { stateSourceObject.Load(2, &e.mask) } +func (n *NoopListener) StateTypeName() string { + return "pkg/waiter.NoopListener" +} + +func (n *NoopListener) StateFields() []string { + return []string{} +} + +func (n *NoopListener) beforeSave() {} + +// +checklocksignore +func (n *NoopListener) StateSave(stateSinkObject state.Sink) { + n.beforeSave() +} + +func (n *NoopListener) afterLoad(context.Context) {} + +// +checklocksignore +func (n *NoopListener) StateLoad(ctx context.Context, stateSourceObject state.Source) { +} + func (q *Queue) StateTypeName() string { return "pkg/waiter.Queue" } @@ -122,6 +143,7 @@ func (e *waiterEntry) StateLoad(ctx context.Context, stateSourceObject state.Sou func init() { state.Register((*Entry)(nil)) + state.Register((*NoopListener)(nil)) state.Register((*Queue)(nil)) state.Register((*waiterList)(nil)) state.Register((*waiterEntry)(nil)) diff --git a/pkg/xdp/xdp_unsafe.go b/pkg/xdp/xdp_unsafe.go index a088b9f..9725c68 100644 --- a/pkg/xdp/xdp_unsafe.go +++ b/pkg/xdp/xdp_unsafe.go @@ -16,7 +16,6 @@ package xdp import ( "fmt" - "reflect" "unsafe" "github.com/sagernet/gvisor/pkg/atomicbitops" @@ -62,20 +61,16 @@ func sizeOfTXQueueDesc() uint64 { } func (fq *FillQueue) init(off unix.XDPMmapOffsets, opts Opts) { - fillQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&fq.ring)) - fillQueueRingHdr.Data = uintptr(unsafe.Pointer(&fq.mem[off.Fr.Desc])) - fillQueueRingHdr.Len = int(opts.NDescriptors) - fillQueueRingHdr.Cap = fillQueueRingHdr.Len + base := unsafe.Pointer(&fq.mem[off.Fr.Desc]) + fq.ring = unsafe.Slice((*uint64)(base), int(opts.NDescriptors)) fq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Producer])) fq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Consumer])) fq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Flags])) } func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts Opts) { - rxQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&rq.ring)) - rxQueueRingHdr.Data = uintptr(unsafe.Pointer(&rq.mem[off.Rx.Desc])) - rxQueueRingHdr.Len = int(opts.NDescriptors) - rxQueueRingHdr.Cap = rxQueueRingHdr.Len + base := unsafe.Pointer(&rq.mem[off.Rx.Desc]) + rq.ring = unsafe.Slice((*unix.XDPDesc)(base), int(opts.NDescriptors)) rq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Producer])) rq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Consumer])) rq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Flags])) @@ -86,10 +81,8 @@ func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts Opts) { } func (cq *CompletionQueue) init(off unix.XDPMmapOffsets, opts Opts) { - completionQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cq.ring)) - completionQueueRingHdr.Data = uintptr(unsafe.Pointer(&cq.mem[off.Cr.Desc])) - completionQueueRingHdr.Len = int(opts.NDescriptors) - completionQueueRingHdr.Cap = completionQueueRingHdr.Len + base := unsafe.Pointer(&cq.mem[off.Cr.Desc]) + cq.ring = unsafe.Slice((*uint64)(base), int(opts.NDescriptors)) cq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Producer])) cq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Consumer])) cq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&cq.mem[off.Cr.Flags])) @@ -100,10 +93,8 @@ func (cq *CompletionQueue) init(off unix.XDPMmapOffsets, opts Opts) { } func (tq *TXQueue) init(off unix.XDPMmapOffsets, opts Opts) { - txQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&tq.ring)) - txQueueRingHdr.Data = uintptr(unsafe.Pointer(&tq.mem[off.Tx.Desc])) - txQueueRingHdr.Len = int(opts.NDescriptors) - txQueueRingHdr.Cap = txQueueRingHdr.Len + base := unsafe.Pointer(&tq.mem[off.Tx.Desc]) + tq.ring = unsafe.Slice((*unix.XDPDesc)(base), int(opts.NDescriptors)) tq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Producer])) tq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Consumer])) tq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&tq.mem[off.Tx.Flags])) diff --git a/sandboxexec/proto/sandbox_options_go_proto/sandbox_options.pb.go b/sandboxexec/proto/sandbox_options_go_proto/sandbox_options.pb.go new file mode 100644 index 0000000..ce4bb41 --- /dev/null +++ b/sandboxexec/proto/sandbox_options_go_proto/sandbox_options.pb.go @@ -0,0 +1,835 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v6.33.4 +// source: sandboxexec/proto/sandbox_options.proto + +package sandbox_options_go_proto + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EnvVar_EnvVarPolicy int32 + +const ( + EnvVar_ENV_VAR_POLICY_UNSPECIFIED EnvVar_EnvVarPolicy = 0 + EnvVar_ENV_VAR_POLICY_FORWARD EnvVar_EnvVarPolicy = 1 + EnvVar_ENV_VAR_POLICY_UNSET EnvVar_EnvVarPolicy = 2 +) + +// Enum value maps for EnvVar_EnvVarPolicy. +var ( + EnvVar_EnvVarPolicy_name = map[int32]string{ + 0: "ENV_VAR_POLICY_UNSPECIFIED", + 1: "ENV_VAR_POLICY_FORWARD", + 2: "ENV_VAR_POLICY_UNSET", + } + EnvVar_EnvVarPolicy_value = map[string]int32{ + "ENV_VAR_POLICY_UNSPECIFIED": 0, + "ENV_VAR_POLICY_FORWARD": 1, + "ENV_VAR_POLICY_UNSET": 2, + } +) + +func (x EnvVar_EnvVarPolicy) Enum() *EnvVar_EnvVarPolicy { + p := new(EnvVar_EnvVarPolicy) + *p = x + return p +} + +func (x EnvVar_EnvVarPolicy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EnvVar_EnvVarPolicy) Descriptor() protoreflect.EnumDescriptor { + return file_sandboxexec_proto_sandbox_options_proto_enumTypes[0].Descriptor() +} + +func (EnvVar_EnvVarPolicy) Type() protoreflect.EnumType { + return &file_sandboxexec_proto_sandbox_options_proto_enumTypes[0] +} + +func (x EnvVar_EnvVarPolicy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EnvVar_EnvVarPolicy.Descriptor instead. +func (EnvVar_EnvVarPolicy) EnumDescriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{3, 0} +} + +type PortMapping_PortMappingType int32 + +const ( + PortMapping_PORT_MAPPING_TYPE_TCP PortMapping_PortMappingType = 0 + PortMapping_PORT_MAPPING_TYPE_UDP PortMapping_PortMappingType = 1 +) + +// Enum value maps for PortMapping_PortMappingType. +var ( + PortMapping_PortMappingType_name = map[int32]string{ + 0: "PORT_MAPPING_TYPE_TCP", + 1: "PORT_MAPPING_TYPE_UDP", + } + PortMapping_PortMappingType_value = map[string]int32{ + "PORT_MAPPING_TYPE_TCP": 0, + "PORT_MAPPING_TYPE_UDP": 1, + } +) + +func (x PortMapping_PortMappingType) Enum() *PortMapping_PortMappingType { + p := new(PortMapping_PortMappingType) + *p = x + return p +} + +func (x PortMapping_PortMappingType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PortMapping_PortMappingType) Descriptor() protoreflect.EnumDescriptor { + return file_sandboxexec_proto_sandbox_options_proto_enumTypes[1].Descriptor() +} + +func (PortMapping_PortMappingType) Type() protoreflect.EnumType { + return &file_sandboxexec_proto_sandbox_options_proto_enumTypes[1] +} + +func (x PortMapping_PortMappingType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use PortMapping_PortMappingType.Descriptor instead. +func (PortMapping_PortMappingType) EnumDescriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{4, 0} +} + +type SandboxOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mounts []*Mount `protobuf:"bytes,1,rep,name=mounts,proto3" json:"mounts,omitempty"` + NetworkingOptions *NetworkingOptions `protobuf:"bytes,2,opt,name=networking_options,json=networkingOptions,proto3" json:"networking_options,omitempty"` + StdoutFile string `protobuf:"bytes,3,opt,name=stdout_file,json=stdoutFile,proto3" json:"stdout_file,omitempty"` + StderrFile string `protobuf:"bytes,4,opt,name=stderr_file,json=stderrFile,proto3" json:"stderr_file,omitempty"` + EnvVars []*EnvVar `protobuf:"bytes,5,rep,name=env_vars,json=envVars,proto3" json:"env_vars,omitempty"` + DebugOptions *DebugOptions `protobuf:"bytes,6,opt,name=debug_options,json=debugOptions,proto3" json:"debug_options,omitempty"` + StartTimeoutMs int64 `protobuf:"varint,7,opt,name=start_timeout_ms,json=startTimeoutMs,proto3" json:"start_timeout_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SandboxOptions) Reset() { + *x = SandboxOptions{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SandboxOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SandboxOptions) ProtoMessage() {} + +func (x *SandboxOptions) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SandboxOptions.ProtoReflect.Descriptor instead. +func (*SandboxOptions) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{0} +} + +func (x *SandboxOptions) GetMounts() []*Mount { + if x != nil { + return x.Mounts + } + return nil +} + +func (x *SandboxOptions) GetNetworkingOptions() *NetworkingOptions { + if x != nil { + return x.NetworkingOptions + } + return nil +} + +func (x *SandboxOptions) GetStdoutFile() string { + if x != nil { + return x.StdoutFile + } + return "" +} + +func (x *SandboxOptions) GetStderrFile() string { + if x != nil { + return x.StderrFile + } + return "" +} + +func (x *SandboxOptions) GetEnvVars() []*EnvVar { + if x != nil { + return x.EnvVars + } + return nil +} + +func (x *SandboxOptions) GetDebugOptions() *DebugOptions { + if x != nil { + return x.DebugOptions + } + return nil +} + +func (x *SandboxOptions) GetStartTimeoutMs() int64 { + if x != nil { + return x.StartTimeoutMs + } + return 0 +} + +type NetworkingOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + EnableExternalNetworking bool `protobuf:"varint,1,opt,name=enable_external_networking,json=enableExternalNetworking,proto3" json:"enable_external_networking,omitempty"` + PortMappings []*PortMapping `protobuf:"bytes,2,rep,name=port_mappings,json=portMappings,proto3" json:"port_mappings,omitempty"` + DomainAllowlist []string `protobuf:"bytes,3,rep,name=domain_allowlist,json=domainAllowlist,proto3" json:"domain_allowlist,omitempty"` + DomainDenylist []string `protobuf:"bytes,4,rep,name=domain_denylist,json=domainDenylist,proto3" json:"domain_denylist,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkingOptions) Reset() { + *x = NetworkingOptions{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkingOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkingOptions) ProtoMessage() {} + +func (x *NetworkingOptions) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkingOptions.ProtoReflect.Descriptor instead. +func (*NetworkingOptions) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{1} +} + +func (x *NetworkingOptions) GetEnableExternalNetworking() bool { + if x != nil { + return x.EnableExternalNetworking + } + return false +} + +func (x *NetworkingOptions) GetPortMappings() []*PortMapping { + if x != nil { + return x.PortMappings + } + return nil +} + +func (x *NetworkingOptions) GetDomainAllowlist() []string { + if x != nil { + return x.DomainAllowlist + } + return nil +} + +func (x *NetworkingOptions) GetDomainDenylist() []string { + if x != nil { + return x.DomainDenylist + } + return nil +} + +type DebugOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + EnableDebugLogs string `protobuf:"bytes,1,opt,name=enable_debug_logs,json=enableDebugLogs,proto3" json:"enable_debug_logs,omitempty"` + DebugLogFile string `protobuf:"bytes,2,opt,name=debug_log_file,json=debugLogFile,proto3" json:"debug_log_file,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DebugOptions) Reset() { + *x = DebugOptions{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DebugOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DebugOptions) ProtoMessage() {} + +func (x *DebugOptions) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DebugOptions.ProtoReflect.Descriptor instead. +func (*DebugOptions) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{2} +} + +func (x *DebugOptions) GetEnableDebugLogs() string { + if x != nil { + return x.EnableDebugLogs + } + return "" +} + +func (x *DebugOptions) GetDebugLogFile() string { + if x != nil { + return x.DebugLogFile + } + return "" +} + +type EnvVar struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Types that are valid to be assigned to PolicyOrValue: + // + // *EnvVar_Value + // *EnvVar_Policy + PolicyOrValue isEnvVar_PolicyOrValue `protobuf_oneof:"policy_or_value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvVar) Reset() { + *x = EnvVar{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvVar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvVar) ProtoMessage() {} + +func (x *EnvVar) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. +func (*EnvVar) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{3} +} + +func (x *EnvVar) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *EnvVar) GetPolicyOrValue() isEnvVar_PolicyOrValue { + if x != nil { + return x.PolicyOrValue + } + return nil +} + +func (x *EnvVar) GetValue() string { + if x != nil { + if x, ok := x.PolicyOrValue.(*EnvVar_Value); ok { + return x.Value + } + } + return "" +} + +func (x *EnvVar) GetPolicy() EnvVar_EnvVarPolicy { + if x != nil { + if x, ok := x.PolicyOrValue.(*EnvVar_Policy); ok { + return x.Policy + } + } + return EnvVar_ENV_VAR_POLICY_UNSPECIFIED +} + +type isEnvVar_PolicyOrValue interface { + isEnvVar_PolicyOrValue() +} + +type EnvVar_Value struct { + Value string `protobuf:"bytes,2,opt,name=value,proto3,oneof"` +} + +type EnvVar_Policy struct { + Policy EnvVar_EnvVarPolicy `protobuf:"varint,3,opt,name=policy,proto3,enum=cloud_gvisor_sandboxexec.EnvVar_EnvVarPolicy,oneof"` +} + +func (*EnvVar_Value) isEnvVar_PolicyOrValue() {} + +func (*EnvVar_Policy) isEnvVar_PolicyOrValue() {} + +type PortMapping struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxPort uint32 `protobuf:"varint,1,opt,name=sandbox_port,json=sandboxPort,proto3" json:"sandbox_port,omitempty"` + HostPort uint32 `protobuf:"varint,2,opt,name=host_port,json=hostPort,proto3" json:"host_port,omitempty"` + PortMappingType PortMapping_PortMappingType `protobuf:"varint,3,opt,name=port_mapping_type,json=portMappingType,proto3,enum=cloud_gvisor_sandboxexec.PortMapping_PortMappingType" json:"port_mapping_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PortMapping) Reset() { + *x = PortMapping{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PortMapping) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PortMapping) ProtoMessage() {} + +func (x *PortMapping) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PortMapping.ProtoReflect.Descriptor instead. +func (*PortMapping) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{4} +} + +func (x *PortMapping) GetSandboxPort() uint32 { + if x != nil { + return x.SandboxPort + } + return 0 +} + +func (x *PortMapping) GetHostPort() uint32 { + if x != nil { + return x.HostPort + } + return 0 +} + +func (x *PortMapping) GetPortMappingType() PortMapping_PortMappingType { + if x != nil { + return x.PortMappingType + } + return PortMapping_PORT_MAPPING_TYPE_TCP +} + +type MountOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Passthrough bool `protobuf:"varint,1,opt,name=passthrough,proto3" json:"passthrough,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MountOptions) Reset() { + *x = MountOptions{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MountOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountOptions) ProtoMessage() {} + +func (x *MountOptions) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountOptions.ProtoReflect.Descriptor instead. +func (*MountOptions) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{5} +} + +func (x *MountOptions) GetPassthrough() bool { + if x != nil { + return x.Passthrough + } + return false +} + +type Mount struct { + state protoimpl.MessageState `protogen:"open.v1"` + Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` + // Types that are valid to be assigned to Mount: + // + // *Mount_HostMount + // *Mount_SandboxTmpfsMount + Mount isMount_Mount `protobuf_oneof:"mount"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Mount) Reset() { + *x = Mount{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Mount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Mount) ProtoMessage() {} + +func (x *Mount) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Mount.ProtoReflect.Descriptor instead. +func (*Mount) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{6} +} + +func (x *Mount) GetTarget() string { + if x != nil { + return x.Target + } + return "" +} + +func (x *Mount) GetMount() isMount_Mount { + if x != nil { + return x.Mount + } + return nil +} + +func (x *Mount) GetHostMount() *HostMount { + if x != nil { + if x, ok := x.Mount.(*Mount_HostMount); ok { + return x.HostMount + } + } + return nil +} + +func (x *Mount) GetSandboxTmpfsMount() *SandboxTmpfsMount { + if x != nil { + if x, ok := x.Mount.(*Mount_SandboxTmpfsMount); ok { + return x.SandboxTmpfsMount + } + } + return nil +} + +type isMount_Mount interface { + isMount_Mount() +} + +type Mount_HostMount struct { + HostMount *HostMount `protobuf:"bytes,2,opt,name=host_mount,json=hostMount,proto3,oneof"` +} + +type Mount_SandboxTmpfsMount struct { + SandboxTmpfsMount *SandboxTmpfsMount `protobuf:"bytes,3,opt,name=sandbox_tmpfs_mount,json=sandboxTmpfsMount,proto3,oneof"` +} + +func (*Mount_HostMount) isMount_Mount() {} + +func (*Mount_SandboxTmpfsMount) isMount_Mount() {} + +type HostMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + HostPath string `protobuf:"bytes,1,opt,name=host_path,json=hostPath,proto3" json:"host_path,omitempty"` + Options *MountOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HostMount) Reset() { + *x = HostMount{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HostMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HostMount) ProtoMessage() {} + +func (x *HostMount) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HostMount.ProtoReflect.Descriptor instead. +func (*HostMount) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{7} +} + +func (x *HostMount) GetHostPath() string { + if x != nil { + return x.HostPath + } + return "" +} + +func (x *HostMount) GetOptions() *MountOptions { + if x != nil { + return x.Options + } + return nil +} + +type SandboxTmpfsMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SandboxTmpfsMount) Reset() { + *x = SandboxTmpfsMount{} + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SandboxTmpfsMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SandboxTmpfsMount) ProtoMessage() {} + +func (x *SandboxTmpfsMount) ProtoReflect() protoreflect.Message { + mi := &file_sandboxexec_proto_sandbox_options_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SandboxTmpfsMount.ProtoReflect.Descriptor instead. +func (*SandboxTmpfsMount) Descriptor() ([]byte, []int) { + return file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP(), []int{8} +} + +var File_sandboxexec_proto_sandbox_options_proto protoreflect.FileDescriptor + +const file_sandboxexec_proto_sandbox_options_proto_rawDesc = "" + + "\n" + + "'sandboxexec/proto/sandbox_options.proto\x12\x18cloud_gvisor_sandboxexec\"\x9b\x03\n" + + "\x0eSandboxOptions\x127\n" + + "\x06mounts\x18\x01 \x03(\v2\x1f.cloud_gvisor_sandboxexec.MountR\x06mounts\x12Z\n" + + "\x12networking_options\x18\x02 \x01(\v2+.cloud_gvisor_sandboxexec.NetworkingOptionsR\x11networkingOptions\x12\x1f\n" + + "\vstdout_file\x18\x03 \x01(\tR\n" + + "stdoutFile\x12\x1f\n" + + "\vstderr_file\x18\x04 \x01(\tR\n" + + "stderrFile\x12;\n" + + "\benv_vars\x18\x05 \x03(\v2 .cloud_gvisor_sandboxexec.EnvVarR\aenvVars\x12K\n" + + "\rdebug_options\x18\x06 \x01(\v2&.cloud_gvisor_sandboxexec.DebugOptionsR\fdebugOptions\x12(\n" + + "\x10start_timeout_ms\x18\a \x01(\x03R\x0estartTimeoutMs\"\xf1\x01\n" + + "\x11NetworkingOptions\x12<\n" + + "\x1aenable_external_networking\x18\x01 \x01(\bR\x18enableExternalNetworking\x12J\n" + + "\rport_mappings\x18\x02 \x03(\v2%.cloud_gvisor_sandboxexec.PortMappingR\fportMappings\x12)\n" + + "\x10domain_allowlist\x18\x03 \x03(\tR\x0fdomainAllowlist\x12'\n" + + "\x0fdomain_denylist\x18\x04 \x03(\tR\x0edomainDenylist\"`\n" + + "\fDebugOptions\x12*\n" + + "\x11enable_debug_logs\x18\x01 \x01(\tR\x0fenableDebugLogs\x12$\n" + + "\x0edebug_log_file\x18\x02 \x01(\tR\fdebugLogFile\"\xf6\x01\n" + + "\x06EnvVar\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x05value\x18\x02 \x01(\tH\x00R\x05value\x12G\n" + + "\x06policy\x18\x03 \x01(\x0e2-.cloud_gvisor_sandboxexec.EnvVar.EnvVarPolicyH\x00R\x06policy\"d\n" + + "\fEnvVarPolicy\x12\x1e\n" + + "\x1aENV_VAR_POLICY_UNSPECIFIED\x10\x00\x12\x1a\n" + + "\x16ENV_VAR_POLICY_FORWARD\x10\x01\x12\x18\n" + + "\x14ENV_VAR_POLICY_UNSET\x10\x02B\x11\n" + + "\x0fpolicy_or_value\"\xf9\x01\n" + + "\vPortMapping\x12!\n" + + "\fsandbox_port\x18\x01 \x01(\rR\vsandboxPort\x12\x1b\n" + + "\thost_port\x18\x02 \x01(\rR\bhostPort\x12a\n" + + "\x11port_mapping_type\x18\x03 \x01(\x0e25.cloud_gvisor_sandboxexec.PortMapping.PortMappingTypeR\x0fportMappingType\"G\n" + + "\x0fPortMappingType\x12\x19\n" + + "\x15PORT_MAPPING_TYPE_TCP\x10\x00\x12\x19\n" + + "\x15PORT_MAPPING_TYPE_UDP\x10\x01\"0\n" + + "\fMountOptions\x12 \n" + + "\vpassthrough\x18\x01 \x01(\bR\vpassthrough\"\xcd\x01\n" + + "\x05Mount\x12\x16\n" + + "\x06target\x18\x01 \x01(\tR\x06target\x12D\n" + + "\n" + + "host_mount\x18\x02 \x01(\v2#.cloud_gvisor_sandboxexec.HostMountH\x00R\thostMount\x12]\n" + + "\x13sandbox_tmpfs_mount\x18\x03 \x01(\v2+.cloud_gvisor_sandboxexec.SandboxTmpfsMountH\x00R\x11sandboxTmpfsMountB\a\n" + + "\x05mount\"j\n" + + "\tHostMount\x12\x1b\n" + + "\thost_path\x18\x01 \x01(\tR\bhostPath\x12@\n" + + "\aoptions\x18\x02 \x01(\v2&.cloud_gvisor_sandboxexec.MountOptionsR\aoptions\"\x13\n" + + "\x11SandboxTmpfsMountb\x06proto3" + +var ( + file_sandboxexec_proto_sandbox_options_proto_rawDescOnce sync.Once + file_sandboxexec_proto_sandbox_options_proto_rawDescData []byte +) + +func file_sandboxexec_proto_sandbox_options_proto_rawDescGZIP() []byte { + file_sandboxexec_proto_sandbox_options_proto_rawDescOnce.Do(func() { + file_sandboxexec_proto_sandbox_options_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sandboxexec_proto_sandbox_options_proto_rawDesc), len(file_sandboxexec_proto_sandbox_options_proto_rawDesc))) + }) + return file_sandboxexec_proto_sandbox_options_proto_rawDescData +} + +var ( + file_sandboxexec_proto_sandbox_options_proto_enumTypes = make([]protoimpl.EnumInfo, 2) + file_sandboxexec_proto_sandbox_options_proto_msgTypes = make([]protoimpl.MessageInfo, 9) + file_sandboxexec_proto_sandbox_options_proto_goTypes = []any{ + (EnvVar_EnvVarPolicy)(0), // 0: cloud_gvisor_sandboxexec.EnvVar.EnvVarPolicy + (PortMapping_PortMappingType)(0), // 1: cloud_gvisor_sandboxexec.PortMapping.PortMappingType + (*SandboxOptions)(nil), // 2: cloud_gvisor_sandboxexec.SandboxOptions + (*NetworkingOptions)(nil), // 3: cloud_gvisor_sandboxexec.NetworkingOptions + (*DebugOptions)(nil), // 4: cloud_gvisor_sandboxexec.DebugOptions + (*EnvVar)(nil), // 5: cloud_gvisor_sandboxexec.EnvVar + (*PortMapping)(nil), // 6: cloud_gvisor_sandboxexec.PortMapping + (*MountOptions)(nil), // 7: cloud_gvisor_sandboxexec.MountOptions + (*Mount)(nil), // 8: cloud_gvisor_sandboxexec.Mount + (*HostMount)(nil), // 9: cloud_gvisor_sandboxexec.HostMount + (*SandboxTmpfsMount)(nil), // 10: cloud_gvisor_sandboxexec.SandboxTmpfsMount + } +) + +var file_sandboxexec_proto_sandbox_options_proto_depIdxs = []int32{ + 8, // 0: cloud_gvisor_sandboxexec.SandboxOptions.mounts:type_name -> cloud_gvisor_sandboxexec.Mount + 3, // 1: cloud_gvisor_sandboxexec.SandboxOptions.networking_options:type_name -> cloud_gvisor_sandboxexec.NetworkingOptions + 5, // 2: cloud_gvisor_sandboxexec.SandboxOptions.env_vars:type_name -> cloud_gvisor_sandboxexec.EnvVar + 4, // 3: cloud_gvisor_sandboxexec.SandboxOptions.debug_options:type_name -> cloud_gvisor_sandboxexec.DebugOptions + 6, // 4: cloud_gvisor_sandboxexec.NetworkingOptions.port_mappings:type_name -> cloud_gvisor_sandboxexec.PortMapping + 0, // 5: cloud_gvisor_sandboxexec.EnvVar.policy:type_name -> cloud_gvisor_sandboxexec.EnvVar.EnvVarPolicy + 1, // 6: cloud_gvisor_sandboxexec.PortMapping.port_mapping_type:type_name -> cloud_gvisor_sandboxexec.PortMapping.PortMappingType + 9, // 7: cloud_gvisor_sandboxexec.Mount.host_mount:type_name -> cloud_gvisor_sandboxexec.HostMount + 10, // 8: cloud_gvisor_sandboxexec.Mount.sandbox_tmpfs_mount:type_name -> cloud_gvisor_sandboxexec.SandboxTmpfsMount + 7, // 9: cloud_gvisor_sandboxexec.HostMount.options:type_name -> cloud_gvisor_sandboxexec.MountOptions + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_sandboxexec_proto_sandbox_options_proto_init() } +func file_sandboxexec_proto_sandbox_options_proto_init() { + if File_sandboxexec_proto_sandbox_options_proto != nil { + return + } + file_sandboxexec_proto_sandbox_options_proto_msgTypes[3].OneofWrappers = []any{ + (*EnvVar_Value)(nil), + (*EnvVar_Policy)(nil), + } + file_sandboxexec_proto_sandbox_options_proto_msgTypes[6].OneofWrappers = []any{ + (*Mount_HostMount)(nil), + (*Mount_SandboxTmpfsMount)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sandboxexec_proto_sandbox_options_proto_rawDesc), len(file_sandboxexec_proto_sandbox_options_proto_rawDesc)), + NumEnums: 2, + NumMessages: 9, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_sandboxexec_proto_sandbox_options_proto_goTypes, + DependencyIndexes: file_sandboxexec_proto_sandbox_options_proto_depIdxs, + EnumInfos: file_sandboxexec_proto_sandbox_options_proto_enumTypes, + MessageInfos: file_sandboxexec_proto_sandbox_options_proto_msgTypes, + }.Build() + File_sandboxexec_proto_sandbox_options_proto = out.File + file_sandboxexec_proto_sandbox_options_proto_goTypes = nil + file_sandboxexec_proto_sandbox_options_proto_depIdxs = nil +} diff --git a/sandboxexec/sandbox/oci.go b/sandboxexec/sandbox/oci.go new file mode 100644 index 0000000..421dd43 --- /dev/null +++ b/sandboxexec/sandbox/oci.go @@ -0,0 +1,165 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// BundleConfig holds configuration for creating an OCI bundle. +type BundleConfig struct { + ID string + RuntimeDir string + EnableNetworking bool + Mounts []Mount + Env []string + Annotations map[string]string + WorkingDir string + Hostname string +} + +// NewBundle creates a temporary OCI bundle on the fly with the given configuration. +func NewBundle(cfg BundleConfig) (string, error) { + // Create a bundle directory for the sandbox. + bundleDir := filepath.Join(cfg.RuntimeDir, cfg.ID) + rootfsDir := filepath.Join(bundleDir, "rootfs") + + if err := os.MkdirAll(rootfsDir, 0o755); err != nil { + return "", fmt.Errorf("failed to create bundle directories: %w", err) + } + + // Define the OCI Specification programmatically. + namespaces := []specs.LinuxNamespace{ + {Type: specs.PIDNamespace}, + {Type: specs.MountNamespace}, + {Type: specs.UTSNamespace}, + {Type: specs.IPCNamespace}, + } + + if os.Geteuid() != 0 { + namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.UserNamespace}) + } + if cfg.EnableNetworking { + namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace}) + } + + spec := &specs.Spec{ + Version: "1.0.0", + Annotations: cfg.Annotations, + Root: &specs.Root{ + Path: "rootfs", + Readonly: false, + }, + Process: &specs.Process{ + Terminal: false, + User: specs.User{UID: 0, GID: 0}, + // Keeps the sandbox alive in the background. + Args: []string{"sleep", "infinity"}, + Cwd: cfg.WorkingDir, + }, + Mounts: []specs.Mount{ + // Mandatory Linux API Filesystems + {Destination: "/proc", Type: "proc", Source: "proc"}, + {Destination: "/dev", Type: "tmpfs", Source: "tmpfs"}, + }, + // enable basic namespaces for gVisor. + Linux: &specs.Linux{ + Namespaces: namespaces, + }, + Hostname: cfg.Hostname, + } + + baseEnv := []string{"PATH=/bin:/usr/bin:/usr/local/bin"} + spec.Process.Env = append(baseEnv, cfg.Env...) + + if os.Geteuid() != 0 { + spec.Linux.UIDMappings = []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: uint32(os.Geteuid()), Size: 1}, + } + spec.Linux.GIDMappings = []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: uint32(os.Getegid()), Size: 1}, + } + } + + // Map host binaries & libraries as readonly. The binaries will be + // executed in gVisor sandbox, not on the host. + for _, p := range []string{"/bin", "/usr", "/lib", "/lib64", "/etc/alternatives"} { + if _, err := os.Stat(p); err == nil { + opts := []string{"rbind", "ro", "nosuid", "nodev"} + if p == "/etc/alternatives" { + opts = []string{"rbind", "ro"} + } + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: p, + Type: "bind", + Source: p, + Options: opts, + }) + } + } + + // Add custom mounts. Custom mounts overriding default host mounts create duplicate OCI + // entries. The later entry overrides the earlier one, as expected by OCI specs. + for _, m := range cfg.Mounts { + switch m.Type { + case MountTypeBind: + opts := []string{"rbind"} + if m.ReadOnly { + opts = append(opts, "ro") + } else { + opts = append(opts, "rw") + } + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: filepath.Clean(m.Destination), + Source: filepath.Clean(m.Source), + Type: "bind", + Options: opts, + }) + case MountTypeTmpfs: + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: filepath.Clean(m.Destination), + Source: "tmpfs", + Type: "tmpfs", + }) + case MountTypeProc: + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: filepath.Clean(m.Destination), + Source: "proc", + Type: "proc", + }) + } + } + + // Write the spec to config.json + configPath := filepath.Join(bundleDir, "config.json") + configFile, err := os.Create(configPath) + if err != nil { + return "", fmt.Errorf("failed to create config.json: %w", err) + } + defer configFile.Close() + + encoder := json.NewEncoder(configFile) + encoder.SetIndent("", " ") + if err := encoder.Encode(spec); err != nil { + return "", fmt.Errorf("failed to encode config.json: %w", err) + } + + return bundleDir, nil +} diff --git a/sandboxexec/sandbox/sandbox.go b/sandboxexec/sandbox/sandbox.go new file mode 100644 index 0000000..102bf17 --- /dev/null +++ b/sandboxexec/sandbox/sandbox.go @@ -0,0 +1,528 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sandbox provides a simple Go API for creating gVisor sandbox +// and executing commands in the sandbox. +package sandbox + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/rand" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Options holds the configuration for a Sandbox. +type Options struct { + runtimeDir string + id string + enableNetworking bool + mounts []Mount + snapshot *Snapshot + env []string + err error + workingDir string + hostname string +} + +// Option configures the Options struct. +type Option func(*Options) + +// MountType represents the type of a mount point inside the sandbox. +type MountType int + +const ( + // MountTypeBind represents a host bind mount. + MountTypeBind MountType = iota + // MountTypeTmpfs represents an in-memory tmpfs mount. + MountTypeTmpfs + // MountTypeProc represents a procfs mount. + MountTypeProc +) + +// Mount holds settings for a custom host bind directory or in-memory mount. +type Mount struct { + Source string + Destination string + Type MountType + ReadOnly bool +} + +// WithRuntimeDir sets a custom runtime directory where bundle and state files are written. +func WithRuntimeDir(runtimeDir string) Option { + return func(o *Options) { + o.runtimeDir = runtimeDir + } +} + +// WithID sets a specific sandbox ID. If not set, a unique ID will be generated automatically. +func WithID(id string) Option { + return func(o *Options) { + o.id = id + } +} + +// WithNetworking configures whether networking is enabled inside the sandbox. +func WithNetworking(enabled bool) Option { + return func(o *Options) { + o.enableNetworking = enabled + } +} + +// WithBindMount adds a custom bind mount from host's source path to the sandbox's destination path. +func WithBindMount(source, destination string, readOnly bool) Option { + return func(o *Options) { + o.mounts = append(o.mounts, Mount{ + Source: filepath.Clean(source), + Destination: filepath.Clean(destination), + Type: MountTypeBind, + ReadOnly: readOnly, + }) + } +} + +// WithTmpfsMount adds an in-memory tmpfs filesystem at the destination path inside the sandbox. +func WithTmpfsMount(destination string) Option { + return func(o *Options) { + o.mounts = append(o.mounts, Mount{ + Destination: filepath.Clean(destination), + Type: MountTypeTmpfs, + }) + } +} + +// WithProcMount adds a procfs mount at the destination path inside the sandbox. +func WithProcMount(destination string) Option { + return func(o *Options) { + o.mounts = append(o.mounts, Mount{ + Destination: filepath.Clean(destination), + Type: MountTypeProc, + }) + } +} + +// WithHostname sets the hostname for the sandbox. +func WithHostname(hostname string) Option { + return func(o *Options) { + o.hostname = hostname + } +} + +// WithSnapshot configures the sandbox to restore state from the given snapshot. +// The sandbox automatically reads the snapshot metadata to determine if it is a +// full Checkpoint/Restore, Filesystem snapshot, or Rootfs Tar snapshot. +func WithSnapshot(snapshot *Snapshot) Option { + return func(o *Options) { + o.snapshot = snapshot + } +} + +// WithEnv sets one or more environment variables in the sandbox process. +// Each env string must be in the "KEY=VALUE" format. +func WithEnv(envs ...string) Option { + return func(o *Options) { + for _, env := range envs { + if !strings.Contains(env, "=") { + o.err = fmt.Errorf("invalid environment variable format, expected KEY=VALUE: %q", env) + return + } + } + o.env = append(o.env, envs...) + } +} + +// WithWorkingDir sets the current working directory for the sandbox process. +// If the path is relative, it will be resolved as an absolute path from the root directory "/". +// This is not a bind mount; it is simply setting the cwd inside the sandbox process. +// Defaults to "/". +func WithWorkingDir(cwd string) Option { + return func(o *Options) { + if cwd == "" { + o.err = fmt.Errorf("working directory cannot be empty") + return + } + // ensure absolute path inside sandbox + if !filepath.IsAbs(cwd) { + cwd = filepath.Join("/", cwd) + } + cwd = filepath.Clean(cwd) + o.workingDir = cwd + } +} + +// Sandbox represents a running gVisor sandbox where applications +// run inside. +type Sandbox struct { + id string + bundleDir string + runscPath string + rootState string +} + +// newID returns a unique ID for the sandbox. +func newID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + // rand.Read never returns an error, and always fills b entirely. + panic(fmt.Sprintf("failed to generate random bytes for sandbox ID: %v", err)) + } + return fmt.Sprintf("%x", b) +} + +// Look for runsc binary from the environment variable RUNSC_PATH, +// then in system PATH. +func runscPath() string { + if path := os.Getenv("RUNSC_PATH"); path != "" { + return path + } + path, err := exec.LookPath("runsc") + if err == nil { + return path + } + panic("runsc binary is not found") +} + +// New spawns a new sandbox as a subprocess, the sandbox +// will be started and running in detached mode. +func New(ctx context.Context, opts ...Option) (*Sandbox, error) { + options := Options{ + enableNetworking: true, + workingDir: "/", + } + for _, o := range opts { + o(&options) + } + + if options.err != nil { + return nil, options.err + } + + if options.runtimeDir == "" { + dir, err := os.MkdirTemp("", "gvisor-sandbox-*") + if err != nil { + return nil, fmt.Errorf("failed to create runtime directory: %v", err) + } + options.runtimeDir = dir + } + + if options.id == "" { + options.id = newID() + } + + if os.Geteuid() != 0 && options.enableNetworking { + return nil, fmt.Errorf("enabling networking requires running as root") + } + + runDir := options.runtimeDir + stateDir := filepath.Join(runDir, "state") + if err := os.MkdirAll(stateDir, 0o700); err != nil { + return nil, fmt.Errorf("failed to create sandbox state directory: %v", err) + } + // Verify that the state directory actually has 0700 permissions. + fi, err := os.Stat(stateDir) + if err != nil { + return nil, fmt.Errorf("failed to stat sandbox state directory: %v", err) + } + if fi.Mode().Perm() != 0o700 { + return nil, fmt.Errorf("sandbox state directory has incorrect permissions: got %v, want %v", fi.Mode().Perm(), os.FileMode(0o700)) + } + + var annotations map[string]string + var globalFlags []string + var runFlags []string + var isCheckpointRestore bool + var checkpointRestoreDir string + + if options.snapshot != nil { + store := options.snapshot.Storage + snapshotID := options.snapshot.ID + if store == nil { + return nil, fmt.Errorf("no snapshot storage configured for restore") + } + + // Fetch metadata.json from store. + metaReader, err := store.GetReader(ctx, snapshotID, MetadataAsset) + if err != nil { + return nil, fmt.Errorf("failed to read snapshot metadata: %w", err) + } + defer metaReader.Close() + + var meta SnapshotMetadata + if err := json.NewDecoder(metaReader).Decode(&meta); err != nil { + return nil, fmt.Errorf("failed to parse snapshot metadata: %w", err) + } + + // Perform restore based on type. + switch meta.Type { + case RootfsTarSnapshot: + tarPath, err := readRootfsTar(ctx, snapshotID, store) + if err != nil { + return nil, err + } + defer os.Remove(tarPath) + + annotations = map[string]string{ + "dev.gvisor.tar.rootfs.upper": tarPath, + } + globalFlags = append(globalFlags, "--allow-rootfs-tar-annotation") + + case FilesystemSnapshot: + fsRestoreDir := filepath.Join(stateDir, "fs-restore") + if err := os.MkdirAll(fsRestoreDir, 0o700); err != nil { + return nil, err + } + // TODO: List assets in store and download all filesystem image assets to fsRestoreDir. + runFlags = append(runFlags, fmt.Sprintf("--fs-restore-image-path=%s", fsRestoreDir)) + + case CheckpointRestore: + checkpointRestoreDir = filepath.Join(stateDir, "checkpoint-restore") + if err := os.MkdirAll(checkpointRestoreDir, 0o700); err != nil { + return nil, err + } + // TODO: List assets in store and download all checkpoint image assets to checkpointRestoreDir. + isCheckpointRestore = true + } + } + bundleDir, err := NewBundle(BundleConfig{ + ID: options.id, + RuntimeDir: runDir, + EnableNetworking: options.enableNetworking, + Mounts: options.mounts, + Env: options.env, + Annotations: annotations, + WorkingDir: options.workingDir, + Hostname: options.hostname, + }) + if err != nil { + return nil, fmt.Errorf("failed to create OCI bundle: %v", err) + } + + sb := &Sandbox{ + id: options.id, + bundleDir: bundleDir, + runscPath: runscPath(), + rootState: stateDir, + } + + // Launch the sandbox in detached mode via os/exec. + args := []string{"--root", sb.rootState} + if os.Geteuid() != 0 { + args = append(args, "--ignore-cgroups") + } + if !options.enableNetworking { + args = append(args, "--network=none") + } + args = append(args, globalFlags...) + + if isCheckpointRestore { + args = append(args, "restore", "--image-path", checkpointRestoreDir, "--detach", sb.id) + } else { + args = append(args, "run") + args = append(args, runFlags...) + args = append(args, "--bundle", sb.bundleDir, "--detach", sb.id) + } + cmd := exec.CommandContext(ctx, sb.runscPath, args...) + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("failed to create sandbox via subprocess: %v", err) + } + + return sb, nil +} + +// Exec runs the given command inside the running sandbox and returns the output. +func (s *Sandbox) Exec(ctx context.Context, cmd string, opts ...string) (stdout string, stderr string, err error) { + args := append([]string{"--root", s.rootState, "exec", s.id, cmd}, opts...) + + commandContext := exec.CommandContext(ctx, s.runscPath, args...) + + var stdoutBuf, stderrBuf bytes.Buffer + + commandContext.Stdout = io.MultiWriter(&stdoutBuf) + commandContext.Stderr = io.MultiWriter(&stderrBuf) + + err = commandContext.Run() + if err != nil { + return "", stderrBuf.String(), fmt.Errorf("exec failed: %v", err) + } + + return stdoutBuf.String(), stderrBuf.String(), err +} + +// Close kills the sandbox processes and cleans up the state directory. +func (s *Sandbox) Close(ctx context.Context) error { + killArgs := []string{"--root", s.rootState, "kill", s.id, "SIGKILL"} + _ = exec.CommandContext(ctx, s.runscPath, killArgs...).Run() + + deleteArgs := []string{"--root", s.rootState, "delete", "--force", s.id} + if err := exec.CommandContext(ctx, s.runscPath, deleteArgs...).Run(); err != nil { + return fmt.Errorf("failed to clean up sandbox state: %v", err) + } + + if err := os.RemoveAll(s.bundleDir); err != nil { + return fmt.Errorf("failed to clean up sandbox bundle directory: %v", err) + } + + if err := os.RemoveAll(s.rootState); err != nil { + return fmt.Errorf("failed to clean up sandbox state directory: %v", err) + } + + return nil +} + +// Bundle returns the path to the OCI bundle directory for this sandbox. +func (s *Sandbox) Bundle() string { + return s.bundleDir +} + +// SnapshotOptions holds configuration for taking a snapshot. +type SnapshotOptions struct { + LeaveRunning bool +} + +// SnapshotOption configures SnapshotOptions. +type SnapshotOption func(*SnapshotOptions) + +// WithLeaveRunning keeps the sandbox running after taking the snapshot. +func WithLeaveRunning(leaveRunning bool) SnapshotOption { + return func(o *SnapshotOptions) { + o.LeaveRunning = leaveRunning + } +} + +func newSnapshotID() SnapshotID { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + panic(fmt.Sprintf("failed to generate random bytes for snapshot ID: %v", err)) + } + return SnapshotID(fmt.Sprintf("snap-%x", b)) +} + +// Snapshot serializes and saves the sandbox state to storage, returning the generated snapshot. +// Depending on the snapshotType, it will perform a full Checkpoint, a Filesystem Snapshot, or a Rootfs Tar Snapshot. +// It also automatically generates and writes "metadata.json" into the storage. +func (s *Sandbox) Snapshot(ctx context.Context, snapshotType SnapshotType, storage SnapshotStorage, opts ...SnapshotOption) (*Snapshot, error) { + options := SnapshotOptions{ + LeaveRunning: false, // Default is false. + } + for _, o := range opts { + o(&options) + } + + snapshotID := newSnapshotID() + + switch snapshotType { + case RootfsTarSnapshot: + if err := s.snapshotRootfsTar(ctx, snapshotID, storage); err != nil { + return nil, err + } + + case FilesystemSnapshot: + // TODO: Run `runsc fscheckpoint --image-path= [--leave-running] `. + // TODO: Walk `` and upload each file to storage. + + case CheckpointRestore: + // TODO: Run `runsc checkpoint --image-path= [--leave-running] `. + // TODO: Walk `` and upload each file to storage. + } + + meta := SnapshotMetadata{ + Type: snapshotType, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } + + metaWriter, err := storage.PutWriter(ctx, snapshotID, MetadataAsset) + if err != nil { + return nil, fmt.Errorf("failed to create metadata.json in storage: %w", err) + } + defer metaWriter.Close() + + if err := json.NewEncoder(metaWriter).Encode(&meta); err != nil { + return nil, fmt.Errorf("failed to write metadata.json to storage: %w", err) + } + + return &Snapshot{ + ID: snapshotID, + Storage: storage, + }, nil +} + +func (s *Sandbox) snapshotRootfsTar(ctx context.Context, snapshotID SnapshotID, storage SnapshotStorage) error { + tarFile, err := os.CreateTemp(os.TempDir(), "rootfs-*.tar") + if err != nil { + return fmt.Errorf("failed to create temp tar file: %w", err) + } + tarPath := tarFile.Name() + tarFile.Close() + defer os.Remove(tarPath) + + cmd := exec.CommandContext(ctx, s.runscPath, "--root", s.rootState, "tar", "rootfs-upper", "--file", tarPath, s.id) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("runsc tar failed: %v (stderr: %q)", err, stderr.String()) + } + + localFile, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("failed to open temp tar file: %w", err) + } + defer localFile.Close() + + storageWriter, err := storage.PutWriter(ctx, snapshotID, RootfsAsset) + if err != nil { + return fmt.Errorf("failed to create storage writer: %w", err) + } + defer storageWriter.Close() + + if _, err := io.Copy(storageWriter, localFile); err != nil { + return fmt.Errorf("failed to upload rootfs tar: %w", err) + } + return nil +} + +func readRootfsTar(ctx context.Context, snapshotID SnapshotID, store SnapshotStorage) (string, error) { + tarFile, err := os.CreateTemp(os.TempDir(), "rootfs-*.tar") + if err != nil { + return "", fmt.Errorf("failed to create temp tar file: %w", err) + } + tarPath := tarFile.Name() + defer tarFile.Close() + + cleanup := true + defer func() { + if cleanup { + os.Remove(tarPath) + } + }() + + storageReader, err := store.GetReader(ctx, snapshotID, RootfsAsset) + if err != nil { + return "", fmt.Errorf("failed to get rootfs reader from storage: %w", err) + } + defer storageReader.Close() + + if _, err := io.Copy(tarFile, storageReader); err != nil { + return "", fmt.Errorf("failed to download rootfs asset: %w", err) + } + + cleanup = false + return tarPath, nil +} diff --git a/sandboxexec/sandbox/sandbox_state_autogen.go b/sandboxexec/sandbox/sandbox_state_autogen.go new file mode 100644 index 0000000..79ebc22 --- /dev/null +++ b/sandboxexec/sandbox/sandbox_state_autogen.go @@ -0,0 +1,3 @@ +// automatically generated by stateify. + +package sandbox diff --git a/sandboxexec/sandbox/storage.go b/sandboxexec/sandbox/storage.go new file mode 100644 index 0000000..cbda552 --- /dev/null +++ b/sandboxexec/sandbox/storage.go @@ -0,0 +1,195 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +// SnapshotType defines the type of snapshot. +type SnapshotType string + +const ( + // CheckpointRestore represents a full process state checkpoint and restore. + CheckpointRestore SnapshotType = "CheckpointRestore" + + // FilesystemSnapshot represents a snapshot of the container's filesystems. + FilesystemSnapshot SnapshotType = "FilesystemSnapshot" + + // RootfsTarSnapshot represents a tar file snapshot of rootfs changes. + RootfsTarSnapshot SnapshotType = "RootfsTarSnapshot" +) + +// SnapshotID defines the type for snapshot IDs. +type SnapshotID string + +// SnapshotMetadata stores the metadata of a snapshot. +type SnapshotMetadata struct { + Type SnapshotType `json:"type"` + CreatedAt string `json:"created_at"` +} + +// Snapshot groups SnapshotID and SnapshotStorage together. +type Snapshot struct { + ID SnapshotID + Storage SnapshotStorage +} + +// Asset defines the type for snapshot asset names. +type Asset string + +const ( + // MetadataAsset is the name of the metadata file. + MetadataAsset Asset = "metadata.json" + // RootfsAsset is the name of the rootfs tarball (if using RootfsTarSnapshot). + RootfsAsset Asset = "rootfs.tar" + // CheckpointAsset is the main checkpoint state file. + CheckpointAsset Asset = "checkpoint.img" + // PagesAsset is the memory pages file. + PagesAsset Asset = "pages.img" + // PagesMetaAsset is the memory pages metadata file. + PagesMetaAsset Asset = "pages_meta.img" +) + +// SnapshotStorage defines a pluggable storage interface for snapshots. +type SnapshotStorage interface { + // PutWriter returns a WriteCloser to write a file asset of a snapshot. + PutWriter(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.WriteCloser, error) + + // GetReader returns a ReadCloser to read a file asset of a snapshot. + GetReader(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.ReadCloser, error) + + // Delete deletes all assets associated with a snapshot ID. + Delete(ctx context.Context, snapshotID SnapshotID) error + + // List returns all snapshot IDs known to this storage. + List(ctx context.Context) ([]SnapshotID, error) + + // Lookup verifies that the snapshot ID exists in this storage and returns a Snapshot. + Lookup(ctx context.Context, snapshotID SnapshotID) (*Snapshot, error) +} + +// ErrSnapshotNotFound is returned when the snapshot ID is not found. +var ErrSnapshotNotFound = errors.New("snapshot not found") + +// FilesystemStorage implements SnapshotStorage using a local directory. +type FilesystemStorage struct { + rootDir string +} + +// NewFilesystemStorage creates a new FilesystemStorage at the given root directory. +// The root directory must already exist. +func NewFilesystemStorage(rootDir string) (*FilesystemStorage, error) { + fi, err := os.Stat(rootDir) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("root directory %q is not a directory", rootDir) + } + return &FilesystemStorage{rootDir: rootDir}, nil +} + +func sanitizeName(s string) (string, error) { + base := filepath.Base(s) + if base == "." || base == ".." || base == "/" || base == "\\" { + return "", fmt.Errorf("invalid name: %q", s) + } + return base, nil +} + +// PutWriter returns a WriteCloser to write a file asset of a snapshot. +func (f *FilesystemStorage) PutWriter(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.WriteCloser, error) { + safeID, err := sanitizeName(string(snapshotID)) + if err != nil { + return nil, fmt.Errorf("invalid snapshot ID: %w", err) + } + safeAsset, err := sanitizeName(string(assetName)) + if err != nil { + return nil, fmt.Errorf("invalid asset name: %w", err) + } + path := filepath.Join(f.rootDir, safeID, safeAsset) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + return os.Create(path) +} + +// GetReader returns a ReadCloser to read a file asset of a snapshot. +func (f *FilesystemStorage) GetReader(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.ReadCloser, error) { + safeID, err := sanitizeName(string(snapshotID)) + if err != nil { + return nil, fmt.Errorf("invalid snapshot ID: %w", err) + } + safeAsset, err := sanitizeName(string(assetName)) + if err != nil { + return nil, fmt.Errorf("invalid asset name: %w", err) + } + path := filepath.Join(f.rootDir, safeID, safeAsset) + return os.Open(path) +} + +// Delete deletes all assets associated with a snapshot ID. +func (f *FilesystemStorage) Delete(ctx context.Context, snapshotID SnapshotID) error { + safeID, err := sanitizeName(string(snapshotID)) + if err != nil { + return fmt.Errorf("invalid snapshot ID: %w", err) + } + path := filepath.Join(f.rootDir, safeID) + return os.RemoveAll(path) +} + +// List returns all snapshot IDs known to this storage. +func (f *FilesystemStorage) List(ctx context.Context) ([]SnapshotID, error) { + entries, err := os.ReadDir(f.rootDir) + if err != nil { + return nil, err + } + ids := make([]SnapshotID, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + ids = append(ids, SnapshotID(entry.Name())) + } + } + return ids, nil +} + +// Lookup returns a Snapshot by a given snapshot ID. +func (f *FilesystemStorage) Lookup(ctx context.Context, snapshotID SnapshotID) (*Snapshot, error) { + safeID, err := sanitizeName(string(snapshotID)) + if err != nil { + return nil, fmt.Errorf("invalid snapshot ID: %w", err) + } + dir := filepath.Join(f.rootDir, safeID) + fi, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrSnapshotNotFound + } + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("snapshot path %q is not a directory", dir) + } + return &Snapshot{ + ID: SnapshotID(safeID), + Storage: f, + }, nil +}