snapshot: sagernet/gvisor v0.0.0-20250811.0-sing-box-mod.1

Содержимое пина, зафиксированного в go.mod sing-box-lx, одним коммитом
без истории. Полная история SagerNet/gvisor — 1.45 ГБ и клонируется в
каждой CI-джобе; наша дельта — одна вставка в одну функцию, история для
неё не нужна.

Module path github.com/sagernet/gvisor сохранён намеренно: на него
опирается replace-директива суперпроекта.

Патч поверх — отдельным коммитом, чтобы дельта читалась одним git show
и переносилась на новый пин копированием.

SPECS/TASKS/048-GVISOR_HANDSHAKE_NIL_CRASH

pkg/tcpip/link/sharedmem/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package sharedmem
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/sharedmem/pipe/pipe.go

@@ -0,0 +1,80 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pipe implements a shared memory ring buffer on which a single reader
+// and a single writer can operate (read/write) concurrently. The ring buffer
+// allows for data of different sizes to be written, and preserves the boundary
+// of the written data.
+//
+// Example usage is as follows:
+//
+//	wb := t.Push(20)
+//	// Write data to wb.
+//	t.Flush()
+//
+//	rb := r.Pull()
+//	// Do something with data in rb.
+//	t.Flush()
+package pipe
+
+import (
+	"math"
+)
+
+const (
+	jump           uint64 = math.MaxUint32 + 1
+	offsetMask     uint64 = math.MaxUint32
+	revolutionMask uint64 = ^offsetMask
+
+	sizeOfSlotHeader        = 8 // sizeof(uint64)
+	slotFree         uint64 = 1 << 63
+	slotSizeMask     uint64 = math.MaxUint32
+)
+
+// payloadToSlotSize calculates the total size of a slot based on its payload
+// size. The  total size is the header size, plus the payload size, plus padding
+// if necessary to make the total size a multiple of sizeOfSlotHeader.
+func payloadToSlotSize(payloadSize uint64) uint64 {
+	s := sizeOfSlotHeader + payloadSize
+	return (s + sizeOfSlotHeader - 1) &^ (sizeOfSlotHeader - 1)
+}
+
+// slotToPayloadSize calculates the payload size of a slot based on the total
+// size of the slot. This is only meant to be used when creating slots that
+// don't carry information (e.g., free slots or wrap slots).
+func slotToPayloadSize(offset uint64) uint64 {
+	return offset - sizeOfSlotHeader
+}
+
+// pipe is a basic data structure used by both (transmit & receive) ends of a
+// pipe. Indices into this pipe are split into two fields: offset, which counts
+// the number of bytes from the beginning of the buffer, and revolution, which
+// counts the number of times the index has wrapped around.
+//
+// +stateify savable
+type pipe struct {
+	buffer []byte
+}
+
+// init initializes the pipe buffer such that its size is a multiple of the size
+// of the slot header.
+func (p *pipe) init(b []byte) {
+	p.buffer = b[:len(b)&^(sizeOfSlotHeader-1)]
+}
+
+// data returns a section of the buffer starting at the given index (which may
+// include revolution information) and with the given size.
+func (p *pipe) data(idx uint64, size uint64) []byte {
+	return p.buffer[(idx&offsetMask)+sizeOfSlotHeader:][:size]
+}

pkg/tcpip/link/sharedmem/pipe/pipe_state_autogen.go

@@ -0,0 +1,111 @@
+// automatically generated by stateify.
+
+package pipe
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (p *pipe) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem/pipe.pipe"
+}
+
+func (p *pipe) StateFields() []string {
+	return []string{
+		"buffer",
+	}
+}
+
+func (p *pipe) beforeSave() {}
+
+// +checklocksignore
+func (p *pipe) StateSave(stateSinkObject state.Sink) {
+	p.beforeSave()
+	stateSinkObject.Save(0, &p.buffer)
+}
+
+func (p *pipe) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (p *pipe) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &p.buffer)
+}
+
+func (r *Rx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem/pipe.Rx"
+}
+
+func (r *Rx) StateFields() []string {
+	return []string{
+		"p",
+		"tail",
+		"head",
+	}
+}
+
+func (r *Rx) beforeSave() {}
+
+// +checklocksignore
+func (r *Rx) StateSave(stateSinkObject state.Sink) {
+	r.beforeSave()
+	stateSinkObject.Save(0, &r.p)
+	stateSinkObject.Save(1, &r.tail)
+	stateSinkObject.Save(2, &r.head)
+}
+
+func (r *Rx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (r *Rx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &r.p)
+	stateSourceObject.Load(1, &r.tail)
+	stateSourceObject.Load(2, &r.head)
+}
+
+func (t *Tx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem/pipe.Tx"
+}
+
+func (t *Tx) StateFields() []string {
+	return []string{
+		"p",
+		"maxPayloadSize",
+		"head",
+		"tail",
+		"next",
+		"tailHeader",
+	}
+}
+
+func (t *Tx) beforeSave() {}
+
+// +checklocksignore
+func (t *Tx) StateSave(stateSinkObject state.Sink) {
+	t.beforeSave()
+	stateSinkObject.Save(0, &t.p)
+	stateSinkObject.Save(1, &t.maxPayloadSize)
+	stateSinkObject.Save(2, &t.head)
+	stateSinkObject.Save(3, &t.tail)
+	stateSinkObject.Save(4, &t.next)
+	stateSinkObject.Save(5, &t.tailHeader)
+}
+
+func (t *Tx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (t *Tx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &t.p)
+	stateSourceObject.Load(1, &t.maxPayloadSize)
+	stateSourceObject.Load(2, &t.head)
+	stateSourceObject.Load(3, &t.tail)
+	stateSourceObject.Load(4, &t.next)
+	stateSourceObject.Load(5, &t.tailHeader)
+}
+
+func init() {
+	state.Register((*pipe)(nil))
+	state.Register((*Rx)(nil))
+	state.Register((*Tx)(nil))
+}

pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipe
+
+import (
+	"unsafe"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+)
+
+func (p *pipe) write(idx uint64, v uint64) {
+	ptr := (*uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0]))
+	*ptr = v
+}
+
+func (p *pipe) writeAtomic(idx uint64, v uint64) {
+	ptr := (*atomicbitops.Uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0]))
+	ptr.Store(v)
+}
+
+func (p *pipe) readAtomic(idx uint64) uint64 {
+	ptr := (*atomicbitops.Uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0]))
+	return ptr.Load()
+}

pkg/tcpip/link/sharedmem/pipe/pipe_unsafe_state_autogen.go

@@ -0,0 +1,3 @@
+// automatically generated by stateify.
+
+package pipe

pkg/tcpip/link/sharedmem/pipe/rx.go

@@ -0,0 +1,108 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipe
+
+// Rx is the receive side of the shared memory ring buffer.
+//
+// +stateify savable
+type Rx struct {
+	p pipe
+
+	tail uint64
+	head uint64
+}
+
+// Init initializes the receive end of the pipe. In the initial state, the next
+// slot to be inspected is the very first one.
+func (r *Rx) Init(b []byte) {
+	r.p.init(b)
+	r.tail = 0xfffffffe * jump
+	r.head = r.tail
+}
+
+// Pull reads the next buffer from the pipe, returning nil if there isn't one
+// currently available.
+//
+// The returned slice is available until Flush() is next called. After that, it
+// must not be touched.
+func (r *Rx) Pull() []byte {
+	if r.head == r.tail+jump {
+		// We've already pulled the whole pipe.
+		return nil
+	}
+
+	header := r.p.readAtomic(r.head)
+	if header&slotFree != 0 {
+		// The next slot is free, we can't pull it yet.
+		return nil
+	}
+
+	payloadSize := header & slotSizeMask
+	newHead := r.head + payloadToSlotSize(payloadSize)
+	headWrap := (r.head & revolutionMask) | uint64(len(r.p.buffer))
+
+	// Check if this is a wrapping slot. If that's the case, it carries no
+	// data, so we just skip it and try again from the first slot.
+	if int64(newHead-headWrap) >= 0 {
+		// If newHead passes the tail, the pipe is either damaged or the
+		// RX view of the pipe has completely wrapped without an
+		// intervening flush.
+		if int64(newHead-(r.tail+jump)) > 0 {
+			return nil
+		}
+		// The pipe is damaged if newHead doesn't point to the start of
+		// the ring.
+		if newHead&offsetMask != 0 {
+			return nil
+		}
+
+		if r.tail == r.head {
+			// If this is the first pull since the last Flush()
+			// call, we flush the state so that the sender can use
+			// this space if it needs to.
+			r.p.writeAtomic(r.head, slotFree|slotToPayloadSize(newHead-r.head))
+			r.tail = newHead
+		}
+
+		r.head = newHead
+		return r.Pull()
+	}
+
+	// Grab the buffer before updating r.head.
+	b := r.p.data(r.head, payloadSize)
+	r.head = newHead
+	return b
+}
+
+// Flush tells the transmitter that all buffers pulled since the last Flush()
+// have been used, so the transmitter is free to used their slots for further
+// transmission.
+func (r *Rx) Flush() {
+	if r.head == r.tail {
+		return
+	}
+	r.p.writeAtomic(r.tail, slotFree|slotToPayloadSize(r.head-r.tail))
+	r.tail = r.head
+}
+
+// Abort unpulls any pulled buffers.
+func (r *Rx) Abort() {
+	r.head = r.tail
+}
+
+// Bytes returns the byte slice on which the pipe operates.
+func (r *Rx) Bytes() []byte {
+	return r.p.buffer
+}

pkg/tcpip/link/sharedmem/pipe/tx.go

@@ -0,0 +1,164 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipe
+
+// Tx is the transmit side of the shared memory ring buffer.
+//
+// +stateify savable
+type Tx struct {
+	p              pipe
+	maxPayloadSize uint64
+
+	head uint64
+	tail uint64
+	next uint64
+
+	tailHeader uint64
+}
+
+// Init initializes the transmit end of the pipe. In the initial state, the next
+// slot to be written is the very first one, and the transmitter has the whole
+// ring buffer available to it.
+func (t *Tx) Init(b []byte) {
+	t.p.init(b)
+	// maxPayloadSize excludes the header of the payload, and the header
+	// of the wrapping message.
+	t.maxPayloadSize = uint64(len(t.p.buffer)) - 2*sizeOfSlotHeader
+	t.tail = 0xfffffffe * jump
+	t.next = t.tail
+	t.head = t.tail + jump
+	t.p.write(t.tail, slotFree)
+}
+
+// Capacity determines how many records of the given size can be written to the
+// pipe before it fills up.
+func (t *Tx) Capacity(recordSize uint64) uint64 {
+	available := uint64(len(t.p.buffer)) - sizeOfSlotHeader
+	entryLen := payloadToSlotSize(recordSize)
+	return available / entryLen
+}
+
+// Push reserves "payloadSize" bytes for transmission in the pipe. The caller
+// populates the returned slice with the data to be transferred and enventually
+// calls Flush() to make the data visible to the reader, or Abort() to make the
+// pipe forget all Push() calls since the last Flush().
+//
+// The returned slice is available until Flush() or Abort() is next called.
+// After that, it must not be touched.
+func (t *Tx) Push(payloadSize uint64) []byte {
+	// Fail request if we know we will never have enough room.
+	if payloadSize > t.maxPayloadSize {
+		return nil
+	}
+
+	// True if TxPipe currently has a pushed message, i.e., it is not
+	// Flush()'ed.
+	messageAhead := t.next != t.tail
+	totalLen := payloadToSlotSize(payloadSize)
+	newNext := t.next + totalLen
+	nextWrap := (t.next & revolutionMask) | uint64(len(t.p.buffer))
+	if int64(newNext-nextWrap) >= 0 {
+		// The new buffer would overflow the pipe, so we push a wrapping
+		// slot, then try to add the actual slot to the front of the
+		// pipe.
+		newNext = (newNext & revolutionMask) + jump
+		if !t.reclaim(newNext) {
+			return nil
+		}
+		wrappingPayloadSize := slotToPayloadSize(newNext - t.next)
+		oldNext := t.next
+		t.next = newNext
+		if messageAhead {
+			t.p.write(oldNext, wrappingPayloadSize)
+		} else {
+			t.tailHeader = wrappingPayloadSize
+			t.Flush()
+		}
+		return t.Push(payloadSize)
+	}
+
+	// Check that we have enough room for the buffer.
+	if !t.reclaim(newNext) {
+		return nil
+	}
+
+	if messageAhead {
+		t.p.write(t.next, payloadSize)
+	} else {
+		t.tailHeader = payloadSize
+	}
+
+	// Grab the buffer before updating t.next.
+	b := t.p.data(t.next, payloadSize)
+	t.next = newNext
+
+	return b
+}
+
+// reclaim attempts to advance the head until at least newNext. If the head is
+// already at or beyond newNext, nothing happens and true is returned; otherwise
+// it tries to reclaim slots that have already been consumed by the receive end
+// of the pipe (they will be marked as free) and returns a boolean indicating
+// whether it was successful in reclaiming enough slots.
+func (t *Tx) reclaim(newNext uint64) bool {
+	for int64(newNext-t.head) > 0 {
+		// Can't reclaim if slot is not free.
+		header := t.p.readAtomic(t.head)
+		if header&slotFree == 0 {
+			return false
+		}
+
+		payloadSize := header & slotSizeMask
+		newHead := t.head + payloadToSlotSize(payloadSize)
+
+		// Check newHead is within bounds and valid.
+		if int64(newHead-t.tail) > int64(jump) || newHead&offsetMask >= uint64(len(t.p.buffer)) {
+			return false
+		}
+
+		t.head = newHead
+	}
+
+	return true
+}
+
+// Abort causes all Push() calls since the last Flush() to be forgotten and
+// therefore they will not be made visible to the receiver.
+func (t *Tx) Abort() {
+	t.next = t.tail
+}
+
+// Flush causes all buffers pushed since the last Flush() [or Abort(), whichever
+// is the most recent] to be made visible to the receiver.
+func (t *Tx) Flush() {
+	if t.next == t.tail {
+		// Nothing to do if there are no pushed buffers.
+		return
+	}
+
+	if t.next != t.head {
+		// The receiver will spin in t.next, so we must make sure that
+		// the slotFree bit is set.
+		t.p.write(t.next, slotFree)
+	}
+
+	t.p.writeAtomic(t.tail, t.tailHeader)
+	t.tail = t.next
+}
+
+// Bytes returns the byte slice on which the pipe operates.
+func (t *Tx) Bytes() []byte {
+	return t.p.buffer
+}

pkg/tcpip/link/sharedmem/queue/queue_state_autogen.go

@@ -0,0 +1,3 @@
+// automatically generated by stateify.
+
+package queue

pkg/tcpip/link/sharedmem/queue/rx.go

@@ -0,0 +1,226 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package queue provides the implementation of transmit and receive queues
+// based on shared memory ring buffers.
+package queue
+
+import (
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+)
+
+const (
+	// Offsets within a posted buffer.
+	postedOffset           = 0
+	postedSize             = 8
+	postedRemainingInGroup = 12
+	postedUserData         = 16
+	postedID               = 24
+
+	sizeOfPostedBuffer = 32
+
+	// Offsets within a received packet header.
+	consumedPacketSize     = 0
+	consumedPacketReserved = 4
+
+	sizeOfConsumedPacketHeader = 8
+
+	// Offsets within a consumed buffer.
+	consumedOffset   = 0
+	consumedSize     = 8
+	consumedUserData = 12
+	consumedID       = 20
+
+	sizeOfConsumedBuffer = 28
+
+	// The following are the allowed states of the shared data area.
+	// EventFDUinitialized is the value stored at the start of the shared data
+	// region when it hasn't been initialized.
+	EventFDUninitialized = 0
+	// EventFDDisabled is the value stored at the start of the shared data region
+	// when notifications using eventFD has been disabled.
+	EventFDDisabled = 1
+	// EventFDEnabled is the value stored at the start of the shared data region
+	// when eventFD should be notified as the peer might be blocked waiting on
+	// notifications.
+	EventFDEnabled = 2
+)
+
+// RxBuffer is the descriptor of a receive buffer.
+type RxBuffer struct {
+	Offset   uint64
+	Size     uint32
+	ID       uint64
+	UserData uint64
+}
+
+// Rx is a receive queue. It is implemented with one tx and one rx pipe: the tx
+// pipe is used to "post" buffers, while the rx pipe is used to receive packets
+// whose contents have been written to previously posted buffers.
+//
+// This struct is thread-compatible.
+type Rx struct {
+	tx                 pipe.Tx
+	rx                 pipe.Rx
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// Init initializes the receive queue with the given pipes, and shared state
+// pointer -- the latter is used to enable/disable eventfd notifications.
+func (r *Rx) Init(tx, rx []byte, sharedEventFDState *atomicbitops.Uint32) {
+	r.sharedEventFDState = sharedEventFDState
+	r.tx.Init(tx)
+	r.rx.Init(rx)
+}
+
+// EnableNotification updates the shared state such that the peer will notify
+// the eventfd when there are packets to be dequeued.
+func (r *Rx) EnableNotification() {
+	r.sharedEventFDState.Store(EventFDEnabled)
+}
+
+// DisableNotification updates the shared state such that the peer will not
+// notify the eventfd.
+func (r *Rx) DisableNotification() {
+	r.sharedEventFDState.Store(EventFDDisabled)
+}
+
+// PostedBuffersLimit returns the maximum number of buffers that can be posted
+// before the tx queue fills up.
+func (r *Rx) PostedBuffersLimit() uint64 {
+	return r.tx.Capacity(sizeOfPostedBuffer)
+}
+
+// PostBuffers makes the given buffers available for receiving data from the
+// peer. Once they are posted, the peer is free to write to them and will
+// eventually post them back for consumption.
+func (r *Rx) PostBuffers(buffers []RxBuffer) bool {
+	for i := range buffers {
+		b := r.tx.Push(sizeOfPostedBuffer)
+		if b == nil {
+			r.tx.Abort()
+			return false
+		}
+
+		pb := &buffers[i]
+		binary.LittleEndian.PutUint64(b[postedOffset:], pb.Offset)
+		binary.LittleEndian.PutUint32(b[postedSize:], pb.Size)
+		binary.LittleEndian.PutUint32(b[postedRemainingInGroup:], 0)
+		binary.LittleEndian.PutUint64(b[postedUserData:], pb.UserData)
+		binary.LittleEndian.PutUint64(b[postedID:], pb.ID)
+	}
+
+	r.tx.Flush()
+	return true
+}
+
+// Dequeue receives buffers that have been previously posted by PostBuffers()
+// and that have been filled by the peer and posted back.
+//
+// This is similar to append() in that new buffers are appended to "bufs", with
+// reallocation only if "bufs" doesn't have enough capacity.
+func (r *Rx) Dequeue(bufs []RxBuffer) ([]RxBuffer, uint32) {
+	for {
+		outBufs := bufs
+		// Pull the next descriptor from the rx pipe.
+		b := r.rx.Pull()
+		if b == nil {
+			return bufs, 0
+		}
+
+		if len(b) < sizeOfConsumedPacketHeader {
+			log.Warningf("Ignoring packet header: size (%v) is less than header size (%v)", len(b), sizeOfConsumedPacketHeader)
+			r.rx.Flush()
+			continue
+		}
+
+		totalDataSize := binary.LittleEndian.Uint32(b[consumedPacketSize:])
+
+		// Calculate the number of buffer descriptors and copy them
+		// over to the output.
+		count := (len(b) - sizeOfConsumedPacketHeader) / sizeOfConsumedBuffer
+		offset := sizeOfConsumedPacketHeader
+		buffersSize := uint32(0)
+		for i := count; i > 0; i-- {
+			s := binary.LittleEndian.Uint32(b[offset+consumedSize:])
+			buffersSize += s
+			if buffersSize < s {
+				// The buffer size overflows an unsigned 32-bit
+				// integer, so break out and force it to be
+				// ignored.
+				totalDataSize = 1
+				buffersSize = 0
+				break
+			}
+
+			outBufs = append(outBufs, RxBuffer{
+				Offset: binary.LittleEndian.Uint64(b[offset+consumedOffset:]),
+				Size:   s,
+				ID:     binary.LittleEndian.Uint64(b[offset+consumedID:]),
+			})
+
+			offset += sizeOfConsumedBuffer
+		}
+
+		r.rx.Flush()
+
+		if buffersSize < totalDataSize {
+			// The descriptor is corrupted, ignore it.
+			log.Warningf("Ignoring packet: actual data size (%v) less than expected size (%v)", buffersSize, totalDataSize)
+			continue
+		}
+
+		return outBufs, totalDataSize
+	}
+}
+
+// Bytes returns the byte slices on which the queue operates.
+func (r *Rx) Bytes() (tx, rx []byte) {
+	return r.tx.Bytes(), r.rx.Bytes()
+}
+
+// DecodeRxBufferHeader decodes the header of a buffer posted on an rx queue.
+func DecodeRxBufferHeader(b []byte) RxBuffer {
+	return RxBuffer{
+		Offset:   binary.LittleEndian.Uint64(b[postedOffset:]),
+		Size:     binary.LittleEndian.Uint32(b[postedSize:]),
+		ID:       binary.LittleEndian.Uint64(b[postedID:]),
+		UserData: binary.LittleEndian.Uint64(b[postedUserData:]),
+	}
+}
+
+// RxCompletionSize returns the number of bytes needed to encode an rx
+// completion containing "count" buffers.
+func RxCompletionSize(count int) uint64 {
+	return sizeOfConsumedPacketHeader + uint64(count)*sizeOfConsumedBuffer
+}
+
+// EncodeRxCompletion encodes an rx completion header.
+func EncodeRxCompletion(b []byte, size, reserved uint32) {
+	binary.LittleEndian.PutUint32(b[consumedPacketSize:], size)
+	binary.LittleEndian.PutUint32(b[consumedPacketReserved:], reserved)
+}
+
+// EncodeRxCompletionBuffer encodes the i-th rx completion buffer header.
+func EncodeRxCompletionBuffer(b []byte, i int, rxb RxBuffer) {
+	b = b[RxCompletionSize(i):]
+	binary.LittleEndian.PutUint64(b[consumedOffset:], rxb.Offset)
+	binary.LittleEndian.PutUint32(b[consumedSize:], rxb.Size)
+	binary.LittleEndian.PutUint64(b[consumedUserData:], rxb.UserData)
+	binary.LittleEndian.PutUint64(b[consumedID:], rxb.ID)
+}

pkg/tcpip/link/sharedmem/queue/tx.go

@@ -0,0 +1,161 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package queue
+
+import (
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+)
+
+const (
+	// Offsets within a packet header.
+	packetID       = 0
+	packetSize     = 8
+	packetReserved = 12
+
+	sizeOfPacketHeader = 16
+
+	// Offsets with a buffer descriptor
+	bufferOffset = 0
+	bufferSize   = 8
+
+	sizeOfBufferDescriptor = 12
+)
+
+// TxBuffer is the descriptor of a transmit buffer.
+type TxBuffer struct {
+	Next   *TxBuffer
+	Offset uint64
+	Size   uint32
+}
+
+// Tx is a transmit queue. It is implemented with one tx and one rx pipe: the
+// tx pipe is used to request the transmission of packets, while the rx pipe
+// is used to receive which transmissions have completed.
+//
+// This struct is thread-compatible.
+type Tx struct {
+	tx                 pipe.Tx
+	rx                 pipe.Rx
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// Init initializes the transmit queue with the given pipes.
+func (t *Tx) Init(tx, rx []byte, sharedEventFDState *atomicbitops.Uint32) {
+	t.tx.Init(tx)
+	t.rx.Init(rx)
+	t.sharedEventFDState = sharedEventFDState
+}
+
+// NotificationsEnabled returns true if eventFD should be used to notify the
+// peer of events (eg. packet transmit etc).
+func (t *Tx) NotificationsEnabled() bool {
+	// Notifications are considered enabled unless explicitly disabled.
+	return t.sharedEventFDState.Load() != EventFDDisabled
+}
+
+// Enqueue queues the given linked list of buffers for transmission as one
+// packet. While it is queued, the caller must not modify them.
+func (t *Tx) Enqueue(id uint64, totalDataLen, bufferCount uint32, buffer *TxBuffer) bool {
+	// Reserve room in the tx pipe.
+	totalLen := sizeOfPacketHeader + uint64(bufferCount)*sizeOfBufferDescriptor
+
+	b := t.tx.Push(totalLen)
+	if b == nil {
+		return false
+	}
+
+	// Initialize the packet and buffer descriptors.
+	binary.LittleEndian.PutUint64(b[packetID:], id)
+	binary.LittleEndian.PutUint32(b[packetSize:], totalDataLen)
+	binary.LittleEndian.PutUint32(b[packetReserved:], 0)
+
+	offset := sizeOfPacketHeader
+	for i := bufferCount; i != 0; i-- {
+		binary.LittleEndian.PutUint64(b[offset+bufferOffset:], buffer.Offset)
+		binary.LittleEndian.PutUint32(b[offset+bufferSize:], buffer.Size)
+		offset += sizeOfBufferDescriptor
+		buffer = buffer.Next
+	}
+
+	t.tx.Flush()
+
+	return true
+}
+
+// CompletedPacket returns the id of the last completed transmission. The
+// returned id, if any, refers to a value passed on a previous call to
+// Enqueue().
+func (t *Tx) CompletedPacket() (id uint64, ok bool) {
+	for {
+		b := t.rx.Pull()
+		if b == nil {
+			return 0, false
+		}
+
+		if len(b) != 8 {
+			t.rx.Flush()
+			log.Warningf("Ignoring completed packet: size (%v) is less than expected (%v)", len(b), 8)
+			continue
+		}
+
+		v := binary.LittleEndian.Uint64(b)
+
+		t.rx.Flush()
+
+		return v, true
+	}
+}
+
+// Bytes returns the byte slices on which the queue operates.
+func (t *Tx) Bytes() (tx, rx []byte) {
+	return t.tx.Bytes(), t.rx.Bytes()
+}
+
+// TxPacketInfo holds information about a packet sent on a tx queue.
+type TxPacketInfo struct {
+	ID          uint64
+	Size        uint32
+	Reserved    uint32
+	BufferCount int
+}
+
+// DecodeTxPacketHeader decodes the header of a packet sent over a tx queue.
+func DecodeTxPacketHeader(b []byte) TxPacketInfo {
+	return TxPacketInfo{
+		ID:          binary.LittleEndian.Uint64(b[packetID:]),
+		Size:        binary.LittleEndian.Uint32(b[packetSize:]),
+		Reserved:    binary.LittleEndian.Uint32(b[packetReserved:]),
+		BufferCount: (len(b) - sizeOfPacketHeader) / sizeOfBufferDescriptor,
+	}
+}
+
+// DecodeTxBufferHeader decodes the header of the i-th buffer of a packet sent
+// over a tx queue.
+func DecodeTxBufferHeader(b []byte, i int) TxBuffer {
+	b = b[sizeOfPacketHeader+i*sizeOfBufferDescriptor:]
+	return TxBuffer{
+		Offset: binary.LittleEndian.Uint64(b[bufferOffset:]),
+		Size:   binary.LittleEndian.Uint32(b[bufferSize:]),
+	}
+}
+
+// EncodeTxCompletion encodes a tx completion header.
+func EncodeTxCompletion(b []byte, id uint64) {
+	binary.LittleEndian.PutUint64(b, id)
+}

pkg/tcpip/link/sharedmem/queuepair.go

@@ -0,0 +1,220 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// DefaultQueueDataSize is the size of the shared memory data region that
+	// holds the scatter/gather buffers.
+	DefaultQueueDataSize = 1 << 20 // 1MiB
+
+	// DefaultQueuePipeSize is the size of the pipe that holds the packet descriptors.
+	//
+	// Assuming each packet data is approximately 1280 bytes (IPv6 Minimum MTU)
+	// then we can hold approximately 1024*1024/1280 ~ 819 packets in the data
+	// area. Which means the pipe needs to be big enough to hold 819
+	// descriptors.
+	//
+	// Each descriptor is approximately 8 (slot descriptor in pipe) +
+	// 16 (packet descriptor) + 12 (for buffer descriptor) assuming each packet is
+	// stored in exactly 1 buffer descriptor (see queue/tx.go and pipe/tx.go.)
+	//
+	// Which means we need approximately 36*819 ~ 29 KiB to store all packet
+	// descriptors. We could go with a 32 KiB pipe but to give it some slack in
+	// how the upper layer may make use of the scatter gather buffers we double
+	// this to hold enough descriptors.
+	DefaultQueuePipeSize = 64 << 10 // 64KiB
+
+	// DefaultSharedDataSize is the size of the sharedData region used to
+	// enable/disable notifications.
+	DefaultSharedDataSize = 4 << 10 // 4KiB
+
+	// DefaultBufferSize is the size of each individual buffer that the data
+	// region is broken down into to hold packet data. Should be larger than
+	// 1500 + 14 (Ethernet header) + 10 (VirtIO header) to fit each packet
+	// in a single buffer.
+	DefaultBufferSize = 2048
+
+	// DefaultTmpDir is the path used to create the memory files if a path
+	// is not provided.
+	DefaultTmpDir = "/dev/shm"
+)
+
+// A QueuePair represents a pair of TX/RX queues.
+type QueuePair struct {
+	// txCfg is the QueueConfig to be used for transmit queue.
+	txCfg QueueConfig
+
+	// rxCfg is the QueueConfig to be used for receive queue.
+	rxCfg QueueConfig
+}
+
+// QueueOptions allows queue specific configuration to be specified when
+// creating a QueuePair.
+type QueueOptions struct {
+	// SharedMemPath is the path to use to create the shared memory backing
+	// files for the queue.
+	//
+	// If unspecified it defaults to "/dev/shm".
+	SharedMemPath string
+}
+
+// NewQueuePair creates a shared memory QueuePair.
+func NewQueuePair(opts QueueOptions) (*QueuePair, error) {
+	txCfg, err := createQueueFDs(opts.SharedMemPath, queueSizes{
+		dataSize:       DefaultQueueDataSize,
+		txPipeSize:     DefaultQueuePipeSize,
+		rxPipeSize:     DefaultQueuePipeSize,
+		sharedDataSize: DefaultSharedDataSize,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tx queue: %s", err)
+	}
+
+	rxCfg, err := createQueueFDs(opts.SharedMemPath, queueSizes{
+		dataSize:       DefaultQueueDataSize,
+		txPipeSize:     DefaultQueuePipeSize,
+		rxPipeSize:     DefaultQueuePipeSize,
+		sharedDataSize: DefaultSharedDataSize,
+	})
+	if err != nil {
+		closeFDs(txCfg)
+		return nil, fmt.Errorf("failed to create rx queue: %s", err)
+	}
+
+	return &QueuePair{
+		txCfg: txCfg,
+		rxCfg: rxCfg,
+	}, nil
+}
+
+// Close closes underlying tx/rx queue fds.
+func (q *QueuePair) Close() {
+	closeFDs(q.txCfg)
+	closeFDs(q.rxCfg)
+}
+
+// TXQueueConfig returns the QueueConfig for the receive queue.
+func (q *QueuePair) TXQueueConfig() QueueConfig {
+	return q.txCfg
+}
+
+// RXQueueConfig returns the QueueConfig for the transmit queue.
+func (q *QueuePair) RXQueueConfig() QueueConfig {
+	return q.rxCfg
+}
+
+type queueSizes struct {
+	dataSize       int64
+	txPipeSize     int64
+	rxPipeSize     int64
+	sharedDataSize int64
+}
+
+func createQueueFDs(sharedMemPath string, s queueSizes) (QueueConfig, error) {
+	success := false
+	var eventFD eventfd.Eventfd
+	var dataFD, txPipeFD, rxPipeFD, sharedDataFD int
+	defer func() {
+		if success {
+			return
+		}
+		closeFDs(QueueConfig{
+			EventFD:      eventFD,
+			DataFD:       dataFD,
+			TxPipeFD:     txPipeFD,
+			RxPipeFD:     rxPipeFD,
+			SharedDataFD: sharedDataFD,
+		})
+	}()
+	eventFD, err := eventfd.Create()
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("eventfd failed: %v", err)
+	}
+	dataFD, err = createFile(sharedMemPath, s.dataSize, false)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create dataFD: %s", err)
+	}
+	txPipeFD, err = createFile(sharedMemPath, s.txPipeSize, true)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create txPipeFD: %s", err)
+	}
+	rxPipeFD, err = createFile(sharedMemPath, s.rxPipeSize, true)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create rxPipeFD: %s", err)
+	}
+	sharedDataFD, err = createFile(sharedMemPath, s.sharedDataSize, false)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create sharedDataFD: %s", err)
+	}
+	success = true
+	return QueueConfig{
+		EventFD:      eventFD,
+		DataFD:       dataFD,
+		TxPipeFD:     txPipeFD,
+		RxPipeFD:     rxPipeFD,
+		SharedDataFD: sharedDataFD,
+	}, nil
+}
+
+func createFile(sharedMemPath string, size int64, initQueue bool) (fd int, err error) {
+	tmpDir := DefaultTmpDir
+	if sharedMemPath != "" {
+		tmpDir = sharedMemPath
+	}
+	f, err := os.CreateTemp(tmpDir, "sharedmem_test")
+	if err != nil {
+		return -1, fmt.Errorf("TempFile failed: %v", err)
+	}
+	defer f.Close()
+	unix.Unlink(f.Name())
+
+	if initQueue {
+		// Write the "slot-free" flag in the initial queue.
+		if _, err := f.WriteAt([]byte{0, 0, 0, 0, 0, 0, 0, 0x80}, 0); err != nil {
+			return -1, fmt.Errorf("WriteAt failed: %v", err)
+		}
+	}
+
+	fd, err = unix.Dup(int(f.Fd()))
+	if err != nil {
+		return -1, fmt.Errorf("unix.Dup(%d) failed: %v", f.Fd(), err)
+	}
+
+	if err := unix.Ftruncate(fd, size); err != nil {
+		unix.Close(fd)
+		return -1, fmt.Errorf("ftruncate(%d, %d) failed: %v", fd, size, err)
+	}
+
+	return fd, nil
+}
+
+func closeFDs(c QueueConfig) {
+	unix.Close(c.DataFD)
+	c.EventFD.Close()
+	unix.Close(c.TxPipeFD)
+	unix.Close(c.RxPipeFD)
+	unix.Close(c.SharedDataFD)
+}

pkg/tcpip/link/sharedmem/rx.go

@@ -0,0 +1,152 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"golang.org/x/sys/unix"
+)
+
+// rx holds all state associated with an rx queue.
+type rx struct {
+	data       []byte
+	sharedData []byte
+	q          queue.Rx
+	eventFD    eventfd.Eventfd
+}
+
+// init initializes all state needed by the rx queue based on the information
+// provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (r *rx) init(mtu uint32, c *QueueConfig) error {
+	// Map in all buffers.
+	txPipe, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+
+	rxPipe, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		return err
+	}
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		return err
+	}
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		unix.Munmap(data)
+		return err
+	}
+
+	// Duplicate the eventFD so that caller can close it but we can still
+	// use it.
+	efd, err := c.EventFD.Dup()
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		unix.Munmap(data)
+		unix.Munmap(sharedData)
+		return err
+	}
+
+	// Initialize state based on buffers.
+	r.q.Init(txPipe, rxPipe, sharedDataPointer(sharedData))
+	r.data = data
+	r.eventFD = efd
+	r.sharedData = sharedData
+
+	return nil
+}
+
+// cleanup releases all resources allocated during init() except r.eventFD. It
+// must only be called if init() has previously succeeded.
+func (r *rx) cleanup() {
+	a, b := r.q.Bytes()
+	unix.Munmap(a)
+	unix.Munmap(b)
+
+	unix.Munmap(r.data)
+	unix.Munmap(r.sharedData)
+}
+
+// notify writes to the tx.eventFD to indicate to the peer that there is data to
+// be read.
+func (r *rx) notify() {
+	r.eventFD.Notify()
+}
+
+// postAndReceive posts the provided buffers (if any), and then tries to read
+// from the receive queue.
+//
+// Capacity permitting, it reuses the posted buffer slice to store the buffers
+// that were read as well.
+//
+// This function will block if there aren't any available packets.
+func (r *rx) postAndReceive(b []queue.RxBuffer, stopRequested *atomicbitops.Uint32) ([]queue.RxBuffer, uint32) {
+	// Post the buffers first. If we cannot post, sleep until we can. We
+	// never post more than will fit concurrently, so it's safe to wait
+	// until enough room is available.
+	if len(b) != 0 && !r.q.PostBuffers(b) {
+		r.q.EnableNotification()
+		for !r.q.PostBuffers(b) {
+			r.eventFD.Wait()
+			if stopRequested.Load() != 0 {
+				r.q.DisableNotification()
+				return nil, 0
+			}
+		}
+		r.q.DisableNotification()
+	}
+
+	// Read the next set of descriptors.
+	b, n := r.q.Dequeue(b[:0])
+	if len(b) != 0 {
+		return b, n
+	}
+
+	// Data isn't immediately available. Enable eventfd notifications.
+	r.q.EnableNotification()
+	for {
+		b, n = r.q.Dequeue(b)
+		if len(b) != 0 {
+			break
+		}
+
+		// Wait for notification.
+		r.eventFD.Wait()
+		if stopRequested.Load() != 0 {
+			r.q.DisableNotification()
+			return nil, 0
+		}
+	}
+	r.q.DisableNotification()
+
+	return b, n
+}

pkg/tcpip/link/sharedmem/server_endpoint_mutex.go

@@ -0,0 +1,96 @@
+package sharedmem
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type serverEndpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var serverEndpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type serverEndpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *serverEndpointRWMutex) Lock() {
+	locking.AddGLock(serverEndpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *serverEndpointRWMutex) NestedLock(i serverEndpointlockNameIndex) {
+	locking.AddGLock(serverEndpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *serverEndpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(serverEndpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *serverEndpointRWMutex) NestedUnlock(i serverEndpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(serverEndpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RLock() {
+	locking.AddGLock(serverEndpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(serverEndpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *serverEndpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var serverEndpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func serverEndpointinitLockNames() {}
+
+func init() {
+	serverEndpointinitLockNames()
+	serverEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(serverEndpointRWMutex{}), serverEndpointlockNames)
+}

pkg/tcpip/link/sharedmem/server_rx.go

@@ -0,0 +1,162 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/cleanup"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"golang.org/x/sys/unix"
+)
+
+// +stateify savable
+type serverRx struct {
+	// packetPipe represents the receive end of the pipe that carries the packet
+	// descriptors sent by the client.
+	packetPipe pipe.Rx
+
+	// completionPipe represents the transmit end of the pipe that will carry
+	// completion notifications from the server to the client.
+	completionPipe pipe.Tx
+
+	// data represents the buffer area where the packet payload is held.
+	data []byte
+
+	// eventFD is used to notify the peer when transmission is completed.
+	eventFD eventfd.Eventfd
+
+	// sharedData the memory region to use to enable/disable notifications.
+	sharedData []byte
+
+	// sharedEventFDState is the memory region in sharedData used to enable
+	// disable notifications on eventFD.
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// init initializes all state needed by the serverTx queue based on the
+// information provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (s *serverRx) init(c *QueueConfig) error {
+	// Map in all buffers.
+	packetPipeMem, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu := cleanup.Make(func() { unix.Munmap(packetPipeMem) })
+	defer cu.Clean()
+
+	completionPipeMem, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(completionPipeMem) })
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(data) })
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(sharedData) })
+
+	// Duplicate the eventFD so that caller can close it but we can still
+	// use it.
+	efd, err := c.EventFD.Dup()
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { efd.Close() })
+
+	s.packetPipe.Init(packetPipeMem)
+	s.completionPipe.Init(completionPipeMem)
+	s.data = data
+	s.eventFD = efd
+	s.sharedData = sharedData
+	s.sharedEventFDState = sharedDataPointer(sharedData)
+
+	cu.Release()
+	return nil
+}
+
+func (s *serverRx) cleanup() {
+	unix.Munmap(s.packetPipe.Bytes())
+	unix.Munmap(s.completionPipe.Bytes())
+	unix.Munmap(s.data)
+	unix.Munmap(s.sharedData)
+	s.eventFD.Close()
+}
+
+// EnableNotification updates the shared state such that the peer will notify
+// the eventfd when there are packets to be dequeued.
+func (s *serverRx) EnableNotification() {
+	s.sharedEventFDState.Store(queue.EventFDEnabled)
+}
+
+// DisableNotification updates the shared state such that the peer will not
+// notify the eventfd.
+func (s *serverRx) DisableNotification() {
+	s.sharedEventFDState.Store(queue.EventFDDisabled)
+}
+
+// completionNotificationSize is size in bytes of a completion notification sent
+// on the completion queue after a transmitted packet has been handled.
+const completionNotificationSize = 8
+
+// receive receives a single packet from the packetPipe.
+func (s *serverRx) receive() *buffer.View {
+	desc := s.packetPipe.Pull()
+	if desc == nil {
+		return nil
+	}
+
+	pktInfo := queue.DecodeTxPacketHeader(desc)
+	contents := buffer.NewView(int(pktInfo.Size))
+	toCopy := pktInfo.Size
+	for i := 0; i < pktInfo.BufferCount; i++ {
+		txBuf := queue.DecodeTxBufferHeader(desc, i)
+		if txBuf.Size <= toCopy {
+			contents.Write(s.data[txBuf.Offset:][:txBuf.Size])
+			toCopy -= txBuf.Size
+			continue
+		}
+		contents.Write(s.data[txBuf.Offset:][:toCopy])
+		break
+	}
+
+	// Flush to let peer know that slots queued for transmission have been handled
+	// and its free to reuse the slots.
+	s.packetPipe.Flush()
+	// Encode packet completion.
+	b := s.completionPipe.Push(completionNotificationSize)
+	queue.EncodeTxCompletion(b, pktInfo.ID)
+	s.completionPipe.Flush()
+	return contents
+}
+
+func (s *serverRx) waitForPackets() {
+	s.eventFD.Wait()
+}

pkg/tcpip/link/sharedmem/server_tx.go

@@ -0,0 +1,194 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/cleanup"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"golang.org/x/sys/unix"
+)
+
+// serverTx represents the server end of the sharedmem queue and is used to send
+// packets to the peer in the buffers posted by the peer in the fillPipe.
+//
+// +stateify savable
+type serverTx struct {
+	// fillPipe represents the receive end of the pipe that carries the RxBuffers
+	// posted by the peer.
+	fillPipe pipe.Rx
+
+	// completionPipe represents the transmit end of the pipe that carries the
+	// descriptors for filled RxBuffers.
+	completionPipe pipe.Tx
+
+	// data represents the buffer area where the packet payload is held.
+	data []byte
+
+	// eventFD is used to notify the peer when fill requests are fulfilled.
+	eventFD eventfd.Eventfd
+
+	// sharedData the memory region to use to enable/disable notifications.
+	sharedData []byte
+
+	// sharedEventFDState is the memory region in sharedData used to enable/disable
+	// notifications on eventFD.
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// init initializes all tstate needed by the serverTx queue based on the
+// information provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (s *serverTx) init(c *QueueConfig) error {
+	// Map in all buffers.
+	fillPipeMem, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu := cleanup.Make(func() { unix.Munmap(fillPipeMem) })
+	defer cu.Clean()
+
+	completionPipeMem, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(completionPipeMem) })
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(data) })
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(sharedData) })
+
+	// Duplicate the eventFD so that caller can close it but we can still
+	// use it.
+	efd, err := c.EventFD.Dup()
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { efd.Close() })
+
+	cu.Release()
+
+	s.fillPipe.Init(fillPipeMem)
+	s.completionPipe.Init(completionPipeMem)
+	s.data = data
+	s.eventFD = efd
+	s.sharedData = sharedData
+	s.sharedEventFDState = sharedDataPointer(sharedData)
+
+	return nil
+}
+
+func (s *serverTx) cleanup() {
+	unix.Munmap(s.fillPipe.Bytes())
+	unix.Munmap(s.completionPipe.Bytes())
+	unix.Munmap(s.data)
+	unix.Munmap(s.sharedData)
+	s.eventFD.Close()
+}
+
+// acquireBuffers acquires enough buffers to hold all the data in views or
+// returns nil if not enough buffers are currently available.
+func (s *serverTx) acquireBuffers(pktBuffer buffer.Buffer, buffers []queue.RxBuffer) (acquiredBuffers []queue.RxBuffer) {
+	acquiredBuffers = buffers[:0]
+	wantBytes := int(pktBuffer.Size())
+	for wantBytes > 0 {
+		var b []byte
+		if b = s.fillPipe.Pull(); b == nil {
+			s.fillPipe.Abort()
+			return nil
+		}
+		rxBuffer := queue.DecodeRxBufferHeader(b)
+		acquiredBuffers = append(acquiredBuffers, rxBuffer)
+		wantBytes -= int(rxBuffer.Size)
+	}
+	return acquiredBuffers
+}
+
+// fillPacket copies the data in the provided views into buffers pulled from the
+// fillPipe and returns a slice of RxBuffers that contain the copied data as
+// well as the total number of bytes copied.
+//
+// To avoid allocations the filledBuffers are appended to the buffers slice
+// which will be grown as required. This method takes ownership of pktBuffer.
+func (s *serverTx) fillPacket(pktBuffer buffer.Buffer, buffers []queue.RxBuffer) (filledBuffers []queue.RxBuffer, totalCopied uint32) {
+	bufs := s.acquireBuffers(pktBuffer, buffers)
+	if bufs == nil {
+		pktBuffer.Release()
+		return nil, 0
+	}
+	br := pktBuffer.AsBufferReader()
+	defer br.Close()
+
+	for i := 0; br.Len() > 0 && i < len(bufs); i++ {
+		buf := bufs[i]
+		copied, err := br.Read(s.data[buf.Offset:][:buf.Size])
+		buf.Size = uint32(copied)
+		// Copy the packet into the posted buffer.
+		totalCopied += bufs[i].Size
+		if err != nil {
+			return bufs, totalCopied
+		}
+	}
+	return bufs, totalCopied
+}
+
+func (s *serverTx) transmit(pkt *stack.PacketBuffer) bool {
+	buffers := make([]queue.RxBuffer, 8)
+	buffers, totalCopied := s.fillPacket(pkt.ToBuffer(), buffers)
+	if totalCopied == 0 {
+		// drop the packet as not enough buffers were probably available
+		// to send.
+		return false
+	}
+	b := s.completionPipe.Push(queue.RxCompletionSize(len(buffers)))
+	if b == nil {
+		return false
+	}
+	queue.EncodeRxCompletion(b, totalCopied, 0 /* reserved */)
+	for i := 0; i < len(buffers); i++ {
+		queue.EncodeRxCompletionBuffer(b, i, buffers[i])
+	}
+	s.completionPipe.Flush()
+	s.fillPipe.Flush()
+	return true
+}
+
+func (s *serverTx) notificationsEnabled() bool {
+	// notifications are considered to be enabled unless explicitly disabled.
+	return s.sharedEventFDState.Load() != queue.EventFDDisabled
+}
+
+func (s *serverTx) notify() {
+	if s.notificationsEnabled() {
+		s.eventFD.Notify()
+	}
+}

pkg/tcpip/link/sharedmem/sharedmem.go

@@ -0,0 +1,559 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+// Package sharedmem provides the implementation of data-link layer endpoints
+// backed by shared memory.
+//
+// Shared memory endpoints can be used in the networking stack by calling New()
+// to create a new endpoint, and then passing it as an argument to
+// Stack.CreateNIC().
+package sharedmem
+
+import (
+	"fmt"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// QueueConfig holds all the file descriptors needed to describe a tx or rx
+// queue over shared memory. It is used when creating new shared memory
+// endpoints to describe tx and rx queues.
+//
+// +stateify savable
+type QueueConfig struct {
+	// DataFD is a file descriptor for the file that contains the data to
+	// be transmitted via this queue. Descriptors contain offsets within
+	// this file.
+	DataFD int
+
+	// EventFD is a file descriptor for the event that is signaled when
+	// data is becomes available in this queue.
+	EventFD eventfd.Eventfd
+
+	// TxPipeFD is a file descriptor for the tx pipe associated with the
+	// queue.
+	TxPipeFD int
+
+	// RxPipeFD is a file descriptor for the rx pipe associated with the
+	// queue.
+	RxPipeFD int
+
+	// SharedDataFD is a file descriptor for the file that contains shared
+	// state between the two ends of the queue. This data specifies, for
+	// example, whether EventFD signaling is enabled or disabled.
+	SharedDataFD int
+}
+
+// FDs returns the FD's in the QueueConfig as a slice of ints. This must
+// be used in conjunction with QueueConfigFromFDs to ensure the order
+// of FDs matches when reconstructing the config when serialized or sent
+// as part of control messages.
+func (q *QueueConfig) FDs() []int {
+	return []int{q.DataFD, q.EventFD.FD(), q.TxPipeFD, q.RxPipeFD, q.SharedDataFD}
+}
+
+// QueueConfigFromFDs constructs a QueueConfig out of a slice of ints where each
+// entry represents an file descriptor. The order of FDs in the slice must be in
+// the order specified below for the config to be valid. QueueConfig.FDs()
+// should be used when the config needs to be serialized or sent as part of a
+// control message to ensure the correct order.
+func QueueConfigFromFDs(fds []int) (QueueConfig, error) {
+	if len(fds) != 5 {
+		return QueueConfig{}, fmt.Errorf("insufficient number of fds: len(fds): %d, want: 5", len(fds))
+	}
+	return QueueConfig{
+		DataFD:       fds[0],
+		EventFD:      eventfd.Wrap(fds[1]),
+		TxPipeFD:     fds[2],
+		RxPipeFD:     fds[3],
+		SharedDataFD: fds[4],
+	}, nil
+}
+
+// Options specify the details about the sharedmem endpoint to be created.
+//
+// +stateify savable
+type Options struct {
+	// MTU is the mtu to use for this endpoint.
+	MTU uint32
+
+	// BufferSize is the size of each scatter/gather buffer that will hold packet
+	// data.
+	//
+	// NOTE: This directly determines number of packets that can be held in
+	// the ring buffer at any time. This does not have to be sized to the MTU as
+	// the shared memory queue design allows usage of more than one buffer to be
+	// used to make up a given packet.
+	BufferSize uint32
+
+	// LinkAddress is the link address for this endpoint (required).
+	LinkAddress tcpip.LinkAddress
+
+	// TX is the transmit queue configuration for this shared memory endpoint.
+	TX QueueConfig
+
+	// RX is the receive queue configuration for this shared memory endpoint.
+	RX QueueConfig
+
+	// PeerFD is the fd for the connected peer which can be used to detect
+	// peer disconnects.
+	PeerFD int
+
+	// OnClosed is a function that is called when the endpoint is being closed
+	// (probably due to peer going away)
+	OnClosed func(err tcpip.Error)
+
+	// TXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityTXChecksumOffload.
+	TXChecksumOffload bool
+
+	// RXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityRXChecksumOffload.
+	RXChecksumOffload bool
+
+	// VirtioNetHeaderRequired if true, indicates that all outbound packets should have
+	// a virtio header and inbound packets should have a virtio header as well.
+	VirtioNetHeaderRequired bool
+
+	// GSOMaxSize is the maximum GSO packet size. It is zero if GSO is
+	// disabled. Note that only gVisor GSO is supported, not host GSO.
+	GSOMaxSize uint32
+}
+
+var (
+	_ stack.LinkEndpoint = (*endpoint)(nil)
+	_ stack.GSOEndpoint  = (*endpoint)(nil)
+)
+
+// +stateify savable
+type endpoint struct {
+	// bufferSize is the size of each individual buffer.
+	// bufferSize is immutable.
+	bufferSize uint32
+
+	// peerFD is an fd to the peer that can be used to detect when the
+	// peer is gone.
+	// peerFD is immutable.
+	peerFD int
+
+	// caps holds the endpoint capabilities.
+	caps stack.LinkEndpointCapabilities
+
+	// hdrSize is the size of the link layer header if any.
+	// hdrSize is immutable.
+	hdrSize uint32
+
+	// gSOMaxSize is the maximum GSO packet size. It is zero if GSO is
+	// disabled. Note that only gVisor GSO is supported, not host GSO.
+	// gsoMaxSize is immutable.
+	gsoMaxSize uint32
+
+	// virtioNetHeaderRequired if true indicates that a virtio header is expected
+	// in all inbound/outbound packets.
+	virtioNetHeaderRequired bool
+
+	// rx is the receive queue.
+	rx rx
+
+	// stopRequested  determines whether the worker goroutines should stop.
+	stopRequested atomicbitops.Uint32
+
+	// Wait group used to indicate that all workers have stopped.
+	completed sync.WaitGroup
+
+	// onClosed is a function to be called when the FD's peer (if any) closes
+	// its end of the communication pipe.
+	// TODO(b/341946753): Restore when netstack is savable.
+	onClosed func(tcpip.Error) `state:"nosave"`
+
+	// mu protects the following fields.
+	mu endpointRWMutex `state:"nosave"`
+
+	// tx is the transmit queue.
+	// +checklocks:mu
+	tx tx
+
+	// workerStarted specifies whether the worker goroutine was started.
+	// +checklocks:mu
+	workerStarted bool
+
+	// addr is the local address of this endpoint.
+	//
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+	// mtu (maximum transmission unit) is the maximum size of a packet.
+	// +checklocks:mu
+	mtu uint32
+}
+
+// New creates a new shared-memory-based endpoint. Buffers will be broken up
+// into buffers of "bufferSize" bytes.
+//
+// In order to release all resources held by the returned endpoint, Close()
+// must be called followed by Wait().
+func New(opts Options) (stack.LinkEndpoint, error) {
+	e := &endpoint{
+		mtu:                     opts.MTU,
+		bufferSize:              opts.BufferSize,
+		addr:                    opts.LinkAddress,
+		peerFD:                  opts.PeerFD,
+		onClosed:                opts.OnClosed,
+		virtioNetHeaderRequired: opts.VirtioNetHeaderRequired,
+		gsoMaxSize:              opts.GSOMaxSize,
+	}
+
+	if err := e.tx.init(opts.BufferSize, &opts.TX); err != nil {
+		return nil, err
+	}
+
+	if err := e.rx.init(opts.BufferSize, &opts.RX); err != nil {
+		e.tx.cleanup()
+		return nil, err
+	}
+
+	e.caps = stack.LinkEndpointCapabilities(0)
+	if opts.RXChecksumOffload {
+		e.caps |= stack.CapabilityRXChecksumOffload
+	}
+
+	if opts.TXChecksumOffload {
+		e.caps |= stack.CapabilityTXChecksumOffload
+	}
+
+	if opts.LinkAddress != "" {
+		e.hdrSize = header.EthernetMinimumSize
+		e.caps |= stack.CapabilityResolutionRequired
+	}
+
+	if opts.VirtioNetHeaderRequired {
+		e.hdrSize += header.VirtioNetHeaderSize
+	}
+
+	return e, nil
+}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (e *endpoint) SetOnCloseAction(func()) {}
+
+// Close frees most resources associated with the endpoint. Wait() must be
+// called after Close() in order to free the rest.
+func (e *endpoint) Close() {
+	// Tell dispatch goroutine to stop, then write to the eventfd so that
+	// it wakes up in case it's sleeping.
+	if e.stopRequested.Swap(1) == 1 {
+		// It is already closed.
+		return
+	}
+	e.rx.eventFD.Notify()
+
+	// Cleanup the queues inline if the worker hasn't started yet; we also
+	// know it won't start from now on because stopRequested is set to 1.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	workerPresent := e.workerStarted
+
+	if !workerPresent {
+		e.tx.cleanup()
+		e.rx.cleanup()
+	}
+}
+
+// Wait implements stack.LinkEndpoint.Wait. It waits until all workers have
+// stopped after a Close() call.
+func (e *endpoint) Wait() {
+	e.completed.Wait()
+	e.rx.eventFD.Close()
+}
+
+// Attach implements stack.LinkEndpoint.Attach. It launches the goroutine that
+// reads packets from the rx queue.
+func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	if dispatcher == nil {
+		e.Close()
+		return
+	}
+	e.mu.Lock()
+	if !e.workerStarted && e.stopRequested.Load() == 0 {
+		e.workerStarted = true
+		e.completed.Add(1)
+
+		// Spin up a goroutine to monitor for peer shutdown.
+		if e.peerFD >= 0 {
+			e.completed.Add(1)
+			go func() {
+				defer e.completed.Done()
+				b := make([]byte, 1)
+				// When sharedmem endpoint is in use the peerFD is never used for any data
+				// transfer and this Read should only return if the peer is shutting down.
+				_, errno := rawfile.BlockingRead(e.peerFD, b)
+				if e.onClosed != nil {
+					if errno == 0 {
+						e.onClosed(nil)
+					} else {
+						e.onClosed(tcpip.TranslateErrno(errno))
+					}
+				}
+			}()
+		}
+
+		// Link endpoints are not savable. When transportation endpoints
+		// are saved, they stop sending outgoing packets and all
+		// incoming packets are rejected.
+		go e.dispatchLoop(dispatcher) // S/R-SAFE: see above.
+	}
+	e.mu.Unlock()
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *endpoint) IsAttached() bool {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.workerStarted
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *endpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+func (e *endpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.caps
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It returns the
+// ethernet frame header size.
+func (e *endpoint) MaxHeaderLength() uint16 {
+	return uint16(e.hdrSize)
+}
+
+// LinkAddress implements stack.LinkEndpoint.LinkAddress. It returns the local
+// link address.
+func (e *endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.addr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.addr = addr
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return
+	}
+
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	eth.Encode(&header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	})
+}
+
+func (e *endpoint) parseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize)
+	return ok
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return true
+	}
+
+	return e.parseHeader(pkt)
+}
+
+func (e *endpoint) AddVirtioNetHeader(pkt *stack.PacketBuffer) {
+	virtio := header.VirtioNetHeader(pkt.VirtioNetHeader().Push(header.VirtioNetHeaderSize))
+	virtio.Encode(&header.VirtioNetHeaderFields{})
+}
+
+// +checklocks:e.mu
+func (e *endpoint) writePacketLocked(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	if e.virtioNetHeaderRequired {
+		e.AddVirtioNetHeader(pkt)
+	}
+
+	// Transmit the packet.
+	b := pkt.ToBuffer()
+	defer b.Release()
+	ok := e.tx.transmit(b)
+	if !ok {
+		return &tcpip.ErrWouldBlock{}
+	}
+
+	return nil
+}
+
+// WritePackets implements stack.LinkEndpoint.WritePackets.
+func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	n := 0
+	var err tcpip.Error
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for _, pkt := range pkts.AsSlice() {
+		if err = e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil {
+			break
+		}
+		n++
+	}
+	// WritePackets never returns an error if it successfully transmitted at least
+	// one packet.
+	if err != nil && n == 0 {
+		return 0, err
+	}
+	e.tx.notify()
+	return n, nil
+}
+
+// dispatchLoop reads packets from the rx queue in a loop and dispatches them
+// to the network stack.
+func (e *endpoint) dispatchLoop(d stack.NetworkDispatcher) {
+	// Post initial set of buffers.
+	limit := e.rx.q.PostedBuffersLimit()
+	if l := uint64(len(e.rx.data)) / uint64(e.bufferSize); limit > l {
+		limit = l
+	}
+	for i := uint64(0); i < limit; i++ {
+		b := queue.RxBuffer{
+			Offset: i * uint64(e.bufferSize),
+			Size:   e.bufferSize,
+			ID:     i,
+		}
+		if !e.rx.q.PostBuffers([]queue.RxBuffer{b}) {
+			log.Warningf("Unable to post %v-th buffer", i)
+		}
+	}
+
+	// Read in a loop until a stop is requested.
+	var rxb []queue.RxBuffer
+	for e.stopRequested.Load() == 0 {
+		var n uint32
+		rxb, n = e.rx.postAndReceive(rxb, &e.stopRequested)
+
+		// Copy data from the shared area to its own buffer, then
+		// prepare to repost the buffer.
+		v := buffer.NewView(int(n))
+		v.Grow(int(n))
+		offset := uint32(0)
+		for i := range rxb {
+			v.WriteAt(e.rx.data[rxb[i].Offset:][:rxb[i].Size], int(offset))
+			offset += rxb[i].Size
+
+			rxb[i].Size = e.bufferSize
+		}
+
+		pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: buffer.MakeWithView(v),
+		})
+
+		if e.virtioNetHeaderRequired {
+			_, ok := pkt.VirtioNetHeader().Consume(header.VirtioNetHeaderSize)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+		}
+
+		var proto tcpip.NetworkProtocolNumber
+		e.mu.RLock()
+		addrLen := len(e.addr)
+		e.mu.RUnlock()
+		if addrLen != 0 {
+			if !e.parseHeader(pkt) {
+				pkt.DecRef()
+				continue
+			}
+			proto = header.Ethernet(pkt.LinkHeader().Slice()).Type()
+		} else {
+			// We don't get any indication of what the packet is, so try to guess
+			// if it's an IPv4 or IPv6 packet.
+			// IP version information is at the first octet, so pulling up 1 byte.
+			h, ok := pkt.Data().PullUp(1)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+			switch header.IPVersion(h) {
+			case header.IPv4Version:
+				proto = header.IPv4ProtocolNumber
+			case header.IPv6Version:
+				proto = header.IPv6ProtocolNumber
+			default:
+				pkt.DecRef()
+				continue
+			}
+		}
+		// Send packet up the stack.
+		d.DeliverNetworkPacket(proto, pkt)
+		pkt.DecRef()
+	}
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	// Clean state.
+	e.tx.cleanup()
+	e.rx.cleanup()
+
+	e.completed.Done()
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType
+func (*endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareEther
+}
+
+// GSOMaxSize implements stack.GSOEndpoint.
+func (e *endpoint) GSOMaxSize() uint32 {
+	return e.gsoMaxSize
+}
+
+// SupportsGSO implements stack.GSOEndpoint.
+func (e *endpoint) SupportedGSO() stack.SupportedGSO {
+	return stack.GVisorGSOSupported
+}

pkg/tcpip/link/sharedmem/sharedmem_server.go

@@ -0,0 +1,399 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// +stateify savable
+type serverEndpoint struct {
+	// bufferSize is the size of each individual buffer.
+	// bufferSize is immutable.
+	bufferSize uint32
+
+	// rx is the receive queue.
+	rx serverRx
+
+	// stopRequested determines whether the worker goroutines should stop.
+	stopRequested atomicbitops.Uint32
+
+	// Wait group used to indicate that all workers have stopped.
+	completed sync.WaitGroup `state:"nosave"`
+
+	// peerFD is an fd to the peer that can be used to detect when the peer is
+	// gone.
+	// peerFD is immutable.
+	peerFD int
+
+	// caps holds the endpoint capabilities.
+	caps stack.LinkEndpointCapabilities
+
+	// hdrSize is the size of the link layer header if any.
+	// hdrSize is immutable.
+	hdrSize uint32
+
+	// virtioNetHeaderRequired if true indicates that a virtio header is expected
+	// in all inbound/outbound packets.
+	virtioNetHeaderRequired bool
+
+	// onClosed is a function to be called when the FD's peer (if any) closes its
+	// end of the communication pipe.
+	onClosed func(tcpip.Error) `state:"nosave"`
+
+	// mu protects the following fields.
+	mu serverEndpointRWMutex `state:"nosave"`
+
+	// tx is the transmit queue.
+	// +checklocks:mu
+	tx serverTx
+
+	// workerStarted specifies whether the worker goroutine was started.
+	// +checklocks:mu
+	workerStarted bool
+
+	// addr is the local address of this endpoint.
+	//
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+	// mtu (maximum transmission unit) is the maximum size of a packet.
+	// +checklocks:mu
+	mtu uint32
+}
+
+// NewServerEndpoint creates a new shared-memory-based endpoint. Buffers will be
+// broken up into buffers of "bufferSize" bytes.
+func NewServerEndpoint(opts Options) (stack.LinkEndpoint, error) {
+	e := &serverEndpoint{
+		mtu:        opts.MTU,
+		bufferSize: opts.BufferSize,
+		addr:       opts.LinkAddress,
+		peerFD:     opts.PeerFD,
+		onClosed:   opts.OnClosed,
+	}
+
+	if err := e.tx.init(&opts.RX); err != nil {
+		return nil, err
+	}
+
+	if err := e.rx.init(&opts.TX); err != nil {
+		e.tx.cleanup()
+		return nil, err
+	}
+
+	e.caps = stack.LinkEndpointCapabilities(0)
+	if opts.RXChecksumOffload {
+		e.caps |= stack.CapabilityRXChecksumOffload
+	}
+
+	if opts.TXChecksumOffload {
+		e.caps |= stack.CapabilityTXChecksumOffload
+	}
+
+	if opts.LinkAddress != "" {
+		e.hdrSize = header.EthernetMinimumSize
+		e.caps |= stack.CapabilityResolutionRequired
+	}
+
+	return e, nil
+}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (*serverEndpoint) SetOnCloseAction(func()) {}
+
+// Close frees all resources associated with the endpoint.
+func (e *serverEndpoint) Close() {
+	// Tell dispatch goroutine to stop, then write to the eventfd so that it wakes
+	// up in case it's sleeping.
+	e.stopRequested.Store(1)
+	e.rx.eventFD.Notify()
+
+	// Cleanup the queues inline if the worker hasn't started yet; we also know it
+	// won't start from now on because stopRequested is set to 1.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	workerPresent := e.workerStarted
+
+	if !workerPresent {
+		e.tx.cleanup()
+		e.rx.cleanup()
+	}
+}
+
+// Wait implements stack.LinkEndpoint.Wait. It waits until all workers have
+// stopped after a Close() call.
+func (e *serverEndpoint) Wait() {
+	e.completed.Wait()
+}
+
+// Attach implements stack.LinkEndpoint.Attach. It launches the goroutine that
+// reads packets from the rx queue.
+func (e *serverEndpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	if !e.workerStarted && e.stopRequested.Load() == 0 {
+		e.workerStarted = true
+		e.completed.Add(1)
+		if e.peerFD >= 0 {
+			e.completed.Add(1)
+			// Spin up a goroutine to monitor for peer shutdown.
+			go func() {
+				b := make([]byte, 1)
+				// When sharedmem endpoint is in use the peerFD is never used for any
+				// data transfer and this Read should only return if the peer is
+				// shutting down.
+				_, errno := rawfile.BlockingRead(e.peerFD, b)
+				if e.onClosed != nil {
+					if errno == 0 {
+						e.onClosed(nil)
+					} else {
+						e.onClosed(tcpip.TranslateErrno(errno))
+					}
+				}
+				e.completed.Done()
+			}()
+		}
+		// Link endpoints are not savable. When transportation endpoints are saved,
+		// they stop sending outgoing packets and all incoming packets are rejected.
+		go e.dispatchLoop(dispatcher) // S/R-SAFE: see above.
+	}
+	e.mu.Unlock()
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *serverEndpoint) IsAttached() bool {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.workerStarted
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *serverEndpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+func (e *serverEndpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *serverEndpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.caps
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It returns the
+// ethernet frame header size.
+func (e *serverEndpoint) MaxHeaderLength() uint16 {
+	return uint16(e.hdrSize)
+}
+
+// LinkAddress implements stack.LinkEndpoint.LinkAddress. It returns the local
+// link address.
+func (e *serverEndpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.addr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *serverEndpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.addr = addr
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *serverEndpoint) AddHeader(pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return
+	}
+
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	eth.Encode(&header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	})
+}
+
+func (e *serverEndpoint) parseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize)
+	return ok
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *serverEndpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return true
+	}
+
+	return e.parseHeader(pkt)
+}
+
+func (e *serverEndpoint) AddVirtioNetHeader(pkt *stack.PacketBuffer) {
+	virtio := header.VirtioNetHeader(pkt.VirtioNetHeader().Push(header.VirtioNetHeaderSize))
+	virtio.Encode(&header.VirtioNetHeaderFields{})
+}
+
+// +checklocks:e.mu
+func (e *serverEndpoint) writePacketLocked(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	if e.virtioNetHeaderRequired {
+		e.AddVirtioNetHeader(pkt)
+	}
+
+	ok := e.tx.transmit(pkt)
+	if !ok {
+		return &tcpip.ErrWouldBlock{}
+	}
+
+	return nil
+}
+
+// WritePacket writes outbound packets to the file descriptor. If it is not
+// currently writable, the packet is dropped.
+// WritePacket implements stack.LinkEndpoint.WritePacket.
+func (e *serverEndpoint) WritePacket(_ stack.RouteInfo, _ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	// Transmit the packet.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if err := e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil {
+		return err
+	}
+	e.tx.notify()
+	return nil
+}
+
+// WritePackets implements stack.LinkEndpoint.WritePackets.
+func (e *serverEndpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	n := 0
+	var err tcpip.Error
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for _, pkt := range pkts.AsSlice() {
+		if err = e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil {
+			break
+		}
+		n++
+	}
+	// WritePackets never returns an error if it successfully transmitted at least
+	// one packet.
+	if err != nil && n == 0 {
+		return 0, err
+	}
+	e.tx.notify()
+	return n, nil
+}
+
+// dispatchLoop reads packets from the rx queue in a loop and dispatches them
+// to the network stack.
+func (e *serverEndpoint) dispatchLoop(d stack.NetworkDispatcher) {
+	for e.stopRequested.Load() == 0 {
+		b := e.rx.receive()
+		if b == nil {
+			e.rx.EnableNotification()
+			// Now pull again to make sure we didn't receive any packets
+			// while notifications were not enabled.
+			for {
+				b = e.rx.receive()
+				if b != nil {
+					// Disable notifications as we only need to be notified when we are going
+					// to block on eventFD. This should prevent the peer from needlessly
+					// writing to eventFD when this end is already awake and processing
+					// packets.
+					e.rx.DisableNotification()
+					break
+				}
+				e.rx.waitForPackets()
+			}
+		}
+		pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: buffer.MakeWithView(b),
+		})
+		if e.virtioNetHeaderRequired {
+			_, ok := pkt.VirtioNetHeader().Consume(header.VirtioNetHeaderSize)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+		}
+		var proto tcpip.NetworkProtocolNumber
+		e.mu.RLock()
+		addrLen := len(e.addr)
+		e.mu.RUnlock()
+		if addrLen != 0 {
+			if !e.parseHeader(pkt) {
+				pkt.DecRef()
+				continue
+			}
+			proto = header.Ethernet(pkt.LinkHeader().Slice()).Type()
+		} else {
+			// We don't get any indication of what the packet is, so try to guess
+			// if it's an IPv4 or IPv6 packet.
+			// IP version information is at the first octet, so pulling up 1 byte.
+			h, ok := pkt.Data().PullUp(1)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+			switch header.IPVersion(h) {
+			case header.IPv4Version:
+				proto = header.IPv4ProtocolNumber
+			case header.IPv6Version:
+				proto = header.IPv6ProtocolNumber
+			default:
+				pkt.DecRef()
+				continue
+			}
+		}
+		// Send packet up the stack.
+		d.DeliverNetworkPacket(proto, pkt)
+		pkt.DecRef()
+	}
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	// Clean state.
+	e.tx.cleanup()
+	e.rx.cleanup()
+
+	e.completed.Done()
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType
+func (e *serverEndpoint) ARPHardwareType() header.ARPHardwareType {
+	if e.hdrSize > 0 {
+		return header.ARPHardwareEther
+	}
+	return header.ARPHardwareNone
+}

pkg/tcpip/link/sharedmem/sharedmem_state_autogen.go

@@ -0,0 +1,309 @@
+// automatically generated by stateify.
+
+//go:build linux && linux && linux && linux && linux && linux
+// +build linux,linux,linux,linux,linux,linux
+
+package sharedmem
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (s *serverRx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.serverRx"
+}
+
+func (s *serverRx) StateFields() []string {
+	return []string{
+		"packetPipe",
+		"completionPipe",
+		"data",
+		"eventFD",
+		"sharedData",
+		"sharedEventFDState",
+	}
+}
+
+func (s *serverRx) beforeSave() {}
+
+// +checklocksignore
+func (s *serverRx) StateSave(stateSinkObject state.Sink) {
+	s.beforeSave()
+	stateSinkObject.Save(0, &s.packetPipe)
+	stateSinkObject.Save(1, &s.completionPipe)
+	stateSinkObject.Save(2, &s.data)
+	stateSinkObject.Save(3, &s.eventFD)
+	stateSinkObject.Save(4, &s.sharedData)
+	stateSinkObject.Save(5, &s.sharedEventFDState)
+}
+
+func (s *serverRx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (s *serverRx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &s.packetPipe)
+	stateSourceObject.Load(1, &s.completionPipe)
+	stateSourceObject.Load(2, &s.data)
+	stateSourceObject.Load(3, &s.eventFD)
+	stateSourceObject.Load(4, &s.sharedData)
+	stateSourceObject.Load(5, &s.sharedEventFDState)
+}
+
+func (s *serverTx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.serverTx"
+}
+
+func (s *serverTx) StateFields() []string {
+	return []string{
+		"fillPipe",
+		"completionPipe",
+		"data",
+		"eventFD",
+		"sharedData",
+		"sharedEventFDState",
+	}
+}
+
+func (s *serverTx) beforeSave() {}
+
+// +checklocksignore
+func (s *serverTx) StateSave(stateSinkObject state.Sink) {
+	s.beforeSave()
+	stateSinkObject.Save(0, &s.fillPipe)
+	stateSinkObject.Save(1, &s.completionPipe)
+	stateSinkObject.Save(2, &s.data)
+	stateSinkObject.Save(3, &s.eventFD)
+	stateSinkObject.Save(4, &s.sharedData)
+	stateSinkObject.Save(5, &s.sharedEventFDState)
+}
+
+func (s *serverTx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (s *serverTx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &s.fillPipe)
+	stateSourceObject.Load(1, &s.completionPipe)
+	stateSourceObject.Load(2, &s.data)
+	stateSourceObject.Load(3, &s.eventFD)
+	stateSourceObject.Load(4, &s.sharedData)
+	stateSourceObject.Load(5, &s.sharedEventFDState)
+}
+
+func (q *QueueConfig) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.QueueConfig"
+}
+
+func (q *QueueConfig) StateFields() []string {
+	return []string{
+		"DataFD",
+		"EventFD",
+		"TxPipeFD",
+		"RxPipeFD",
+		"SharedDataFD",
+	}
+}
+
+func (q *QueueConfig) beforeSave() {}
+
+// +checklocksignore
+func (q *QueueConfig) StateSave(stateSinkObject state.Sink) {
+	q.beforeSave()
+	stateSinkObject.Save(0, &q.DataFD)
+	stateSinkObject.Save(1, &q.EventFD)
+	stateSinkObject.Save(2, &q.TxPipeFD)
+	stateSinkObject.Save(3, &q.RxPipeFD)
+	stateSinkObject.Save(4, &q.SharedDataFD)
+}
+
+func (q *QueueConfig) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (q *QueueConfig) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &q.DataFD)
+	stateSourceObject.Load(1, &q.EventFD)
+	stateSourceObject.Load(2, &q.TxPipeFD)
+	stateSourceObject.Load(3, &q.RxPipeFD)
+	stateSourceObject.Load(4, &q.SharedDataFD)
+}
+
+func (o *Options) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.Options"
+}
+
+func (o *Options) StateFields() []string {
+	return []string{
+		"MTU",
+		"BufferSize",
+		"LinkAddress",
+		"TX",
+		"RX",
+		"PeerFD",
+		"OnClosed",
+		"TXChecksumOffload",
+		"RXChecksumOffload",
+		"VirtioNetHeaderRequired",
+		"GSOMaxSize",
+	}
+}
+
+func (o *Options) beforeSave() {}
+
+// +checklocksignore
+func (o *Options) StateSave(stateSinkObject state.Sink) {
+	o.beforeSave()
+	stateSinkObject.Save(0, &o.MTU)
+	stateSinkObject.Save(1, &o.BufferSize)
+	stateSinkObject.Save(2, &o.LinkAddress)
+	stateSinkObject.Save(3, &o.TX)
+	stateSinkObject.Save(4, &o.RX)
+	stateSinkObject.Save(5, &o.PeerFD)
+	stateSinkObject.Save(6, &o.OnClosed)
+	stateSinkObject.Save(7, &o.TXChecksumOffload)
+	stateSinkObject.Save(8, &o.RXChecksumOffload)
+	stateSinkObject.Save(9, &o.VirtioNetHeaderRequired)
+	stateSinkObject.Save(10, &o.GSOMaxSize)
+}
+
+func (o *Options) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &o.MTU)
+	stateSourceObject.Load(1, &o.BufferSize)
+	stateSourceObject.Load(2, &o.LinkAddress)
+	stateSourceObject.Load(3, &o.TX)
+	stateSourceObject.Load(4, &o.RX)
+	stateSourceObject.Load(5, &o.PeerFD)
+	stateSourceObject.Load(6, &o.OnClosed)
+	stateSourceObject.Load(7, &o.TXChecksumOffload)
+	stateSourceObject.Load(8, &o.RXChecksumOffload)
+	stateSourceObject.Load(9, &o.VirtioNetHeaderRequired)
+	stateSourceObject.Load(10, &o.GSOMaxSize)
+}
+
+func (e *endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.endpoint"
+}
+
+func (e *endpoint) StateFields() []string {
+	return []string{
+		"bufferSize",
+		"peerFD",
+		"caps",
+		"hdrSize",
+		"gsoMaxSize",
+		"virtioNetHeaderRequired",
+		"rx",
+		"stopRequested",
+		"completed",
+		"tx",
+		"workerStarted",
+		"addr",
+		"mtu",
+	}
+}
+
+func (e *endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.bufferSize)
+	stateSinkObject.Save(1, &e.peerFD)
+	stateSinkObject.Save(2, &e.caps)
+	stateSinkObject.Save(3, &e.hdrSize)
+	stateSinkObject.Save(4, &e.gsoMaxSize)
+	stateSinkObject.Save(5, &e.virtioNetHeaderRequired)
+	stateSinkObject.Save(6, &e.rx)
+	stateSinkObject.Save(7, &e.stopRequested)
+	stateSinkObject.Save(8, &e.completed)
+	stateSinkObject.Save(9, &e.tx)
+	stateSinkObject.Save(10, &e.workerStarted)
+	stateSinkObject.Save(11, &e.addr)
+	stateSinkObject.Save(12, &e.mtu)
+}
+
+func (e *endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.bufferSize)
+	stateSourceObject.Load(1, &e.peerFD)
+	stateSourceObject.Load(2, &e.caps)
+	stateSourceObject.Load(3, &e.hdrSize)
+	stateSourceObject.Load(4, &e.gsoMaxSize)
+	stateSourceObject.Load(5, &e.virtioNetHeaderRequired)
+	stateSourceObject.Load(6, &e.rx)
+	stateSourceObject.Load(7, &e.stopRequested)
+	stateSourceObject.Load(8, &e.completed)
+	stateSourceObject.Load(9, &e.tx)
+	stateSourceObject.Load(10, &e.workerStarted)
+	stateSourceObject.Load(11, &e.addr)
+	stateSourceObject.Load(12, &e.mtu)
+}
+
+func (e *serverEndpoint) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.serverEndpoint"
+}
+
+func (e *serverEndpoint) StateFields() []string {
+	return []string{
+		"bufferSize",
+		"rx",
+		"stopRequested",
+		"peerFD",
+		"caps",
+		"hdrSize",
+		"virtioNetHeaderRequired",
+		"tx",
+		"workerStarted",
+		"addr",
+		"mtu",
+	}
+}
+
+func (e *serverEndpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *serverEndpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.bufferSize)
+	stateSinkObject.Save(1, &e.rx)
+	stateSinkObject.Save(2, &e.stopRequested)
+	stateSinkObject.Save(3, &e.peerFD)
+	stateSinkObject.Save(4, &e.caps)
+	stateSinkObject.Save(5, &e.hdrSize)
+	stateSinkObject.Save(6, &e.virtioNetHeaderRequired)
+	stateSinkObject.Save(7, &e.tx)
+	stateSinkObject.Save(8, &e.workerStarted)
+	stateSinkObject.Save(9, &e.addr)
+	stateSinkObject.Save(10, &e.mtu)
+}
+
+func (e *serverEndpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *serverEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.bufferSize)
+	stateSourceObject.Load(1, &e.rx)
+	stateSourceObject.Load(2, &e.stopRequested)
+	stateSourceObject.Load(3, &e.peerFD)
+	stateSourceObject.Load(4, &e.caps)
+	stateSourceObject.Load(5, &e.hdrSize)
+	stateSourceObject.Load(6, &e.virtioNetHeaderRequired)
+	stateSourceObject.Load(7, &e.tx)
+	stateSourceObject.Load(8, &e.workerStarted)
+	stateSourceObject.Load(9, &e.addr)
+	stateSourceObject.Load(10, &e.mtu)
+}
+
+func init() {
+	state.Register((*serverRx)(nil))
+	state.Register((*serverTx)(nil))
+	state.Register((*QueueConfig)(nil))
+	state.Register((*Options)(nil))
+	state.Register((*endpoint)(nil))
+	state.Register((*serverEndpoint)(nil))
+}

pkg/tcpip/link/sharedmem/sharedmem_unsafe.go

@@ -0,0 +1,59 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sharedmem
+
+import (
+	"fmt"
+	"reflect"
+	"unsafe"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/memutil"
+	"golang.org/x/sys/unix"
+)
+
+// sharedDataPointer converts the shared data slice into a pointer so that it
+// can be used in atomic operations.
+func sharedDataPointer(sharedData []byte) *atomicbitops.Uint32 {
+	return (*atomicbitops.Uint32)(unsafe.Pointer(&sharedData[0:4][0]))
+}
+
+// getBuffer returns a memory region mapped to the full contents of the given
+// file descriptor.
+func getBuffer(fd int) ([]byte, error) {
+	var s unix.Stat_t
+	if err := unix.Fstat(fd, &s); err != nil {
+		return nil, err
+	}
+
+	// Check that size doesn't overflow an int.
+	if s.Size > int64(^uint(0)>>1) {
+		return nil, unix.EDOM
+	}
+
+	addr, err := memutil.MapFile(0 /* addr */, uintptr(s.Size), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_FILE, uintptr(fd), 0 /*offset*/)
+	if err != nil {
+		return nil, fmt.Errorf("failed to map memory for buffer fd: %d, error: %s", fd, err)
+	}
+
+	// Use unsafe to convert addr into a []byte.
+	var b []byte
+	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	hdr.Data = addr
+	hdr.Len = int(s.Size)
+	hdr.Cap = int(s.Size)
+
+	return b, nil
+}

pkg/tcpip/link/sharedmem/sharedmem_unsafe_state_autogen.go

@@ -0,0 +1,3 @@
+// automatically generated by stateify.
+
+package sharedmem

pkg/tcpip/link/sharedmem/tx.go

@@ -0,0 +1,279 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sharedmem
+
+import (
+	"math"
+
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	nilID = math.MaxUint64
+)
+
+// tx holds all state associated with a tx queue.
+type tx struct {
+	data         []byte
+	q            queue.Tx
+	ids          idManager
+	bufs         bufferManager
+	eventFD      eventfd.Eventfd
+	sharedData   []byte
+	sharedDataFD int
+}
+
+// init initializes all state needed by the tx queue based on the information
+// provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (t *tx) init(bufferSize uint32, c *QueueConfig) error {
+	// Map in all buffers.
+	txPipe, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+
+	rxPipe, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		return err
+	}
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		return err
+	}
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		unix.Munmap(data)
+	}
+
+	// Initialize state based on buffers.
+	t.q.Init(txPipe, rxPipe, sharedDataPointer(sharedData))
+	t.ids.init()
+	t.bufs.init(0, len(data), int(bufferSize))
+	t.data = data
+	t.eventFD = c.EventFD
+	t.sharedDataFD = c.SharedDataFD
+	t.sharedData = sharedData
+
+	return nil
+}
+
+// cleanup releases all resources allocated during init(). It must only be
+// called if init() has previously succeeded.
+func (t *tx) cleanup() {
+	a, b := t.q.Bytes()
+	unix.Munmap(a)
+	unix.Munmap(b)
+	unix.Munmap(t.data)
+}
+
+// transmit sends a packet made of bufs. Returns a boolean that specifies
+// whether the packet was successfully transmitted.
+func (t *tx) transmit(transmitBuf buffer.Buffer) bool {
+	// Pull completions from the tx queue and add their buffers back to the
+	// pool so that we can reuse them.
+	for {
+		id, ok := t.q.CompletedPacket()
+		if !ok {
+			break
+		}
+
+		if buf := t.ids.remove(id); buf != nil {
+			t.bufs.free(buf)
+		}
+	}
+
+	bSize := t.bufs.entrySize
+	total := uint32(transmitBuf.Size())
+	bufCount := (total + bSize - 1) / bSize
+
+	// Allocate enough buffers to hold all the data.
+	var buf *queue.TxBuffer
+	for i := bufCount; i != 0; i-- {
+		b := t.bufs.alloc()
+		if b == nil {
+			// Failed to get all buffers. Return to the pool
+			// whatever we had managed to get.
+			if buf != nil {
+				t.bufs.free(buf)
+			}
+			return false
+		}
+		b.Next = buf
+		buf = b
+	}
+
+	// Copy data into allocated buffers.
+	nBuf := buf
+	var dBuf []byte
+	transmitBuf.Apply(func(v *buffer.View) {
+		for v.Size() > 0 {
+			if len(dBuf) == 0 {
+				dBuf = t.data[nBuf.Offset:][:nBuf.Size]
+				nBuf = nBuf.Next
+			}
+			n := copy(dBuf, v.AsSlice())
+			v.TrimFront(n)
+			dBuf = dBuf[n:]
+		}
+	})
+
+	// Get an id for this packet and send it out.
+	id := t.ids.add(buf)
+	if !t.q.Enqueue(id, total, bufCount, buf) {
+		t.ids.remove(id)
+		t.bufs.free(buf)
+		return false
+	}
+
+	return true
+}
+
+// notify writes to the tx.eventFD to indicate to the peer that there is data to
+// be read.
+func (t *tx) notify() {
+	if t.q.NotificationsEnabled() {
+		t.eventFD.Notify()
+	}
+}
+
+// idDescriptor is used by idManager to either point to a tx buffer (in case
+// the ID is assigned) or to the next free element (if the id is not assigned).
+type idDescriptor struct {
+	buf      *queue.TxBuffer
+	nextFree uint64
+}
+
+// idManager is a manager of tx buffer identifiers. It assigns unique IDs to
+// tx buffers that are added to it; the IDs can only be reused after they have
+// been removed.
+//
+// The ID assignments are stored so that the tx buffers can be retrieved from
+// the IDs previously assigned to them.
+type idManager struct {
+	// ids is a slice containing all tx buffers. The ID is the index into
+	// this slice.
+	ids []idDescriptor
+
+	// freeList a list of free IDs.
+	freeList uint64
+}
+
+// init initializes the id manager.
+func (m *idManager) init() {
+	m.freeList = nilID
+}
+
+// add assigns an ID to the given tx buffer.
+func (m *idManager) add(b *queue.TxBuffer) uint64 {
+	if i := m.freeList; i != nilID {
+		// There is an id available in the free list, just use it.
+		m.ids[i].buf = b
+		m.freeList = m.ids[i].nextFree
+		return i
+	}
+
+	// We need to expand the id descriptor.
+	m.ids = append(m.ids, idDescriptor{buf: b})
+	return uint64(len(m.ids) - 1)
+}
+
+// remove retrieves the tx buffer associated with the given ID, and removes the
+// ID from the assigned table so that it can be reused in the future.
+func (m *idManager) remove(i uint64) *queue.TxBuffer {
+	if i >= uint64(len(m.ids)) {
+		return nil
+	}
+
+	desc := &m.ids[i]
+	b := desc.buf
+	if b == nil {
+		// The provided id is not currently assigned.
+		return nil
+	}
+
+	desc.buf = nil
+	desc.nextFree = m.freeList
+	m.freeList = i
+
+	return b
+}
+
+// bufferManager manages a buffer region broken up into smaller, equally sized
+// buffers. Smaller buffers can be allocated and freed.
+type bufferManager struct {
+	freeList  *queue.TxBuffer
+	curOffset uint64
+	limit     uint64
+	entrySize uint32
+}
+
+// init initializes the buffer manager.
+func (b *bufferManager) init(initialOffset, size, entrySize int) {
+	b.freeList = nil
+	b.curOffset = uint64(initialOffset)
+	b.limit = uint64(initialOffset + size/entrySize*entrySize)
+	b.entrySize = uint32(entrySize)
+}
+
+// alloc allocates a buffer from the manager, if one is available.
+func (b *bufferManager) alloc() *queue.TxBuffer {
+	if b.freeList != nil {
+		// There is a descriptor ready for reuse in the free list.
+		d := b.freeList
+		b.freeList = d.Next
+		d.Next = nil
+		return d
+	}
+
+	if b.curOffset < b.limit {
+		// There is room available in the never-used range, so create
+		// a new descriptor for it.
+		d := &queue.TxBuffer{
+			Offset: b.curOffset,
+			Size:   b.entrySize,
+		}
+		b.curOffset += uint64(b.entrySize)
+		return d
+	}
+
+	return nil
+}
+
+// free returns all buffers in the list to the buffer manager so that they can
+// be reused.
+func (b *bufferManager) free(d *queue.TxBuffer) {
+	// Find the last buffer in the list.
+	last := d
+	for last.Next != nil {
+		last = last.Next
+	}
+
+	// Push list onto free list.
+	last.Next = b.freeList
+	b.freeList = d
+}