snapshot: sagernet/gvisor v0.0.0-20250811.0-sing-box-mod.1

Содержимое пина, зафиксированного в go.mod sing-box-lx, одним коммитом
без истории. Полная история SagerNet/gvisor — 1.45 ГБ и клонируется в
каждой CI-джобе; наша дельта — одна вставка в одну функцию, история для
неё не нужна.

Module path github.com/sagernet/gvisor сохранён намеренно: на него
опирается replace-директива суперпроекта.

Патч поверх — отдельным коммитом, чтобы дельта читалась одним git show
и переносилась на новый пин копированием.

SPECS/TASKS/048-GVISOR_HANDSHAKE_NIL_CRASH

pkg/tcpip/link/channel/channel.go

@@ -0,0 +1,321 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package channel provides the implementation of channel-based data-link layer
+// endpoints. Such endpoints allow injection of inbound packets and store
+// outbound packets in a channel.
+package channel
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// Notification is the interface for receiving notification from the packet
+// queue.
+type Notification interface {
+	// WriteNotify will be called when a write happens to the queue.
+	WriteNotify()
+}
+
+// NotificationHandle is an opaque handle to the registered notification target.
+// It can be used to unregister the notification when no longer interested.
+//
+// +stateify savable
+type NotificationHandle struct {
+	n Notification
+}
+
+type queue struct {
+	// c is the outbound packet channel.
+	c  chan *stack.PacketBuffer
+	mu queueRWMutex
+	// +checklocks:mu
+	notify []*NotificationHandle
+	// +checklocks:mu
+	closed bool
+}
+
+func (q *queue) Close() {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	if !q.closed {
+		close(q.c)
+	}
+	q.closed = true
+}
+
+func (q *queue) Read() *stack.PacketBuffer {
+	select {
+	case p := <-q.c:
+		return p
+	default:
+		return nil
+	}
+}
+
+func (q *queue) ReadContext(ctx context.Context) *stack.PacketBuffer {
+	select {
+	case pkt := <-q.c:
+		return pkt
+	case <-ctx.Done():
+		return nil
+	}
+}
+
+func (q *queue) Write(pkt *stack.PacketBuffer) tcpip.Error {
+	// q holds the PacketBuffer.
+	q.mu.RLock()
+	if q.closed {
+		q.mu.RUnlock()
+		return &tcpip.ErrClosedForSend{}
+	}
+
+	wrote := false
+	p := pkt.Clone()
+	select {
+	case q.c <- p:
+		wrote = true
+	default:
+		p.DecRef()
+	}
+	notify := q.notify
+	q.mu.RUnlock()
+
+	if wrote {
+		// Send notification outside of lock.
+		for _, h := range notify {
+			h.n.WriteNotify()
+		}
+		return nil
+	}
+	return &tcpip.ErrNoBufferSpace{}
+}
+
+func (q *queue) Num() int {
+	return len(q.c)
+}
+
+func (q *queue) AddNotify(notify Notification) *NotificationHandle {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	h := &NotificationHandle{n: notify}
+	q.notify = append(q.notify, h)
+	return h
+}
+
+func (q *queue) RemoveNotify(handle *NotificationHandle) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	// Make a copy, since we reads the array outside of lock when notifying.
+	notify := make([]*NotificationHandle, 0, len(q.notify))
+	for _, h := range q.notify {
+		if h != handle {
+			notify = append(notify, h)
+		}
+	}
+	q.notify = notify
+}
+
+var (
+	_ stack.LinkEndpoint = (*Endpoint)(nil)
+	_ stack.GSOEndpoint  = (*Endpoint)(nil)
+)
+
+// Endpoint is link layer endpoint that stores outbound packets in a channel
+// and allows injection of inbound packets.
+//
+// +stateify savable
+type Endpoint struct {
+	LinkEPCapabilities stack.LinkEndpointCapabilities
+	SupportedGSOKind   stack.SupportedGSO
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+	// +checklocks:mu
+	linkAddr tcpip.LinkAddress
+	// +checklocks:mu
+	mtu uint32
+
+	// Outbound packet queue.
+	q *queue
+}
+
+// New creates a new channel endpoint.
+func New(size int, mtu uint32, linkAddr tcpip.LinkAddress) *Endpoint {
+	return &Endpoint{
+		q: &queue{
+			c: make(chan *stack.PacketBuffer, size),
+		},
+		mtu:      mtu,
+		linkAddr: linkAddr,
+	}
+}
+
+// Close closes e. Further packet injections will return an error, and all pending
+// packets are discarded. Close may be called concurrently with WritePackets.
+func (e *Endpoint) Close() {
+	e.q.Close()
+	e.Drain()
+}
+
+// Read does non-blocking read one packet from the outbound packet queue.
+func (e *Endpoint) Read() *stack.PacketBuffer {
+	return e.q.Read()
+}
+
+// ReadContext does blocking read for one packet from the outbound packet queue.
+// It can be cancelled by ctx, and in this case, it returns nil.
+func (e *Endpoint) ReadContext(ctx context.Context) *stack.PacketBuffer {
+	return e.q.ReadContext(ctx)
+}
+
+// Drain removes all outbound packets from the channel and counts them.
+func (e *Endpoint) Drain() int {
+	c := 0
+	for pkt := e.Read(); pkt != nil; pkt = e.Read() {
+		pkt.DecRef()
+		c++
+	}
+	return c
+}
+
+// NumQueued returns the number of packet queued for outbound.
+func (e *Endpoint) NumQueued() int {
+	return e.q.Num()
+}
+
+// InjectInbound injects an inbound packet. If the endpoint is not attached, the
+// packet is not delivered.
+func (e *Endpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverNetworkPacket(protocol, pkt)
+	}
+}
+
+// Attach saves the stack network-layer dispatcher for use later when packets
+// are injected.
+func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dispatcher = dispatcher
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *Endpoint) IsAttached() bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.dispatcher != nil
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *Endpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+// SetMTU implements stack.LinkEndpoint.SetMTU.
+func (e *Endpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.LinkEPCapabilities
+}
+
+// GSOMaxSize implements stack.GSOEndpoint.
+func (*Endpoint) GSOMaxSize() uint32 {
+	return 1 << 15
+}
+
+// SupportedGSO implements stack.GSOEndpoint.
+func (e *Endpoint) SupportedGSO() stack.SupportedGSO {
+	return e.SupportedGSOKind
+}
+
+// MaxHeaderLength returns the maximum size of the link layer header. Given it
+// doesn't have a header, it just returns 0.
+func (*Endpoint) MaxHeaderLength() uint16 {
+	return 0
+}
+
+// LinkAddress returns the link address of this endpoint.
+func (e *Endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.linkAddr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.linkAddr = addr
+}
+
+// WritePackets stores outbound packets into the channel.
+// Multiple concurrent calls are permitted.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	n := 0
+	for _, pkt := range pkts.AsSlice() {
+		if err := e.q.Write(pkt); err != nil {
+			if _, ok := err.(*tcpip.ErrNoBufferSpace); !ok && n == 0 {
+				return 0, err
+			}
+			break
+		}
+		n++
+	}
+
+	return n, nil
+}
+
+// Wait implements stack.LinkEndpoint.Wait.
+func (*Endpoint) Wait() {}
+
+// AddNotify adds a notification target for receiving event about outgoing
+// packets.
+func (e *Endpoint) AddNotify(notify Notification) *NotificationHandle {
+	return e.q.AddNotify(notify)
+}
+
+// RemoveNotify removes handle from the list of notification targets.
+func (e *Endpoint) RemoveNotify(handle *NotificationHandle) {
+	e.q.RemoveNotify(handle)
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (*Endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareNone
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (*Endpoint) AddHeader(*stack.PacketBuffer) {}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (*Endpoint) ParseHeader(*stack.PacketBuffer) bool { return true }
+
+// SetOnCloseAction implements stack.LinkEndpoint.
+func (*Endpoint) SetOnCloseAction(func()) {}

pkg/tcpip/link/channel/channel_state_autogen.go

@@ -0,0 +1,79 @@
+// automatically generated by stateify.
+
+package channel
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (n *NotificationHandle) StateTypeName() string {
+	return "pkg/tcpip/link/channel.NotificationHandle"
+}
+
+func (n *NotificationHandle) StateFields() []string {
+	return []string{
+		"n",
+	}
+}
+
+func (n *NotificationHandle) beforeSave() {}
+
+// +checklocksignore
+func (n *NotificationHandle) StateSave(stateSinkObject state.Sink) {
+	n.beforeSave()
+	stateSinkObject.Save(0, &n.n)
+}
+
+func (n *NotificationHandle) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (n *NotificationHandle) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &n.n)
+}
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/channel.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"LinkEPCapabilities",
+		"SupportedGSOKind",
+		"dispatcher",
+		"linkAddr",
+		"mtu",
+		"q",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.LinkEPCapabilities)
+	stateSinkObject.Save(1, &e.SupportedGSOKind)
+	stateSinkObject.Save(2, &e.dispatcher)
+	stateSinkObject.Save(3, &e.linkAddr)
+	stateSinkObject.Save(4, &e.mtu)
+	stateSinkObject.Save(5, &e.q)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.LinkEPCapabilities)
+	stateSourceObject.Load(1, &e.SupportedGSOKind)
+	stateSourceObject.Load(2, &e.dispatcher)
+	stateSourceObject.Load(3, &e.linkAddr)
+	stateSourceObject.Load(4, &e.mtu)
+	stateSourceObject.Load(5, &e.q)
+}
+
+func init() {
+	state.Register((*NotificationHandle)(nil))
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/channel/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package channel
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/channel/queue_mutex.go

@@ -0,0 +1,96 @@
+package channel
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type queueRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var queuelockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type queuelockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *queueRWMutex) Lock() {
+	locking.AddGLock(queueprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *queueRWMutex) NestedLock(i queuelockNameIndex) {
+	locking.AddGLock(queueprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *queueRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(queueprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *queueRWMutex) NestedUnlock(i queuelockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(queueprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *queueRWMutex) RLock() {
+	locking.AddGLock(queueprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *queueRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(queueprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *queueRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *queueRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *queueRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var queueprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func queueinitLockNames() {}
+
+func init() {
+	queueinitLockNames()
+	queueprefixIndex = locking.NewMutexClass(reflect.TypeOf(queueRWMutex{}), queuelockNames)
+}

pkg/tcpip/link/ethernet/ethernet.go

@@ -0,0 +1,121 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package ethernet provides an implementation of an ethernet link endpoint that
+// wraps an inner link endpoint.
+package ethernet
+
+import (
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/nested"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+var (
+	_ stack.NetworkDispatcher = (*Endpoint)(nil)
+	_ stack.LinkEndpoint      = (*Endpoint)(nil)
+)
+
+// New returns an ethernet link endpoint that wraps an inner link endpoint.
+func New(ep stack.LinkEndpoint) *Endpoint {
+	var e Endpoint
+	e.Endpoint.Init(ep, &e)
+	return &e
+}
+
+// Endpoint is an ethernet endpoint.
+//
+// It adds an ethernet header to packets before sending them out through its
+// inner link endpoint and consumes an ethernet header before sending the
+// packet to the stack.
+//
+// +stateify savable
+type Endpoint struct {
+	nested.Endpoint
+}
+
+// LinkAddress implements stack.LinkEndpoint.
+func (e *Endpoint) LinkAddress() tcpip.LinkAddress {
+	if l := e.Endpoint.LinkAddress(); len(l) != 0 {
+		return l
+	}
+	return header.UnspecifiedEthernetAddress
+}
+
+// MTU implements stack.LinkEndpoint.
+func (e *Endpoint) MTU() uint32 {
+	return e.Endpoint.MTU()
+}
+
+// DeliverNetworkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverNetworkPacket(_ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	if !e.ParseHeader(pkt) {
+		return
+	}
+	eth := header.Ethernet(pkt.LinkHeader().Slice())
+	dst := eth.DestinationAddress()
+	if dst == header.EthernetBroadcastAddress {
+		pkt.PktType = tcpip.PacketBroadcast
+	} else if header.IsMulticastEthernetAddress(dst) {
+		pkt.PktType = tcpip.PacketMulticast
+	} else if dst == e.LinkAddress() {
+		pkt.PktType = tcpip.PacketHost
+	} else {
+		pkt.PktType = tcpip.PacketOtherHost
+	}
+
+	// Note, there is no need to check the destination link address here since
+	// the ethernet hardware filters frames based on their destination addresses.
+	e.Endpoint.DeliverNetworkPacket(eth.Type() /* protocol */, pkt)
+}
+
+// Capabilities implements stack.LinkEndpoint.
+func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	c := e.Endpoint.Capabilities()
+	if c&stack.CapabilityLoopback == 0 {
+		c |= stack.CapabilityResolutionRequired
+	}
+	return c
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.
+func (e *Endpoint) MaxHeaderLength() uint16 {
+	return header.EthernetMinimumSize + e.Endpoint.MaxHeaderLength()
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.
+func (e *Endpoint) ARPHardwareType() header.ARPHardwareType {
+	if a := e.Endpoint.ARPHardwareType(); a != header.ARPHardwareNone {
+		return a
+	}
+	return header.ARPHardwareEther
+}
+
+// AddHeader implements stack.LinkEndpoint.
+func (*Endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	fields := header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	}
+	eth.Encode(&fields)
+}
+
+// ParseHeader implements stack.LinkEndpoint.
+func (*Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize)
+	return ok
+}

pkg/tcpip/link/ethernet/ethernet_state_autogen.go

@@ -0,0 +1,38 @@
+// automatically generated by stateify.
+
+package ethernet
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/ethernet.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"Endpoint",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.Endpoint)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.Endpoint)
+}
+
+func init() {
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/fdbased/endpoint.go

@@ -0,0 +1,906 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+// Package fdbased provides the implementation of data-link layer endpoints
+// backed by boundary-preserving file descriptors (e.g., TUN devices,
+// seqpacket/datagram sockets).
+//
+// FD based endpoints can be used in the networking stack by calling New() to
+// create a new endpoint, and then passing it as an argument to
+// Stack.CreateNIC().
+//
+// FD based endpoints can use more than one file descriptor to read incoming
+// packets. If there are more than one FDs specified and the underlying FD is an
+// AF_PACKET then the endpoint will enable FANOUT mode on the socket so that the
+// host kernel will consistently hash the packets to the sockets. This ensures
+// that packets for the same TCP streams are not reordered.
+//
+// Similarly if more than one FD's are specified where the underlying FD is not
+// AF_PACKET then it's the caller's responsibility to ensure that all inbound
+// packets on the descriptors are consistently 5 tuple hashed to one of the
+// descriptors to prevent TCP reordering.
+//
+// Since netstack today does not compute 5 tuple hashes for outgoing packets we
+// only use the first FD to write outbound packets. Once 5 tuple hashes for
+// all outbound packets are available we will make use of all underlying FD's to
+// write outbound packets.
+package fdbased
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"golang.org/x/sys/unix"
+)
+
+// linkDispatcher reads packets from the link FD and dispatches them to the
+// NetworkDispatcher.
+type linkDispatcher interface {
+	Stop()
+	dispatch() (bool, tcpip.Error)
+	release()
+}
+
+// PacketDispatchMode are the various supported methods of receiving and
+// dispatching packets from the underlying FD.
+type PacketDispatchMode int
+
+// BatchSize is the number of packets to write in each syscall. It is 47
+// because when GVisorGSO is in use then a single 65KB TCP segment can get
+// split into 46 segments of 1420 bytes and a single 216 byte segment.
+const BatchSize = 47
+
+const (
+	// Readv is the default dispatch mode and is the least performant of the
+	// dispatch options but the one that is supported by all underlying FD
+	// types.
+	Readv PacketDispatchMode = iota
+	// RecvMMsg enables use of recvmmsg() syscall instead of readv() to
+	// read inbound packets. This reduces # of syscalls needed to process
+	// packets.
+	//
+	// NOTE: recvmmsg() is only supported for sockets, so if the underlying
+	// FD is not a socket then the code will still fall back to the readv()
+	// path.
+	RecvMMsg
+	// PacketMMap enables use of PACKET_RX_RING to receive packets from the
+	// NIC. PacketMMap requires that the underlying FD be an AF_PACKET. The
+	// primary use-case for this is runsc which uses an AF_PACKET FD to
+	// receive packets from the veth device.
+	PacketMMap
+)
+
+func (p PacketDispatchMode) String() string {
+	switch p {
+	case Readv:
+		return "Readv"
+	case RecvMMsg:
+		return "RecvMMsg"
+	case PacketMMap:
+		return "PacketMMap"
+	default:
+		return fmt.Sprintf("unknown packet dispatch mode '%d'", p)
+	}
+}
+
+var (
+	_ stack.LinkEndpoint = (*endpoint)(nil)
+	_ stack.GSOEndpoint  = (*endpoint)(nil)
+)
+
+// +stateify savable
+type fdInfo struct {
+	fd       int
+	isSocket bool
+}
+
+// +stateify savable
+type endpoint struct {
+	// fds is the set of file descriptors each identifying one inbound/outbound
+	// channel. The endpoint will dispatch from all inbound channels as well as
+	// hash outbound packets to specific channels based on the packet hash.
+	fds []fdInfo
+
+	// hdrSize specifies the link-layer header size. If set to 0, no header
+	// is added/removed; otherwise an ethernet header is used.
+	hdrSize int
+
+	// caps holds the endpoint capabilities.
+	caps stack.LinkEndpointCapabilities
+
+	// closed is a function to be called when the FD's peer (if any) closes
+	// its end of the communication pipe.
+	closed func(tcpip.Error) `state:"nosave"`
+
+	inboundDispatchers []linkDispatcher
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+
+	// packetDispatchMode controls the packet dispatcher used by this
+	// endpoint.
+	packetDispatchMode PacketDispatchMode
+
+	// gsoMaxSize is the maximum GSO packet size. It is zero if GSO is
+	// disabled.
+	gsoMaxSize uint32
+
+	// wg keeps track of running goroutines.
+	wg sync.WaitGroup `state:"nosave"`
+
+	// gsoKind is the supported kind of GSO.
+	gsoKind stack.SupportedGSO
+
+	// maxSyscallHeaderBytes has the same meaning as
+	// Options.MaxSyscallHeaderBytes.
+	maxSyscallHeaderBytes uintptr
+
+	// writevMaxIovs is the maximum number of iovecs that may be passed to
+	// rawfile.NonBlockingWriteIovec, as possibly limited by
+	// maxSyscallHeaderBytes. (No analogous limit is defined for
+	// rawfile.NonBlockingSendMMsg, since in that case the maximum number of
+	// iovecs also depends on the number of mmsghdrs. Instead, if sendBatch
+	// encounters a packet whose iovec count is limited by
+	// maxSyscallHeaderBytes, it falls back to writing the packet using writev
+	// via WritePacket.)
+	writevMaxIovs int
+
+	// addr is the address of the endpoint.
+	//
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+
+	// mtu (maximum transmission unit) is the maximum size of a packet.
+	// +checklocks:mu
+	mtu uint32
+}
+
+// Options specify the details about the fd-based endpoint to be created.
+//
+// +stateify savable
+type Options struct {
+	// FDs is a set of FDs used to read/write packets.
+	FDs []int
+
+	// MTU is the mtu to use for this endpoint.
+	MTU uint32
+
+	// EthernetHeader if true, indicates that the endpoint should read/write
+	// ethernet frames instead of IP packets.
+	EthernetHeader bool
+
+	// ClosedFunc is a function to be called when an endpoint's peer (if
+	// any) closes its end of the communication pipe.
+	ClosedFunc func(tcpip.Error)
+
+	// Address is the link address for this endpoint. Only used if
+	// EthernetHeader is true.
+	Address tcpip.LinkAddress
+
+	// SaveRestore if true, indicates that this NIC capability set should
+	// include CapabilitySaveRestore
+	SaveRestore bool
+
+	// DisconnectOk if true, indicates that this NIC capability set should
+	// include CapabilityDisconnectOk.
+	DisconnectOk bool
+
+	// GSOMaxSize is the maximum GSO packet size. It is zero if GSO is
+	// disabled.
+	GSOMaxSize uint32
+
+	// GVisorGSOEnabled indicates whether Gvisor GSO is enabled or not.
+	GVisorGSOEnabled bool
+
+	// PacketDispatchMode specifies the type of inbound dispatcher to be
+	// used for this endpoint.
+	PacketDispatchMode PacketDispatchMode
+
+	// TXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityTXChecksumOffload.
+	TXChecksumOffload bool
+
+	// RXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityRXChecksumOffload.
+	RXChecksumOffload bool
+
+	// If MaxSyscallHeaderBytes is non-zero, it is the maximum number of bytes
+	// of struct iovec, msghdr, and mmsghdr that may be passed by each host
+	// system call.
+	MaxSyscallHeaderBytes int
+
+	// InterfaceIndex is the interface index of the underlying device.
+	InterfaceIndex int
+
+	// GRO enables generic receive offload.
+	GRO bool
+
+	// ProcessorsPerChannel is the number of goroutines used to handle packets
+	// from each FD.
+	ProcessorsPerChannel int
+}
+
+// fanoutID is used for AF_PACKET based endpoints to enable PACKET_FANOUT
+// support in the host kernel. This allows us to use multiple FD's to receive
+// from the same underlying NIC. The fanoutID needs to be the same for a given
+// set of FD's that point to the same NIC. Trying to set the PACKET_FANOUT
+// option for an FD with a fanoutID already in use by another FD for a different
+// NIC will return an EINVAL.
+//
+// Since fanoutID must be unique within the network namespace, we start with
+// the PID to avoid collisions. The only way to be sure of avoiding collisions
+// is to run in a new network namespace.
+var fanoutID atomicbitops.Int32 = atomicbitops.FromInt32(int32(unix.Getpid()))
+
+// New creates a new fd-based endpoint.
+//
+// Makes fd non-blocking, but does not take ownership of fd, which must remain
+// open for the lifetime of the returned endpoint (until after the endpoint has
+// stopped being using and Wait returns).
+func New(opts *Options) (stack.LinkEndpoint, error) {
+	caps := stack.LinkEndpointCapabilities(0)
+	if opts.RXChecksumOffload {
+		caps |= stack.CapabilityRXChecksumOffload
+	}
+
+	if opts.TXChecksumOffload {
+		caps |= stack.CapabilityTXChecksumOffload
+	}
+
+	hdrSize := 0
+	if opts.EthernetHeader {
+		hdrSize = header.EthernetMinimumSize
+		caps |= stack.CapabilityResolutionRequired
+	}
+
+	if opts.SaveRestore {
+		caps |= stack.CapabilitySaveRestore
+	}
+
+	if opts.DisconnectOk {
+		caps |= stack.CapabilityDisconnectOk
+	}
+
+	if len(opts.FDs) == 0 {
+		return nil, fmt.Errorf("opts.FD is empty, at least one FD must be specified")
+	}
+
+	if opts.MaxSyscallHeaderBytes < 0 {
+		return nil, fmt.Errorf("opts.MaxSyscallHeaderBytes is negative")
+	}
+
+	e := &endpoint{
+		mtu:                   opts.MTU,
+		caps:                  caps,
+		closed:                opts.ClosedFunc,
+		addr:                  opts.Address,
+		hdrSize:               hdrSize,
+		packetDispatchMode:    opts.PacketDispatchMode,
+		maxSyscallHeaderBytes: uintptr(opts.MaxSyscallHeaderBytes),
+		writevMaxIovs:         rawfile.MaxIovs,
+	}
+	if e.maxSyscallHeaderBytes != 0 {
+		if max := int(e.maxSyscallHeaderBytes / rawfile.SizeofIovec); max < e.writevMaxIovs {
+			e.writevMaxIovs = max
+		}
+	}
+
+	// Increment fanoutID to ensure that we don't re-use the same fanoutID
+	// for the next endpoint.
+	fid := fanoutID.Add(1)
+
+	// Create per channel dispatchers.
+	for _, fd := range opts.FDs {
+		if err := unix.SetNonblock(fd, true); err != nil {
+			return nil, fmt.Errorf("unix.SetNonblock(%v) failed: %v", fd, err)
+		}
+
+		isSocket, err := isSocketFD(fd)
+		if err != nil {
+			return nil, err
+		}
+		e.fds = append(e.fds, fdInfo{fd: fd, isSocket: isSocket})
+		if opts.GSOMaxSize != 0 {
+			if opts.GVisorGSOEnabled {
+				e.gsoKind = stack.GVisorGSOSupported
+			} else {
+				e.gsoKind = stack.HostGSOSupported
+			}
+			e.gsoMaxSize = opts.GSOMaxSize
+		}
+		if opts.ProcessorsPerChannel == 0 {
+			opts.ProcessorsPerChannel = max(1, runtime.GOMAXPROCS(0)/len(opts.FDs))
+		}
+
+		inboundDispatcher, err := createInboundDispatcher(e, fd, isSocket, fid, opts)
+		if err != nil {
+			return nil, fmt.Errorf("createInboundDispatcher(...) = %v", err)
+		}
+		e.inboundDispatchers = append(e.inboundDispatchers, inboundDispatcher)
+	}
+
+	return e, nil
+}
+
+func createInboundDispatcher(e *endpoint, fd int, isSocket bool, fID int32, opts *Options) (linkDispatcher, error) {
+	// By default use the readv() dispatcher as it works with all kinds of
+	// FDs (tap/tun/unix domain sockets and af_packet).
+	inboundDispatcher, err := newReadVDispatcher(fd, e, opts)
+	if err != nil {
+		return nil, fmt.Errorf("newReadVDispatcher(%d, %+v) = %v", fd, e, err)
+	}
+
+	if isSocket {
+		sa, err := unix.Getsockname(fd)
+		if err != nil {
+			return nil, fmt.Errorf("unix.Getsockname(%d) = %v", fd, err)
+		}
+		switch sa.(type) {
+		case *unix.SockaddrLinklayer:
+			// Enable PACKET_FANOUT mode if the underlying socket is of type
+			// AF_PACKET. We do not enable PACKET_FANOUT_FLAG_DEFRAG as that will
+			// prevent gvisor from receiving fragmented packets and the host does the
+			// reassembly on our behalf before delivering the fragments. This makes it
+			// hard to test fragmentation reassembly code in Netstack.
+			//
+			// See: include/uapi/linux/if_packet.h (struct fanout_args).
+			//
+			// NOTE: We are using SetSockOptInt here even though the underlying
+			// option is actually a struct. The code follows the example in the
+			// kernel documentation as described at the link below:
+			//
+			// See: https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt
+			//
+			// This works out because the actual implementation for the option zero
+			// initializes the structure and will initialize the max_members field
+			// to a proper value if zero.
+			//
+			// See: https://github.com/torvalds/linux/blob/7acac4b3196caee5e21fb5ea53f8bc124e6a16fc/net/packet/af_packet.c#L3881
+			const fanoutType = unix.PACKET_FANOUT_HASH
+			fanoutArg := (int(fID) & 0xffff) | fanoutType<<16
+			if err := unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_FANOUT, fanoutArg); err != nil {
+				return nil, fmt.Errorf("failed to enable PACKET_FANOUT option: %v", err)
+			}
+		}
+
+		switch e.packetDispatchMode {
+		case PacketMMap:
+			inboundDispatcher, err = newPacketMMapDispatcher(fd, e, opts)
+			if err != nil {
+				return nil, fmt.Errorf("newPacketMMapDispatcher(%d, %+v) = %v", fd, e, err)
+			}
+		case RecvMMsg:
+			// If the provided FD is a socket then we optimize
+			// packet reads by using recvmmsg() instead of read() to
+			// read packets in a batch.
+			inboundDispatcher, err = newRecvMMsgDispatcher(fd, e, opts)
+			if err != nil {
+				return nil, fmt.Errorf("newRecvMMsgDispatcher(%d, %+v) = %v", fd, e, err)
+			}
+		case Readv:
+		default:
+			return nil, fmt.Errorf("unknown dispatch mode %d", e.packetDispatchMode)
+		}
+	}
+	return inboundDispatcher, nil
+}
+
+func isSocketFD(fd int) (bool, error) {
+	var stat unix.Stat_t
+	if err := unix.Fstat(fd, &stat); err != nil {
+		return false, fmt.Errorf("unix.Fstat(%v,...) failed: %v", fd, err)
+	}
+	return (stat.Mode & unix.S_IFSOCK) == unix.S_IFSOCK, nil
+}
+
+// Attach launches the goroutine that reads packets from the file descriptor and
+// dispatches them via the provided dispatcher. If one is already attached,
+// then nothing happens.
+//
+// Attach implements stack.LinkEndpoint.Attach.
+func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+
+	// nil means the NIC is being removed.
+	if dispatcher == nil && e.dispatcher != nil {
+		for _, dispatcher := range e.inboundDispatchers {
+			dispatcher.Stop()
+		}
+		e.dispatcher = nil
+		// NOTE(gvisor.dev/issue/11456): Unlock e.mu before e.Wait().
+		e.mu.Unlock()
+		e.Wait()
+		return
+	}
+	defer e.mu.Unlock()
+	if dispatcher != nil && e.dispatcher == nil {
+		e.dispatcher = dispatcher
+		// Link endpoints are not savable. When transportation endpoints are
+		// saved, they stop sending outgoing packets and all incoming packets
+		// are rejected.
+		for i := range e.inboundDispatchers {
+			e.wg.Add(1)
+			go func(i int) { // S/R-SAFE: See above.
+				e.dispatchLoop(e.inboundDispatchers[i])
+				e.wg.Done()
+			}(i)
+		}
+	}
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *endpoint) IsAttached() bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.dispatcher != nil
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *endpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+// SetMTU implements stack.LinkEndpoint.SetMTU.
+func (e *endpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.caps
+}
+
+// MaxHeaderLength returns the maximum size of the link-layer header.
+func (e *endpoint) MaxHeaderLength() uint16 {
+	return uint16(e.hdrSize)
+}
+
+// LinkAddress returns the link address of this endpoint.
+func (e *endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.addr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.addr = addr
+}
+
+// Wait implements stack.LinkEndpoint.Wait. It waits for the endpoint to stop
+// reading from its FD.
+func (e *endpoint) Wait() {
+	e.wg.Wait()
+}
+
+// virtioNetHdr is declared in linux/virtio_net.h.
+type virtioNetHdr struct {
+	flags      uint8
+	gsoType    uint8
+	hdrLen     uint16
+	gsoSize    uint16
+	csumStart  uint16
+	csumOffset uint16
+}
+
+// marshal serializes h to a newly-allocated byte slice, in little-endian byte
+// order.
+//
+// Note: Virtio v1.0 onwards specifies little-endian as the byte ordering used
+// for general serialization. This makes it difficult to use go-marshal for
+// virtio types, as go-marshal implicitly uses the native byte ordering.
+func (h *virtioNetHdr) marshal() []byte {
+	buf := [virtioNetHdrSize]byte{
+		0: byte(h.flags),
+		1: byte(h.gsoType),
+
+		// Manually lay out the fields in little-endian byte order. Little endian =>
+		// least significant bit goes to the lower address.
+
+		2: byte(h.hdrLen),
+		3: byte(h.hdrLen >> 8),
+
+		4: byte(h.gsoSize),
+		5: byte(h.gsoSize >> 8),
+
+		6: byte(h.csumStart),
+		7: byte(h.csumStart >> 8),
+
+		8: byte(h.csumOffset),
+		9: byte(h.csumOffset >> 8),
+	}
+	return buf[:]
+}
+
+// These constants are declared in linux/virtio_net.h.
+const (
+	_VIRTIO_NET_HDR_F_NEEDS_CSUM = 1
+
+	_VIRTIO_NET_HDR_GSO_TCPV4 = 1
+	_VIRTIO_NET_HDR_GSO_TCPV6 = 4
+)
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	if e.hdrSize > 0 {
+		// Add ethernet header if needed.
+		eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+		eth.Encode(&header.EthernetFields{
+			SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+			DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+			Type:    pkt.NetworkProtocolNumber,
+		})
+	}
+}
+
+func (e *endpoint) parseHeader(pkt *stack.PacketBuffer) (header.Ethernet, bool) {
+	if e.hdrSize <= 0 {
+		return nil, true
+	}
+	hdrBytes, ok := pkt.LinkHeader().Consume(e.hdrSize)
+	if !ok {
+		return nil, false
+	}
+	hdr := header.Ethernet(hdrBytes)
+	pkt.NetworkProtocolNumber = hdr.Type()
+	return hdr, true
+}
+
+// parseInboundHeader parses the link header of pkt and returns true if the
+// header is well-formed and sent to this endpoint's MAC or the broadcast
+// address.
+func (e *endpoint) parseInboundHeader(pkt *stack.PacketBuffer, wantAddr tcpip.LinkAddress) bool {
+	hdr, ok := e.parseHeader(pkt)
+	if !ok || e.hdrSize <= 0 {
+		return ok
+	}
+	dstAddr := hdr.DestinationAddress()
+	// Per RFC 9542 2.1 on the least significant bit of the first octet of
+	// a MAC address: "If it is zero, the MAC address is unicast. If it is
+	// a one, the address is groupcast (multicast or broadcast)." Multicast
+	// and broadcast are the same thing to ethernet; they are both sent to
+	// everyone.
+	return dstAddr == wantAddr || byte(dstAddr[0])&0x01 == 1
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := e.parseHeader(pkt)
+	return ok
+}
+
+// writePacket writes outbound packets to the file descriptor. If it is not
+// currently writable, the packet is dropped.
+func (e *endpoint) writePacket(pkt *stack.PacketBuffer) tcpip.Error {
+	fdInfo := e.fds[pkt.Hash%uint32(len(e.fds))]
+	fd := fdInfo.fd
+	var vnetHdrBuf []byte
+	if e.gsoKind == stack.HostGSOSupported {
+		vnetHdr := virtioNetHdr{}
+		if pkt.GSOOptions.Type != stack.GSONone {
+			vnetHdr.hdrLen = uint16(pkt.HeaderSize())
+			if pkt.GSOOptions.NeedsCsum {
+				vnetHdr.flags = _VIRTIO_NET_HDR_F_NEEDS_CSUM
+				vnetHdr.csumStart = pkt.GSOOptions.L3HdrLen
+				vnetHdr.csumOffset = pkt.GSOOptions.CsumOffset
+			}
+			if uint16(pkt.Data().Size()) > pkt.GSOOptions.MSS {
+				switch pkt.GSOOptions.Type {
+				case stack.GSOTCPv4:
+					vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV4
+				case stack.GSOTCPv6:
+					vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV6
+				default:
+					panic(fmt.Sprintf("Unknown gso type: %v", pkt.GSOOptions.Type))
+				}
+				vnetHdr.gsoSize = pkt.GSOOptions.MSS
+			}
+		}
+		vnetHdrBuf = vnetHdr.marshal()
+	}
+
+	views := pkt.AsSlices()
+	numIovecs := len(views)
+	if len(vnetHdrBuf) != 0 {
+		numIovecs++
+	}
+	if numIovecs > e.writevMaxIovs {
+		numIovecs = e.writevMaxIovs
+	}
+
+	// Allocate small iovec arrays on the stack.
+	var iovecsArr [8]unix.Iovec
+	iovecs := iovecsArr[:0]
+	if numIovecs > len(iovecsArr) {
+		iovecs = make([]unix.Iovec, 0, numIovecs)
+	}
+	iovecs = rawfile.AppendIovecFromBytes(iovecs, vnetHdrBuf, numIovecs)
+	for _, v := range views {
+		iovecs = rawfile.AppendIovecFromBytes(iovecs, v, numIovecs)
+	}
+	if errno := rawfile.NonBlockingWriteIovec(fd, iovecs); errno != 0 {
+		return tcpip.TranslateErrno(errno)
+	}
+	return nil
+}
+
+func (e *endpoint) sendBatch(batchFDInfo fdInfo, pkts []*stack.PacketBuffer) (int, tcpip.Error) {
+	// Degrade to writePacket if underlying fd is not a socket.
+	if !batchFDInfo.isSocket {
+		var written int
+		var err tcpip.Error
+		for written < len(pkts) {
+			if err = e.writePacket(pkts[written]); err != nil {
+				break
+			}
+			written++
+		}
+		return written, err
+	}
+
+	// Send a batch of packets through batchFD.
+	batchFD := batchFDInfo.fd
+	mmsgHdrsStorage := make([]rawfile.MMsgHdr, 0, len(pkts))
+	packets := 0
+	for packets < len(pkts) {
+		mmsgHdrs := mmsgHdrsStorage
+		batch := pkts[packets:]
+		syscallHeaderBytes := uintptr(0)
+		for _, pkt := range batch {
+			var vnetHdrBuf []byte
+			if e.gsoKind == stack.HostGSOSupported {
+				vnetHdr := virtioNetHdr{}
+				if pkt.GSOOptions.Type != stack.GSONone {
+					vnetHdr.hdrLen = uint16(pkt.HeaderSize())
+					if pkt.GSOOptions.NeedsCsum {
+						vnetHdr.flags = _VIRTIO_NET_HDR_F_NEEDS_CSUM
+						vnetHdr.csumStart = pkt.GSOOptions.L3HdrLen
+						vnetHdr.csumOffset = pkt.GSOOptions.CsumOffset
+					}
+					if pkt.GSOOptions.Type != stack.GSONone && uint16(pkt.Data().Size()) > pkt.GSOOptions.MSS {
+						switch pkt.GSOOptions.Type {
+						case stack.GSOTCPv4:
+							vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV4
+						case stack.GSOTCPv6:
+							vnetHdr.gsoType = _VIRTIO_NET_HDR_GSO_TCPV6
+						default:
+							panic(fmt.Sprintf("Unknown gso type: %v", pkt.GSOOptions.Type))
+						}
+						vnetHdr.gsoSize = pkt.GSOOptions.MSS
+					}
+				}
+				vnetHdrBuf = vnetHdr.marshal()
+			}
+
+			views, offset := pkt.AsViewList()
+			var skipped int
+			var view *buffer.View
+			for view = views.Front(); view != nil && offset >= view.Size(); view = view.Next() {
+				offset -= view.Size()
+				skipped++
+			}
+
+			// We've made it to the usable views.
+			numIovecs := views.Len() - skipped
+			if len(vnetHdrBuf) != 0 {
+				numIovecs++
+			}
+			if numIovecs > rawfile.MaxIovs {
+				numIovecs = rawfile.MaxIovs
+			}
+			if e.maxSyscallHeaderBytes != 0 {
+				syscallHeaderBytes += rawfile.SizeofMMsgHdr + uintptr(numIovecs)*rawfile.SizeofIovec
+				if syscallHeaderBytes > e.maxSyscallHeaderBytes {
+					// We can't fit this packet into this call to sendmmsg().
+					// We could potentially do so if we reduced numIovecs
+					// further, but this might incur considerable extra
+					// copying. Leave it to the next batch instead.
+					break
+				}
+			}
+
+			// We can't easily allocate iovec arrays on the stack here since
+			// they will escape this loop iteration via mmsgHdrs.
+			iovecs := make([]unix.Iovec, 0, numIovecs)
+			iovecs = rawfile.AppendIovecFromBytes(iovecs, vnetHdrBuf, numIovecs)
+			// At most one slice has a non-zero offset.
+			iovecs = rawfile.AppendIovecFromBytes(iovecs, view.AsSlice()[offset:], numIovecs)
+			for view = view.Next(); view != nil; view = view.Next() {
+				iovecs = rawfile.AppendIovecFromBytes(iovecs, view.AsSlice(), numIovecs)
+			}
+
+			var mmsgHdr rawfile.MMsgHdr
+			mmsgHdr.Msg.Iov = &iovecs[0]
+			mmsgHdr.Msg.SetIovlen(len(iovecs))
+			mmsgHdrs = append(mmsgHdrs, mmsgHdr)
+		}
+
+		if len(mmsgHdrs) == 0 {
+			// We can't fit batch[0] into a mmsghdr while staying under
+			// e.maxSyscallHeaderBytes. Use WritePacket, which will avoid the
+			// mmsghdr (by using writev) and re-buffer iovecs more aggressively
+			// if necessary (by using e.writevMaxIovs instead of
+			// rawfile.MaxIovs).
+			pkt := batch[0]
+			if err := e.writePacket(pkt); err != nil {
+				return packets, err
+			}
+			packets++
+		} else {
+			for len(mmsgHdrs) > 0 {
+				sent, errno := rawfile.NonBlockingSendMMsg(batchFD, mmsgHdrs)
+				if errno != 0 {
+					return packets, tcpip.TranslateErrno(errno)
+				}
+				packets += sent
+				mmsgHdrs = mmsgHdrs[sent:]
+			}
+		}
+	}
+
+	return packets, nil
+}
+
+// WritePackets writes outbound packets to the underlying file descriptors. If
+// one is not currently writable, the packet is dropped.
+//
+// Being a batch API, each packet in pkts should have the following
+// fields populated:
+//   - pkt.EgressRoute
+//   - pkt.GSOOptions
+//   - pkt.NetworkProtocolNumber
+func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	// Preallocate to avoid repeated reallocation as we append to batch.
+	batch := make([]*stack.PacketBuffer, 0, BatchSize)
+	batchFDInfo := fdInfo{fd: -1, isSocket: false}
+	sentPackets := 0
+	for _, pkt := range pkts.AsSlice() {
+		if len(batch) == 0 {
+			batchFDInfo = e.fds[pkt.Hash%uint32(len(e.fds))]
+		}
+		pktFDInfo := e.fds[pkt.Hash%uint32(len(e.fds))]
+		if sendNow := pktFDInfo != batchFDInfo; !sendNow {
+			batch = append(batch, pkt)
+			continue
+		}
+		n, err := e.sendBatch(batchFDInfo, batch)
+		sentPackets += n
+		if err != nil {
+			return sentPackets, err
+		}
+		batch = batch[:0]
+		batch = append(batch, pkt)
+		batchFDInfo = pktFDInfo
+	}
+
+	if len(batch) != 0 {
+		n, err := e.sendBatch(batchFDInfo, batch)
+		sentPackets += n
+		if err != nil {
+			return sentPackets, err
+		}
+	}
+	return sentPackets, nil
+}
+
+// InjectOutbound implements stack.InjectableEndpoint.InjectOutbound.
+func (e *endpoint) InjectOutbound(dest tcpip.Address, packet *buffer.View) tcpip.Error {
+	if errno := rawfile.NonBlockingWrite(e.fds[0].fd, packet.AsSlice()); errno != 0 {
+		return tcpip.TranslateErrno(errno)
+	}
+	return nil
+}
+
+// dispatchLoop reads packets from the file descriptor in a loop and dispatches
+// them to the network stack.
+func (e *endpoint) dispatchLoop(inboundDispatcher linkDispatcher) tcpip.Error {
+	for {
+		cont, err := inboundDispatcher.dispatch()
+		if err != nil || !cont {
+			if e.closed != nil {
+				e.closed(err)
+			}
+			inboundDispatcher.release()
+			return err
+		}
+	}
+}
+
+// GSOMaxSize implements stack.GSOEndpoint.
+func (e *endpoint) GSOMaxSize() uint32 {
+	return e.gsoMaxSize
+}
+
+// SupportedGSO implements stack.GSOEndpoint.
+func (e *endpoint) SupportedGSO() stack.SupportedGSO {
+	return e.gsoKind
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (e *endpoint) ARPHardwareType() header.ARPHardwareType {
+	if e.hdrSize > 0 {
+		return header.ARPHardwareEther
+	}
+	return header.ARPHardwareNone
+}
+
+// Close implements stack.LinkEndpoint.
+func (e *endpoint) Close() {}
+
+// SetOnCloseAction implements stack.LinkEndpoint.
+func (*endpoint) SetOnCloseAction(func()) {}
+
+// InjectableEndpoint is an injectable fd-based endpoint. The endpoint writes
+// to the FD, but does not read from it. All reads come from injected packets.
+//
+// +stateify savable
+type InjectableEndpoint struct {
+	endpoint
+
+	mu injectableEndpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+}
+
+// Attach saves the stack network-layer dispatcher for use later when packets
+// are injected.
+func (e *InjectableEndpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dispatcher = dispatcher
+}
+
+// InjectInbound injects an inbound packet. If the endpoint is not attached, the
+// packet is not delivered.
+func (e *InjectableEndpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverNetworkPacket(protocol, pkt)
+	}
+}
+
+// NewInjectable creates a new fd-based InjectableEndpoint.
+func NewInjectable(fd int, mtu uint32, capabilities stack.LinkEndpointCapabilities) (*InjectableEndpoint, error) {
+	unix.SetNonblock(fd, true)
+	isSocket, err := isSocketFD(fd)
+	if err != nil {
+		return nil, err
+	}
+
+	return &InjectableEndpoint{endpoint: endpoint{
+		fds:           []fdInfo{{fd: fd, isSocket: isSocket}},
+		mtu:           mtu,
+		caps:          capabilities,
+		writevMaxIovs: rawfile.MaxIovs,
+	}}, nil
+}

pkg/tcpip/link/fdbased/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package fdbased
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/fdbased/endpoint_unsafe.go

@@ -0,0 +1,24 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package fdbased
+
+import (
+	"unsafe"
+)
+
+const virtioNetHdrSize = int(unsafe.Sizeof(virtioNetHdr{}))

pkg/tcpip/link/fdbased/fdbased_nonlinux_state_autogen.go

@@ -0,0 +1,6 @@
+// automatically generated by stateify.
+
+//go:build !linux || (!amd64 && !arm64)
+// +build !linux !amd64,!arm64
+
+package fdbased

pkg/tcpip/link/fdbased/fdbased_state_autogen.go

@@ -0,0 +1,438 @@
+// automatically generated by stateify.
+
+//go:build linux && ((linux && amd64) || (linux && arm64)) && linux && linux
+// +build linux
+// +build linux,amd64 linux,arm64
+// +build linux
+// +build linux
+
+package fdbased
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (f *fdInfo) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.fdInfo"
+}
+
+func (f *fdInfo) StateFields() []string {
+	return []string{
+		"fd",
+		"isSocket",
+	}
+}
+
+func (f *fdInfo) beforeSave() {}
+
+// +checklocksignore
+func (f *fdInfo) StateSave(stateSinkObject state.Sink) {
+	f.beforeSave()
+	stateSinkObject.Save(0, &f.fd)
+	stateSinkObject.Save(1, &f.isSocket)
+}
+
+func (f *fdInfo) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (f *fdInfo) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &f.fd)
+	stateSourceObject.Load(1, &f.isSocket)
+}
+
+func (e *endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.endpoint"
+}
+
+func (e *endpoint) StateFields() []string {
+	return []string{
+		"fds",
+		"hdrSize",
+		"caps",
+		"inboundDispatchers",
+		"dispatcher",
+		"packetDispatchMode",
+		"gsoMaxSize",
+		"gsoKind",
+		"maxSyscallHeaderBytes",
+		"writevMaxIovs",
+		"addr",
+		"mtu",
+	}
+}
+
+func (e *endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.fds)
+	stateSinkObject.Save(1, &e.hdrSize)
+	stateSinkObject.Save(2, &e.caps)
+	stateSinkObject.Save(3, &e.inboundDispatchers)
+	stateSinkObject.Save(4, &e.dispatcher)
+	stateSinkObject.Save(5, &e.packetDispatchMode)
+	stateSinkObject.Save(6, &e.gsoMaxSize)
+	stateSinkObject.Save(7, &e.gsoKind)
+	stateSinkObject.Save(8, &e.maxSyscallHeaderBytes)
+	stateSinkObject.Save(9, &e.writevMaxIovs)
+	stateSinkObject.Save(10, &e.addr)
+	stateSinkObject.Save(11, &e.mtu)
+}
+
+func (e *endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.fds)
+	stateSourceObject.Load(1, &e.hdrSize)
+	stateSourceObject.Load(2, &e.caps)
+	stateSourceObject.Load(3, &e.inboundDispatchers)
+	stateSourceObject.Load(4, &e.dispatcher)
+	stateSourceObject.Load(5, &e.packetDispatchMode)
+	stateSourceObject.Load(6, &e.gsoMaxSize)
+	stateSourceObject.Load(7, &e.gsoKind)
+	stateSourceObject.Load(8, &e.maxSyscallHeaderBytes)
+	stateSourceObject.Load(9, &e.writevMaxIovs)
+	stateSourceObject.Load(10, &e.addr)
+	stateSourceObject.Load(11, &e.mtu)
+}
+
+func (o *Options) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.Options"
+}
+
+func (o *Options) StateFields() []string {
+	return []string{
+		"FDs",
+		"MTU",
+		"EthernetHeader",
+		"ClosedFunc",
+		"Address",
+		"SaveRestore",
+		"DisconnectOk",
+		"GSOMaxSize",
+		"GVisorGSOEnabled",
+		"PacketDispatchMode",
+		"TXChecksumOffload",
+		"RXChecksumOffload",
+		"MaxSyscallHeaderBytes",
+		"InterfaceIndex",
+		"GRO",
+		"ProcessorsPerChannel",
+	}
+}
+
+func (o *Options) beforeSave() {}
+
+// +checklocksignore
+func (o *Options) StateSave(stateSinkObject state.Sink) {
+	o.beforeSave()
+	stateSinkObject.Save(0, &o.FDs)
+	stateSinkObject.Save(1, &o.MTU)
+	stateSinkObject.Save(2, &o.EthernetHeader)
+	stateSinkObject.Save(3, &o.ClosedFunc)
+	stateSinkObject.Save(4, &o.Address)
+	stateSinkObject.Save(5, &o.SaveRestore)
+	stateSinkObject.Save(6, &o.DisconnectOk)
+	stateSinkObject.Save(7, &o.GSOMaxSize)
+	stateSinkObject.Save(8, &o.GVisorGSOEnabled)
+	stateSinkObject.Save(9, &o.PacketDispatchMode)
+	stateSinkObject.Save(10, &o.TXChecksumOffload)
+	stateSinkObject.Save(11, &o.RXChecksumOffload)
+	stateSinkObject.Save(12, &o.MaxSyscallHeaderBytes)
+	stateSinkObject.Save(13, &o.InterfaceIndex)
+	stateSinkObject.Save(14, &o.GRO)
+	stateSinkObject.Save(15, &o.ProcessorsPerChannel)
+}
+
+func (o *Options) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &o.FDs)
+	stateSourceObject.Load(1, &o.MTU)
+	stateSourceObject.Load(2, &o.EthernetHeader)
+	stateSourceObject.Load(3, &o.ClosedFunc)
+	stateSourceObject.Load(4, &o.Address)
+	stateSourceObject.Load(5, &o.SaveRestore)
+	stateSourceObject.Load(6, &o.DisconnectOk)
+	stateSourceObject.Load(7, &o.GSOMaxSize)
+	stateSourceObject.Load(8, &o.GVisorGSOEnabled)
+	stateSourceObject.Load(9, &o.PacketDispatchMode)
+	stateSourceObject.Load(10, &o.TXChecksumOffload)
+	stateSourceObject.Load(11, &o.RXChecksumOffload)
+	stateSourceObject.Load(12, &o.MaxSyscallHeaderBytes)
+	stateSourceObject.Load(13, &o.InterfaceIndex)
+	stateSourceObject.Load(14, &o.GRO)
+	stateSourceObject.Load(15, &o.ProcessorsPerChannel)
+}
+
+func (e *InjectableEndpoint) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.InjectableEndpoint"
+}
+
+func (e *InjectableEndpoint) StateFields() []string {
+	return []string{
+		"endpoint",
+		"dispatcher",
+	}
+}
+
+func (e *InjectableEndpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *InjectableEndpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.endpoint)
+	stateSinkObject.Save(1, &e.dispatcher)
+}
+
+func (e *InjectableEndpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *InjectableEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.endpoint)
+	stateSourceObject.Load(1, &e.dispatcher)
+}
+
+func (d *packetMMapDispatcher) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.packetMMapDispatcher"
+}
+
+func (d *packetMMapDispatcher) StateFields() []string {
+	return []string{
+		"StopFD",
+		"fd",
+		"e",
+		"ringBuffer",
+		"ringOffset",
+		"mgr",
+	}
+}
+
+func (d *packetMMapDispatcher) beforeSave() {}
+
+// +checklocksignore
+func (d *packetMMapDispatcher) StateSave(stateSinkObject state.Sink) {
+	d.beforeSave()
+	stateSinkObject.Save(0, &d.StopFD)
+	stateSinkObject.Save(1, &d.fd)
+	stateSinkObject.Save(2, &d.e)
+	stateSinkObject.Save(3, &d.ringBuffer)
+	stateSinkObject.Save(4, &d.ringOffset)
+	stateSinkObject.Save(5, &d.mgr)
+}
+
+func (d *packetMMapDispatcher) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (d *packetMMapDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &d.StopFD)
+	stateSourceObject.Load(1, &d.fd)
+	stateSourceObject.Load(2, &d.e)
+	stateSourceObject.Load(3, &d.ringBuffer)
+	stateSourceObject.Load(4, &d.ringOffset)
+	stateSourceObject.Load(5, &d.mgr)
+}
+
+func (b *iovecBuffer) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.iovecBuffer"
+}
+
+func (b *iovecBuffer) StateFields() []string {
+	return []string{
+		"views",
+		"sizes",
+		"skipsVnetHdr",
+		"pulledIndex",
+	}
+}
+
+func (b *iovecBuffer) beforeSave() {}
+
+// +checklocksignore
+func (b *iovecBuffer) StateSave(stateSinkObject state.Sink) {
+	b.beforeSave()
+	stateSinkObject.Save(0, &b.views)
+	stateSinkObject.Save(1, &b.sizes)
+	stateSinkObject.Save(2, &b.skipsVnetHdr)
+	stateSinkObject.Save(3, &b.pulledIndex)
+}
+
+func (b *iovecBuffer) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (b *iovecBuffer) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &b.views)
+	stateSourceObject.Load(1, &b.sizes)
+	stateSourceObject.Load(2, &b.skipsVnetHdr)
+	stateSourceObject.Load(3, &b.pulledIndex)
+}
+
+func (d *readVDispatcher) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.readVDispatcher"
+}
+
+func (d *readVDispatcher) StateFields() []string {
+	return []string{
+		"StopFD",
+		"fd",
+		"e",
+		"buf",
+		"mgr",
+	}
+}
+
+func (d *readVDispatcher) beforeSave() {}
+
+// +checklocksignore
+func (d *readVDispatcher) StateSave(stateSinkObject state.Sink) {
+	d.beforeSave()
+	stateSinkObject.Save(0, &d.StopFD)
+	stateSinkObject.Save(1, &d.fd)
+	stateSinkObject.Save(2, &d.e)
+	stateSinkObject.Save(3, &d.buf)
+	stateSinkObject.Save(4, &d.mgr)
+}
+
+func (d *readVDispatcher) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (d *readVDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &d.StopFD)
+	stateSourceObject.Load(1, &d.fd)
+	stateSourceObject.Load(2, &d.e)
+	stateSourceObject.Load(3, &d.buf)
+	stateSourceObject.Load(4, &d.mgr)
+}
+
+func (r *recvMMsgDispatcher) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.recvMMsgDispatcher"
+}
+
+func (r *recvMMsgDispatcher) StateFields() []string {
+	return []string{
+		"StopFD",
+		"fd",
+		"e",
+		"bufs",
+		"pkts",
+		"gro",
+		"mgr",
+	}
+}
+
+func (r *recvMMsgDispatcher) beforeSave() {}
+
+// +checklocksignore
+func (r *recvMMsgDispatcher) StateSave(stateSinkObject state.Sink) {
+	r.beforeSave()
+	stateSinkObject.Save(0, &r.StopFD)
+	stateSinkObject.Save(1, &r.fd)
+	stateSinkObject.Save(2, &r.e)
+	stateSinkObject.Save(3, &r.bufs)
+	stateSinkObject.Save(4, &r.pkts)
+	stateSinkObject.Save(5, &r.gro)
+	stateSinkObject.Save(6, &r.mgr)
+}
+
+// +checklocksignore
+func (r *recvMMsgDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &r.StopFD)
+	stateSourceObject.Load(1, &r.fd)
+	stateSourceObject.Load(2, &r.e)
+	stateSourceObject.Load(3, &r.bufs)
+	stateSourceObject.Load(4, &r.pkts)
+	stateSourceObject.Load(5, &r.gro)
+	stateSourceObject.Load(6, &r.mgr)
+	stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) })
+}
+
+func (p *processor) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.processor"
+}
+
+func (p *processor) StateFields() []string {
+	return []string{
+		"pkts",
+		"e",
+		"gro",
+		"sleeper",
+		"packetWaker",
+		"closeWaker",
+	}
+}
+
+func (p *processor) beforeSave() {}
+
+// +checklocksignore
+func (p *processor) StateSave(stateSinkObject state.Sink) {
+	p.beforeSave()
+	stateSinkObject.Save(0, &p.pkts)
+	stateSinkObject.Save(1, &p.e)
+	stateSinkObject.Save(2, &p.gro)
+	stateSinkObject.Save(3, &p.sleeper)
+	stateSinkObject.Save(4, &p.packetWaker)
+	stateSinkObject.Save(5, &p.closeWaker)
+}
+
+func (p *processor) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (p *processor) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &p.pkts)
+	stateSourceObject.Load(1, &p.e)
+	stateSourceObject.Load(2, &p.gro)
+	stateSourceObject.Load(3, &p.sleeper)
+	stateSourceObject.Load(4, &p.packetWaker)
+	stateSourceObject.Load(5, &p.closeWaker)
+}
+
+func (m *processorManager) StateTypeName() string {
+	return "pkg/tcpip/link/fdbased.processorManager"
+}
+
+func (m *processorManager) StateFields() []string {
+	return []string{
+		"processors",
+		"seed",
+		"e",
+		"ready",
+	}
+}
+
+func (m *processorManager) beforeSave() {}
+
+// +checklocksignore
+func (m *processorManager) StateSave(stateSinkObject state.Sink) {
+	m.beforeSave()
+	stateSinkObject.Save(0, &m.processors)
+	stateSinkObject.Save(1, &m.seed)
+	stateSinkObject.Save(2, &m.e)
+	stateSinkObject.Save(3, &m.ready)
+}
+
+// +checklocksignore
+func (m *processorManager) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &m.processors)
+	stateSourceObject.Load(1, &m.seed)
+	stateSourceObject.Load(2, &m.e)
+	stateSourceObject.Load(3, &m.ready)
+	stateSourceObject.AfterLoad(func() { m.afterLoad(ctx) })
+}
+
+func init() {
+	state.Register((*fdInfo)(nil))
+	state.Register((*endpoint)(nil))
+	state.Register((*Options)(nil))
+	state.Register((*InjectableEndpoint)(nil))
+	state.Register((*packetMMapDispatcher)(nil))
+	state.Register((*iovecBuffer)(nil))
+	state.Register((*readVDispatcher)(nil))
+	state.Register((*recvMMsgDispatcher)(nil))
+	state.Register((*processor)(nil))
+	state.Register((*processorManager)(nil))
+}

pkg/tcpip/link/fdbased/fdbased_unsafe_state_autogen.go

@@ -0,0 +1,7 @@
+// automatically generated by stateify.
+
+//go:build linux && ((linux && amd64) || (linux && arm64))
+// +build linux
+// +build linux,amd64 linux,arm64
+
+package fdbased

pkg/tcpip/link/fdbased/injectable_endpoint_mutex.go

@@ -0,0 +1,96 @@
+package fdbased
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type injectableEndpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var injectableEndpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type injectableEndpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) Lock() {
+	locking.AddGLock(injectableEndpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) NestedLock(i injectableEndpointlockNameIndex) {
+	locking.AddGLock(injectableEndpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(injectableEndpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) NestedUnlock(i injectableEndpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(injectableEndpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) RLock() {
+	locking.AddGLock(injectableEndpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(injectableEndpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *injectableEndpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var injectableEndpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func injectableEndpointinitLockNames() {}
+
+func init() {
+	injectableEndpointinitLockNames()
+	injectableEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(injectableEndpointRWMutex{}), injectableEndpointlockNames)
+}

pkg/tcpip/link/fdbased/mmap.go

@@ -0,0 +1,199 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build (linux && amd64) || (linux && arm64)
+// +build linux,amd64 linux,arm64
+
+package fdbased
+
+import (
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/stopfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	tPacketAlignment = uintptr(16)
+	tpStatusKernel   = 0
+	tpStatusUser     = 1
+	tpStatusCopy     = 2
+	tpStatusLosing   = 4
+)
+
+// We overallocate the frame size to accommodate space for the
+// TPacketHdr+RawSockAddrLinkLayer+MAC header and any padding.
+//
+// Memory allocated for the ring buffer: tpBlockSize * tpBlockNR = 2 MiB
+//
+// NOTE:
+//
+//	Frames need to be aligned at 16 byte boundaries.
+//	BlockSize needs to be page aligned.
+//
+//	For details see PACKET_MMAP setting constraints in
+//	https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt
+const (
+	tpFrameSize = 65536 + 128
+	tpBlockSize = tpFrameSize * 32
+	tpBlockNR   = 1
+	tpFrameNR   = (tpBlockSize * tpBlockNR) / tpFrameSize
+)
+
+// tPacketAlign aligns the pointer v at a tPacketAlignment boundary. Direct
+// translation of the TPACKET_ALIGN macro in <linux/if_packet.h>.
+func tPacketAlign(v uintptr) uintptr {
+	return (v + tPacketAlignment - 1) & uintptr(^(tPacketAlignment - 1))
+}
+
+// tPacketReq is the tpacket_req structure as described in
+// https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt
+type tPacketReq struct {
+	tpBlockSize uint32
+	tpBlockNR   uint32
+	tpFrameSize uint32
+	tpFrameNR   uint32
+}
+
+// tPacketHdr is tpacket_hdr structure as described in <linux/if_packet.h>
+type tPacketHdr []byte
+
+const (
+	tpStatusOffset  = 0
+	tpLenOffset     = 8
+	tpSnapLenOffset = 12
+	tpMacOffset     = 16
+	tpNetOffset     = 18
+	tpSecOffset     = 20
+	tpUSecOffset    = 24
+)
+
+func (t tPacketHdr) tpLen() uint32 {
+	return binary.LittleEndian.Uint32(t[tpLenOffset:])
+}
+
+func (t tPacketHdr) tpSnapLen() uint32 {
+	return binary.LittleEndian.Uint32(t[tpSnapLenOffset:])
+}
+
+func (t tPacketHdr) tpMac() uint16 {
+	return binary.LittleEndian.Uint16(t[tpMacOffset:])
+}
+
+func (t tPacketHdr) tpNet() uint16 {
+	return binary.LittleEndian.Uint16(t[tpNetOffset:])
+}
+
+func (t tPacketHdr) tpSec() uint32 {
+	return binary.LittleEndian.Uint32(t[tpSecOffset:])
+}
+
+func (t tPacketHdr) tpUSec() uint32 {
+	return binary.LittleEndian.Uint32(t[tpUSecOffset:])
+}
+
+func (t tPacketHdr) Payload() []byte {
+	return t[uint32(t.tpMac()) : uint32(t.tpMac())+t.tpSnapLen()]
+}
+
+// packetMMapDispatcher uses PACKET_RX_RING's to read/dispatch inbound packets.
+// See: mmap_amd64_unsafe.go for implementation details.
+//
+// +stateify savable
+type packetMMapDispatcher struct {
+	stopfd.StopFD
+	// fd is the file descriptor used to send and receive packets.
+	fd int
+
+	// e is the endpoint this dispatcher is attached to.
+	e *endpoint
+
+	// ringBuffer is only used when PacketMMap dispatcher is used and points
+	// to the start of the mmapped PACKET_RX_RING buffer.
+	ringBuffer []byte
+
+	// ringOffset is the current offset into the ring buffer where the next
+	// inbound packet will be placed by the kernel.
+	ringOffset int
+
+	// mgr is the processor goroutine manager.
+	mgr *processorManager
+}
+
+func (d *packetMMapDispatcher) release() {
+	d.mgr.close()
+}
+
+func (d *packetMMapDispatcher) readMMappedPackets() (stack.PacketBufferList, bool, tcpip.Error) {
+	var pkts stack.PacketBufferList
+	hdr := tPacketHdr(d.ringBuffer[d.ringOffset*tpFrameSize:])
+	for hdr.tpStatus()&tpStatusUser == 0 {
+		stopped, errno := rawfile.BlockingPollUntilStopped(d.EFD, d.fd, unix.POLLIN|unix.POLLERR)
+		if errno != 0 {
+			if errno == unix.EINTR {
+				continue
+			}
+			return pkts, stopped, tcpip.TranslateErrno(errno)
+		}
+		if stopped {
+			return pkts, true, nil
+		}
+		if hdr.tpStatus()&tpStatusCopy != 0 {
+			// This frame is truncated so skip it after flipping the
+			// buffer to the kernel.
+			hdr.setTPStatus(tpStatusKernel)
+			d.ringOffset = (d.ringOffset + 1) % tpFrameNR
+			hdr = (tPacketHdr)(d.ringBuffer[d.ringOffset*tpFrameSize:])
+			continue
+		}
+	}
+
+	for hdr.tpStatus()&tpStatusUser == 1 {
+		// Copy out the packet from the mmapped frame to a locally owned buffer.
+		pkts.PushBack(stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: buffer.MakeWithView(buffer.NewViewWithData(hdr.Payload())),
+		}))
+		// Release packet to kernel.
+		hdr.setTPStatus(tpStatusKernel)
+		d.ringOffset = (d.ringOffset + 1) % tpFrameNR
+		hdr = tPacketHdr(d.ringBuffer[d.ringOffset*tpFrameSize:])
+	}
+	return pkts, false, nil
+}
+
+// dispatch reads packets from an mmaped ring buffer and dispatches them to the
+// network stack.
+func (d *packetMMapDispatcher) dispatch() (bool, tcpip.Error) {
+	pkts, stopped, err := d.readMMappedPackets()
+	defer pkts.Reset()
+	if err != nil || stopped {
+		return false, err
+	}
+	d.e.mu.RLock()
+	addr := d.e.addr
+	d.e.mu.RUnlock()
+	for _, pkt := range pkts.AsSlice() {
+		if d.e.parseInboundHeader(pkt, addr) {
+			d.mgr.queuePacket(pkt, d.e.hdrSize > 0)
+		}
+	}
+	if pkts.Len() > 0 {
+		d.mgr.wakeReady()
+	}
+	return true, nil
+}

pkg/tcpip/link/fdbased/mmap_nonlinux.go

@@ -0,0 +1,24 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !linux || (!amd64 && !arm64)
+// +build !linux !amd64,!arm64
+
+package fdbased
+
+// Stubbed out version for non-linux/non-amd64/non-arm64 platforms.
+
+func newPacketMMapDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) {
+	return nil, nil
+}

pkg/tcpip/link/fdbased/mmap_unsafe.go

@@ -0,0 +1,92 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build (linux && amd64) || (linux && arm64)
+// +build linux,amd64 linux,arm64
+
+package fdbased
+
+import (
+	"fmt"
+	"unsafe"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/stopfd"
+	"golang.org/x/sys/unix"
+)
+
+// tPacketHdrlen is the TPACKET_HDRLEN variable defined in <linux/if_packet.h>.
+var tPacketHdrlen = tPacketAlign(unsafe.Sizeof(tPacketHdr{}) + unsafe.Sizeof(unix.RawSockaddrLinklayer{}))
+
+// tpStatus returns the frame status field.
+// The status is concurrently updated by the kernel as a result we must
+// use atomic operations to prevent races.
+func (t tPacketHdr) tpStatus() uint32 {
+	hdr := unsafe.Pointer(&t[0])
+	statusPtr := unsafe.Pointer(uintptr(hdr) + uintptr(tpStatusOffset))
+	return (*atomicbitops.Uint32)(statusPtr).Load()
+}
+
+// setTPStatus set's the frame status to the provided status.
+// The status is concurrently updated by the kernel as a result we must
+// use atomic operations to prevent races.
+func (t tPacketHdr) setTPStatus(status uint32) {
+	hdr := unsafe.Pointer(&t[0])
+	statusPtr := unsafe.Pointer(uintptr(hdr) + uintptr(tpStatusOffset))
+	(*atomicbitops.Uint32)(statusPtr).Store(status)
+}
+
+func newPacketMMapDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) {
+	stopFD, err := stopfd.New()
+	if err != nil {
+		return nil, err
+	}
+	d := &packetMMapDispatcher{
+		StopFD: stopFD,
+		fd:     fd,
+		e:      e,
+	}
+	pageSize := unix.Getpagesize()
+	if tpBlockSize%pageSize != 0 {
+		return nil, fmt.Errorf("tpBlockSize: %d is not page aligned, pagesize: %d", tpBlockSize, pageSize)
+	}
+	tReq := tPacketReq{
+		tpBlockSize: uint32(tpBlockSize),
+		tpBlockNR:   uint32(tpBlockNR),
+		tpFrameSize: uint32(tpFrameSize),
+		tpFrameNR:   uint32(tpFrameNR),
+	}
+	// Setup PACKET_RX_RING.
+	if err := setsockopt(d.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, unsafe.Pointer(&tReq), unsafe.Sizeof(tReq)); err != nil {
+		return nil, fmt.Errorf("failed to enable PACKET_RX_RING: %v", err)
+	}
+	// Let's mmap the blocks.
+	sz := tpBlockSize * tpBlockNR
+	buf, err := unix.Mmap(d.fd, 0, sz, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED)
+	if err != nil {
+		return nil, fmt.Errorf("unix.Mmap(...,0, %v, ...) failed = %v", sz, err)
+	}
+	d.mgr = newProcessorManager(opts, e)
+	d.mgr.start()
+	d.ringBuffer = buf
+	return d, nil
+}
+
+func setsockopt(fd, level, name int, val unsafe.Pointer, vallen uintptr) error {
+	if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), uintptr(val), vallen, 0); errno != 0 {
+		return error(errno)
+	}
+
+	return nil
+}

pkg/tcpip/link/fdbased/packet_dispatchers.go

@@ -0,0 +1,330 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package fdbased
+
+import (
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/stopfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack/gro"
+	"golang.org/x/sys/unix"
+)
+
+// BufConfig defines the shape of the buffer used to read packets from the NIC.
+// The duplication of 256 is intended so that the sum of the elements can cover
+// the maximum packet size we expect to receive. See TestBufConfigMaxLength.
+var BufConfig = []int{128, 256, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768}
+
+// +stateify savable
+type iovecBuffer struct {
+	// buffer is the actual buffer that holds the packet contents. Some contents
+	// are reused across calls to pullBuffer if number of requested bytes is
+	// smaller than the number of bytes allocated in the buffer.
+	views []*buffer.View
+
+	// iovecs are initialized with base pointers/len of the corresponding
+	// entries in the views defined above, except when GSO is enabled
+	// (skipsVnetHdr) then the first iovec points to a buffer for the vnet header
+	// which is stripped before the views are passed up the stack for further
+	// processing.
+	iovecs []unix.Iovec `state:"nosave"`
+
+	// sizes is an array of buffer sizes for the underlying views. sizes is
+	// immutable.
+	sizes []int
+
+	// skipsVnetHdr is true if virtioNetHdr is to skipped.
+	skipsVnetHdr bool
+
+	// pulledIndex is the index of the last []byte buffer pulled from the
+	// underlying buffer storage during a call to pullBuffers. It is -1
+	// if no buffer is pulled.
+	pulledIndex int
+}
+
+func newIovecBuffer(sizes []int, skipsVnetHdr bool) *iovecBuffer {
+	b := &iovecBuffer{
+		views:        make([]*buffer.View, len(sizes)),
+		sizes:        sizes,
+		skipsVnetHdr: skipsVnetHdr,
+	}
+	niov := len(b.views)
+	if b.skipsVnetHdr {
+		niov++
+	}
+	b.iovecs = make([]unix.Iovec, niov)
+	return b
+}
+
+func (b *iovecBuffer) nextIovecs() []unix.Iovec {
+	vnetHdrOff := 0
+	if b.skipsVnetHdr {
+		var vnetHdr [virtioNetHdrSize]byte
+		// The kernel adds virtioNetHdr before each packet, but
+		// we don't use it, so we allocate a buffer for it,
+		// add it in iovecs but don't add it in a view.
+		b.iovecs[0] = unix.Iovec{Base: &vnetHdr[0]}
+		b.iovecs[0].SetLen(virtioNetHdrSize)
+		vnetHdrOff++
+	}
+
+	for i := range b.views {
+		if b.views[i] != nil {
+			break
+		}
+		v := buffer.NewViewSize(b.sizes[i])
+		b.views[i] = v
+		b.iovecs[i+vnetHdrOff] = unix.Iovec{Base: v.BasePtr()}
+		b.iovecs[i+vnetHdrOff].SetLen(v.Size())
+	}
+	return b.iovecs
+}
+
+// pullBuffer extracts the enough underlying storage from b.buffer to hold n
+// bytes. It removes this storage from b.buffer, returns a new buffer
+// that holds the storage, and updates pulledIndex to indicate which part
+// of b.buffer's storage must be reallocated during the next call to
+// nextIovecs.
+func (b *iovecBuffer) pullBuffer(n int) buffer.Buffer {
+	var views []*buffer.View
+	c := 0
+	if b.skipsVnetHdr {
+		c += virtioNetHdrSize
+		if c >= n {
+			// Nothing in the packet.
+			return buffer.Buffer{}
+		}
+	}
+	// Remove the used views from the buffer.
+	for i, v := range b.views {
+		c += v.Size()
+		if c >= n {
+			b.views[i].CapLength(v.Size() - (c - n))
+			views = append(views, b.views[:i+1]...)
+			break
+		}
+	}
+	for i := range views {
+		b.views[i] = nil
+	}
+	if b.skipsVnetHdr {
+		// Exclude the size of the vnet header.
+		n -= virtioNetHdrSize
+	}
+	pulled := buffer.Buffer{}
+	for _, v := range views {
+		pulled.Append(v)
+	}
+	pulled.Truncate(int64(n))
+	return pulled
+}
+
+func (b *iovecBuffer) release() {
+	for _, v := range b.views {
+		if v != nil {
+			v.Release()
+			v = nil
+		}
+	}
+}
+
+// readVDispatcher uses readv() system call to read inbound packets and
+// dispatches them.
+//
+// +stateify savable
+type readVDispatcher struct {
+	stopfd.StopFD
+	// fd is the file descriptor used to send and receive packets.
+	fd int
+
+	// e is the endpoint this dispatcher is attached to.
+	e *endpoint
+
+	// buf is the iovec buffer that contains the packet contents.
+	buf *iovecBuffer
+
+	// mgr is the processor goroutine manager.
+	mgr *processorManager
+}
+
+func newReadVDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) {
+	stopFD, err := stopfd.New()
+	if err != nil {
+		return nil, err
+	}
+	d := &readVDispatcher{
+		StopFD: stopFD,
+		fd:     fd,
+		e:      e,
+	}
+	skipsVnetHdr := d.e.gsoKind == stack.HostGSOSupported
+	d.buf = newIovecBuffer(BufConfig, skipsVnetHdr)
+	d.mgr = newProcessorManager(opts, e)
+	d.mgr.start()
+	return d, nil
+}
+
+func (d *readVDispatcher) release() {
+	d.buf.release()
+	d.mgr.close()
+}
+
+// dispatch reads one packet from the file descriptor and dispatches it.
+func (d *readVDispatcher) dispatch() (bool, tcpip.Error) {
+	n, errno := rawfile.BlockingReadvUntilStopped(d.EFD, d.fd, d.buf.nextIovecs())
+	if n <= 0 || errno != 0 {
+		return false, tcpip.TranslateErrno(errno)
+	}
+
+	pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+		Payload: d.buf.pullBuffer(n),
+	})
+	defer pkt.DecRef()
+
+	d.e.mu.RLock()
+	addr := d.e.addr
+	d.e.mu.RUnlock()
+	if !d.e.parseInboundHeader(pkt, addr) {
+		return false, nil
+	}
+	d.mgr.queuePacket(pkt, d.e.hdrSize > 0)
+	d.mgr.wakeReady()
+	return true, nil
+}
+
+// recvMMsgDispatcher uses the recvmmsg system call to read inbound packets and
+// dispatches them.
+//
+// +stateify savable
+type recvMMsgDispatcher struct {
+	stopfd.StopFD
+	// fd is the file descriptor used to send and receive packets.
+	fd int
+
+	// e is the endpoint this dispatcher is attached to.
+	e *endpoint
+
+	// bufs is an array of iovec buffers that contain packet contents.
+	bufs []*iovecBuffer
+
+	// msgHdrs is an array of MMsgHdr objects where each MMsghdr is used to
+	// reference an array of iovecs in the iovecs field defined above.  This
+	// array is passed as the parameter to recvmmsg call to retrieve
+	// potentially more than 1 packet per unix.
+	msgHdrs []rawfile.MMsgHdr `state:"nosave"`
+
+	// pkts is reused to avoid allocations.
+	pkts stack.PacketBufferList
+
+	// gro coalesces incoming packets to increase throughput.
+	gro gro.GRO
+
+	// mgr is the processor goroutine manager.
+	mgr *processorManager
+}
+
+const (
+	// MaxMsgsPerRecv is the maximum number of packets we want to retrieve
+	// in a single RecvMMsg call.
+	MaxMsgsPerRecv = 8
+)
+
+func newRecvMMsgDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, error) {
+	stopFD, err := stopfd.New()
+	if err != nil {
+		return nil, err
+	}
+	d := &recvMMsgDispatcher{
+		StopFD:  stopFD,
+		fd:      fd,
+		e:       e,
+		bufs:    make([]*iovecBuffer, MaxMsgsPerRecv),
+		msgHdrs: make([]rawfile.MMsgHdr, MaxMsgsPerRecv),
+	}
+	skipsVnetHdr := d.e.gsoKind == stack.HostGSOSupported
+	for i := range d.bufs {
+		d.bufs[i] = newIovecBuffer(BufConfig, skipsVnetHdr)
+	}
+	d.gro.Init(opts.GRO)
+	d.mgr = newProcessorManager(opts, e)
+	d.mgr.start()
+
+	return d, nil
+}
+
+func (d *recvMMsgDispatcher) release() {
+	for _, iov := range d.bufs {
+		iov.release()
+	}
+	d.mgr.close()
+}
+
+// recvMMsgDispatch reads more than one packet at a time from the file
+// descriptor and dispatches it.
+func (d *recvMMsgDispatcher) dispatch() (bool, tcpip.Error) {
+	// Fill message headers.
+	for k := range d.msgHdrs {
+		if d.msgHdrs[k].Msg.Iovlen > 0 {
+			break
+		}
+		iovecs := d.bufs[k].nextIovecs()
+		iovLen := len(iovecs)
+		d.msgHdrs[k].Len = 0
+		d.msgHdrs[k].Msg.Iov = &iovecs[0]
+		d.msgHdrs[k].Msg.SetIovlen(iovLen)
+	}
+
+	nMsgs, errno := rawfile.BlockingRecvMMsgUntilStopped(d.EFD, d.fd, d.msgHdrs)
+	if errno != 0 {
+		return false, tcpip.TranslateErrno(errno)
+	}
+	if nMsgs == -1 {
+		return false, nil
+	}
+
+	// Process each of received packets.
+
+	d.e.mu.RLock()
+	addr := d.e.addr
+	dsp := d.e.dispatcher
+	d.e.mu.RUnlock()
+
+	d.gro.Dispatcher = dsp
+	defer d.pkts.Reset()
+
+	for k := 0; k < nMsgs; k++ {
+		n := int(d.msgHdrs[k].Len)
+		pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: d.bufs[k].pullBuffer(n),
+		})
+		d.pkts.PushBack(pkt)
+
+		// Mark that this iovec has been processed.
+		d.msgHdrs[k].Msg.Iovlen = 0
+
+		if d.e.parseInboundHeader(pkt, addr) {
+			pkt.RXChecksumValidated = d.e.caps&stack.CapabilityRXChecksumOffload != 0
+			d.mgr.queuePacket(pkt, d.e.hdrSize > 0)
+		}
+	}
+	d.mgr.wakeReady()
+
+	return true, nil
+}

pkg/tcpip/link/fdbased/processor_mutex.go

@@ -0,0 +1,64 @@
+package fdbased
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// Mutex is sync.Mutex with the correctness validator.
+type processorMutex struct {
+	mu sync.Mutex
+}
+
+var processorprefixIndex *locking.MutexClass
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var processorlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type processorlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *processorMutex) Lock() {
+	locking.AddGLock(processorprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *processorMutex) NestedLock(i processorlockNameIndex) {
+	locking.AddGLock(processorprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *processorMutex) Unlock() {
+	locking.DelGLock(processorprefixIndex, -1)
+	m.mu.Unlock()
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *processorMutex) NestedUnlock(i processorlockNameIndex) {
+	locking.DelGLock(processorprefixIndex, int(i))
+	m.mu.Unlock()
+}
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func processorinitLockNames() {}
+
+func init() {
+	processorinitLockNames()
+	processorprefixIndex = locking.NewMutexClass(reflect.TypeOf(processorMutex{}), processorlockNames)
+}

pkg/tcpip/link/fdbased/processors.go

@@ -0,0 +1,278 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package fdbased
+
+import (
+	"context"
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/rand"
+	"github.com/sagernet/gvisor/pkg/sleep"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/hash/jenkins"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack/gro"
+)
+
+// +stateify savable
+type processor struct {
+	mu processorMutex `state:"nosave"`
+	// +checklocks:mu
+	pkts stack.PacketBufferList
+
+	e           *endpoint
+	gro         gro.GRO
+	sleeper     sleep.Sleeper
+	packetWaker sleep.Waker
+	closeWaker  sleep.Waker
+}
+
+func (p *processor) start(wg *sync.WaitGroup) {
+	defer wg.Done()
+	defer p.sleeper.Done()
+	for {
+		switch w := p.sleeper.Fetch(true); {
+		case w == &p.packetWaker:
+			p.deliverPackets()
+		case w == &p.closeWaker:
+			p.mu.Lock()
+			p.pkts.Reset()
+			p.mu.Unlock()
+			return
+		}
+	}
+}
+
+func (p *processor) deliverPackets() {
+	p.e.mu.RLock()
+	p.gro.Dispatcher = p.e.dispatcher
+	p.e.mu.RUnlock()
+	if p.gro.Dispatcher == nil {
+		p.mu.Lock()
+		p.pkts.Reset()
+		p.mu.Unlock()
+		return
+	}
+
+	p.mu.Lock()
+	for p.pkts.Len() > 0 {
+		pkt := p.pkts.PopFront()
+		p.mu.Unlock()
+		p.gro.Enqueue(pkt)
+		pkt.DecRef()
+		p.mu.Lock()
+	}
+	p.mu.Unlock()
+	p.gro.Flush()
+}
+
+// processorManager handles starting, closing, and queuing packets on processor
+// goroutines.
+//
+// +stateify savable
+type processorManager struct {
+	processors []processor
+	seed       uint32
+	wg         sync.WaitGroup `state:"nosave"`
+	e          *endpoint
+	ready      []bool
+}
+
+// newProcessorManager creates a new processor manager.
+func newProcessorManager(opts *Options, e *endpoint) *processorManager {
+	m := &processorManager{}
+	m.seed = rand.Uint32()
+	m.ready = make([]bool, opts.ProcessorsPerChannel)
+	m.processors = make([]processor, opts.ProcessorsPerChannel)
+	m.e = e
+	m.wg.Add(opts.ProcessorsPerChannel)
+
+	for i := range m.processors {
+		p := &m.processors[i]
+		p.sleeper.AddWaker(&p.packetWaker)
+		p.sleeper.AddWaker(&p.closeWaker)
+		p.gro.Init(opts.GRO)
+		p.e = e
+	}
+
+	return m
+}
+
+// start starts the processor goroutines if the processor manager is configured
+// with more than one processor.
+func (m *processorManager) start() {
+	for i := range m.processors {
+		p := &m.processors[i]
+		// Only start processor in a separate goroutine if we have multiple of them.
+		if len(m.processors) > 1 {
+			go p.start(&m.wg)
+		}
+	}
+}
+
+// afterLoad is invoked by stateify.
+func (m *processorManager) afterLoad(context.Context) {
+	m.wg.Add(len(m.processors))
+	m.start()
+}
+
+func (m *processorManager) connectionHash(cid *connectionID) uint32 {
+	var payload [4]byte
+	binary.LittleEndian.PutUint16(payload[0:], cid.srcPort)
+	binary.LittleEndian.PutUint16(payload[2:], cid.dstPort)
+
+	h := jenkins.Sum32(m.seed)
+	h.Write(payload[:])
+	h.Write(cid.srcAddr)
+	h.Write(cid.dstAddr)
+	return h.Sum32()
+}
+
+// queuePacket queues a packet to be delivered to the appropriate processor.
+func (m *processorManager) queuePacket(pkt *stack.PacketBuffer, hasEthHeader bool) {
+	var pIdx uint32
+	cid, nonConnectionPkt := tcpipConnectionID(pkt)
+	if !hasEthHeader {
+		if nonConnectionPkt {
+			// If there's no eth header this should be a standard tcpip packet. If
+			// it isn't the packet is invalid so drop it.
+			return
+		}
+		pkt.NetworkProtocolNumber = cid.proto
+	}
+	if len(m.processors) == 1 || nonConnectionPkt {
+		// If the packet is not associated with an active connection, use the
+		// first processor.
+		pIdx = 0
+	} else {
+		pIdx = m.connectionHash(&cid) % uint32(len(m.processors))
+	}
+	p := &m.processors[pIdx]
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.pkts.PushBack(pkt.IncRef())
+	m.ready[pIdx] = true
+}
+
+type connectionID struct {
+	srcAddr, dstAddr []byte
+	srcPort, dstPort uint16
+	proto            tcpip.NetworkProtocolNumber
+}
+
+// tcpipConnectionID returns a tcpip connection id tuple based on the data found
+// in the packet. It returns true if the packet is not associated with an active
+// connection (e.g ARP, NDP, etc). The method assumes link headers have already
+// been processed if they were present.
+func tcpipConnectionID(pkt *stack.PacketBuffer) (connectionID, bool) {
+	var cid connectionID
+	h, ok := pkt.Data().PullUp(1)
+	if !ok {
+		// Skip this packet.
+		return cid, true
+	}
+
+	const tcpSrcDstPortLen = 4
+	switch header.IPVersion(h) {
+	case header.IPv4Version:
+		hdrLen := header.IPv4(h).HeaderLength()
+		h, ok = pkt.Data().PullUp(int(hdrLen) + tcpSrcDstPortLen)
+		if !ok {
+			return cid, true
+		}
+		ipHdr := header.IPv4(h[:hdrLen])
+		tcpHdr := header.TCP(h[hdrLen:][:tcpSrcDstPortLen])
+
+		cid.srcAddr = ipHdr.SourceAddressSlice()
+		cid.dstAddr = ipHdr.DestinationAddressSlice()
+		// All fragment packets need to be processed by the same goroutine, so
+		// only record the TCP ports if this is not a fragment packet.
+		if ipHdr.IsValid(pkt.Data().Size()) && !ipHdr.More() && ipHdr.FragmentOffset() == 0 {
+			cid.srcPort = tcpHdr.SourcePort()
+			cid.dstPort = tcpHdr.DestinationPort()
+		}
+		cid.proto = header.IPv4ProtocolNumber
+	case header.IPv6Version:
+		h, ok = pkt.Data().PullUp(header.IPv6FixedHeaderSize + tcpSrcDstPortLen)
+		if !ok {
+			return cid, true
+		}
+		ipHdr := header.IPv6(h)
+
+		var tcpHdr header.TCP
+		if tcpip.TransportProtocolNumber(ipHdr.NextHeader()) == header.TCPProtocolNumber {
+			tcpHdr = header.TCP(h[header.IPv6FixedHeaderSize:][:tcpSrcDstPortLen])
+		} else {
+			// Slow path for IPv6 extension headers :(.
+			dataBuf := pkt.Data().ToBuffer()
+			dataBuf.TrimFront(header.IPv6MinimumSize)
+			it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(ipHdr.NextHeader()), dataBuf)
+			defer it.Release()
+			for {
+				hdr, done, err := it.Next()
+				if done || err != nil {
+					break
+				}
+				hdr.Release()
+			}
+			h, ok = pkt.Data().PullUp(int(it.HeaderOffset()) + tcpSrcDstPortLen)
+			if !ok {
+				return cid, true
+			}
+			tcpHdr = header.TCP(h[it.HeaderOffset():][:tcpSrcDstPortLen])
+		}
+		cid.srcAddr = ipHdr.SourceAddressSlice()
+		cid.dstAddr = ipHdr.DestinationAddressSlice()
+		cid.srcPort = tcpHdr.SourcePort()
+		cid.dstPort = tcpHdr.DestinationPort()
+		cid.proto = header.IPv6ProtocolNumber
+	default:
+		return cid, true
+	}
+	return cid, false
+}
+
+func (m *processorManager) close() {
+	if len(m.processors) < 2 {
+		return
+	}
+	for i := range m.processors {
+		p := &m.processors[i]
+		p.closeWaker.Assert()
+	}
+}
+
+// wakeReady wakes up all processors that have a packet queued. If there is only
+// one processor, the method delivers the packet inline without waking a
+// goroutine.
+func (m *processorManager) wakeReady() {
+	for i, ready := range m.ready {
+		if !ready {
+			continue
+		}
+		p := &m.processors[i]
+		if len(m.processors) > 1 {
+			p.packetWaker.Assert()
+		} else {
+			p.deliverPackets()
+		}
+		m.ready[i] = false
+	}
+}

pkg/tcpip/link/fdbased/save_restore.go

@@ -0,0 +1,26 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fdbased
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/rawfile"
+)
+
+// afterLoad is invoked by stateify.
+func (r *recvMMsgDispatcher) afterLoad(context.Context) {
+	r.msgHdrs = make([]rawfile.MMsgHdr, MaxMsgsPerRecv)
+}

pkg/tcpip/link/loopback/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package loopback
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/loopback/loopback.go

@@ -0,0 +1,146 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package loopback provides the implementation of loopback data-link layer
+// endpoints. Such endpoints just turn outbound packets into inbound ones.
+//
+// Loopback endpoints can be used in the networking stack by calling New() to
+// create a new endpoint, and then passing it as an argument to
+// Stack.CreateNIC().
+package loopback
+
+import (
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+const (
+	loopbackMTU = 65536
+)
+
+// +stateify savable
+type endpoint struct {
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+	// +checklocks:mu
+	mtu uint32
+}
+
+// New creates a new loopback endpoint. This link-layer endpoint just turns
+// outbound packets into inbound packets.
+func New() stack.LinkEndpoint {
+	return &endpoint{
+		mtu: loopbackMTU,
+	}
+}
+
+// Attach implements stack.LinkEndpoint.Attach. It just saves the stack network-
+// layer dispatcher for later use when packets need to be dispatched.
+func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dispatcher = dispatcher
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *endpoint) IsAttached() bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.dispatcher != nil
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *endpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+// SetMTU implements stack.LinkEndpoint.SetMTU. It has no impact.
+func (e *endpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities. Loopback advertises
+// itself as supporting checksum offload, but in reality it's just omitted.
+func (*endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return stack.CapabilityRXChecksumOffload | stack.CapabilityTXChecksumOffload | stack.CapabilitySaveRestore | stack.CapabilityLoopback
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. Given that the
+// loopback interface doesn't have a header, it just returns 0.
+func (*endpoint) MaxHeaderLength() uint16 {
+	return 0
+}
+
+// LinkAddress returns the link address of this endpoint.
+func (e *endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.addr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.addr = addr
+}
+
+// Wait implements stack.LinkEndpoint.Wait.
+func (*endpoint) Wait() {}
+
+// WritePackets implements stack.LinkEndpoint.WritePackets. If the endpoint is
+// not attached, the packets are not delivered.
+func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	for _, pkt := range pkts.AsSlice() {
+		// In order to properly loop back to the inbound side we must create a
+		// fresh packet that only contains the underlying payload with no headers
+		// or struct fields set.
+		newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: pkt.ToBuffer(),
+		})
+		if d != nil {
+			d.DeliverNetworkPacket(pkt.NetworkProtocolNumber, newPkt)
+		}
+		newPkt.DecRef()
+	}
+	return pkts.Len(), nil
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (*endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareLoopback
+}
+
+// AddHeader implements stack.LinkEndpoint.
+func (*endpoint) AddHeader(*stack.PacketBuffer) {}
+
+// ParseHeader implements stack.LinkEndpoint.
+func (*endpoint) ParseHeader(*stack.PacketBuffer) bool { return true }
+
+// Close implements stack.LinkEndpoint.
+func (*endpoint) Close() {}
+
+// SetOnCloseAction implements stack.LinkEndpoint.
+func (*endpoint) SetOnCloseAction(func()) {}

pkg/tcpip/link/loopback/loopback_state_autogen.go

@@ -0,0 +1,44 @@
+// automatically generated by stateify.
+
+package loopback
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/loopback.endpoint"
+}
+
+func (e *endpoint) StateFields() []string {
+	return []string{
+		"dispatcher",
+		"addr",
+		"mtu",
+	}
+}
+
+func (e *endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.dispatcher)
+	stateSinkObject.Save(1, &e.addr)
+	stateSinkObject.Save(2, &e.mtu)
+}
+
+func (e *endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.dispatcher)
+	stateSourceObject.Load(1, &e.addr)
+	stateSourceObject.Load(2, &e.mtu)
+}
+
+func init() {
+	state.Register((*endpoint)(nil))
+}

pkg/tcpip/link/muxed/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package muxed
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/muxed/injectable.go

@@ -0,0 +1,174 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package muxed provides a muxed link endpoints.
+package muxed
+
+import (
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// InjectableEndpoint is an injectable multi endpoint. The endpoint has
+// trivial routing rules that determine which InjectableEndpoint a given packet
+// will be written to. Note that HandleLocal works differently for this
+// endpoint (see WritePacket).
+//
+// +stateify savable
+type InjectableEndpoint struct {
+	routes map[tcpip.Address]stack.InjectableLinkEndpoint
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+}
+
+// MTU implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) MTU() uint32 {
+	minMTU := ^uint32(0)
+	for _, endpoint := range m.routes {
+		if endpointMTU := endpoint.MTU(); endpointMTU < minMTU {
+			minMTU = endpointMTU
+		}
+	}
+	return minMTU
+}
+
+// SetMTU implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) SetMTU(mtu uint32) {
+	for _, endpoint := range m.routes {
+		endpoint.SetMTU(mtu)
+	}
+}
+
+// Capabilities implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) Capabilities() stack.LinkEndpointCapabilities {
+	minCapabilities := stack.LinkEndpointCapabilities(^uint(0))
+	for _, endpoint := range m.routes {
+		minCapabilities &= endpoint.Capabilities()
+	}
+	return minCapabilities
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) MaxHeaderLength() uint16 {
+	minHeaderLen := ^uint16(0)
+	for _, endpoint := range m.routes {
+		if headerLen := endpoint.MaxHeaderLength(); headerLen < minHeaderLen {
+			minHeaderLen = headerLen
+		}
+	}
+	return minHeaderLen
+}
+
+// LinkAddress implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) LinkAddress() tcpip.LinkAddress {
+	return ""
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (m *InjectableEndpoint) SetLinkAddress(tcpip.LinkAddress) {}
+
+// Attach implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	for _, endpoint := range m.routes {
+		endpoint.Attach(dispatcher)
+	}
+	m.mu.Lock()
+	m.dispatcher = dispatcher
+	m.mu.Unlock()
+}
+
+// IsAttached implements stack.LinkEndpoint.
+func (m *InjectableEndpoint) IsAttached() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.dispatcher != nil
+}
+
+// InjectInbound implements stack.InjectableLinkEndpoint.
+func (m *InjectableEndpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	m.mu.RLock()
+	d := m.dispatcher
+	m.mu.RUnlock()
+	d.DeliverNetworkPacket(protocol, pkt)
+}
+
+// WritePackets writes outbound packets to the appropriate
+// LinkInjectableEndpoint based on the RemoteAddress. HandleLocal only works if
+// pkt.EgressRoute.RemoteAddress has a route registered in this endpoint.
+func (m *InjectableEndpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	i := 0
+	for _, pkt := range pkts.AsSlice() {
+		endpoint, ok := m.routes[pkt.EgressRoute.RemoteAddress]
+		if !ok {
+			return i, &tcpip.ErrHostUnreachable{}
+		}
+
+		var tmpPkts stack.PacketBufferList
+		tmpPkts.PushBack(pkt)
+
+		n, err := endpoint.WritePackets(tmpPkts)
+		if err != nil {
+			return i, err
+		}
+
+		i += n
+	}
+
+	return i, nil
+}
+
+// InjectOutbound writes outbound packets to the appropriate
+// LinkInjectableEndpoint based on the dest address.
+func (m *InjectableEndpoint) InjectOutbound(dest tcpip.Address, packet *buffer.View) tcpip.Error {
+	endpoint, ok := m.routes[dest]
+	if !ok {
+		return &tcpip.ErrHostUnreachable{}
+	}
+	return endpoint.InjectOutbound(dest, packet)
+}
+
+// Wait implements stack.LinkEndpoint.Wait.
+func (m *InjectableEndpoint) Wait() {
+	for _, ep := range m.routes {
+		ep.Wait()
+	}
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (*InjectableEndpoint) ARPHardwareType() header.ARPHardwareType {
+	panic("unsupported operation")
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (*InjectableEndpoint) AddHeader(*stack.PacketBuffer) {}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (*InjectableEndpoint) ParseHeader(*stack.PacketBuffer) bool { return true }
+
+// Close implements stack.LinkEndpoint.
+func (*InjectableEndpoint) Close() {}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (*InjectableEndpoint) SetOnCloseAction(func()) {}
+
+// NewInjectableEndpoint creates a new multi-endpoint injectable endpoint.
+func NewInjectableEndpoint(routes map[tcpip.Address]stack.InjectableLinkEndpoint) *InjectableEndpoint {
+	return &InjectableEndpoint{
+		routes: routes,
+	}
+}

pkg/tcpip/link/muxed/muxed_state_autogen.go

@@ -0,0 +1,41 @@
+// automatically generated by stateify.
+
+package muxed
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (m *InjectableEndpoint) StateTypeName() string {
+	return "pkg/tcpip/link/muxed.InjectableEndpoint"
+}
+
+func (m *InjectableEndpoint) StateFields() []string {
+	return []string{
+		"routes",
+		"dispatcher",
+	}
+}
+
+func (m *InjectableEndpoint) beforeSave() {}
+
+// +checklocksignore
+func (m *InjectableEndpoint) StateSave(stateSinkObject state.Sink) {
+	m.beforeSave()
+	stateSinkObject.Save(0, &m.routes)
+	stateSinkObject.Save(1, &m.dispatcher)
+}
+
+func (m *InjectableEndpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (m *InjectableEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &m.routes)
+	stateSourceObject.Load(1, &m.dispatcher)
+}
+
+func init() {
+	state.Register((*InjectableEndpoint)(nil))
+}

pkg/tcpip/link/nested/nested.go

@@ -0,0 +1,185 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package nested provides helpers to implement the pattern of nested
+// stack.LinkEndpoints.
+package nested
+
+import (
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// Endpoint is a wrapper around stack.LinkEndpoint and stack.NetworkDispatcher
+// that can be used to implement nesting safely by providing lifecycle
+// concurrency guards.
+//
+// See the tests in this package for example usage.
+//
+// +stateify savable
+type Endpoint struct {
+	child    stack.LinkEndpoint
+	embedder stack.NetworkDispatcher
+
+	// mu protects dispatcher.
+	mu         sync.RWMutex `state:"nosave"`
+	dispatcher stack.NetworkDispatcher
+}
+
+var (
+	_ stack.GSOEndpoint       = (*Endpoint)(nil)
+	_ stack.LinkEndpoint      = (*Endpoint)(nil)
+	_ stack.NetworkDispatcher = (*Endpoint)(nil)
+)
+
+// Init initializes a nested.Endpoint that uses embedder as the dispatcher for
+// child on Attach.
+//
+// See the tests in this package for example usage.
+func (e *Endpoint) Init(child stack.LinkEndpoint, embedder stack.NetworkDispatcher) {
+	e.child = child
+	e.embedder = embedder
+}
+
+// DeliverNetworkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverNetworkPacket(protocol, pkt)
+	}
+}
+
+// DeliverLinkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverLinkPacket(protocol, pkt)
+	}
+}
+
+// Attach implements stack.LinkEndpoint.
+func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	e.dispatcher = dispatcher
+	e.mu.Unlock()
+	// If we're attaching to a valid dispatcher, pass embedder as the dispatcher
+	// to our child, otherwise detach the child by giving it a nil dispatcher.
+	var pass stack.NetworkDispatcher
+	if dispatcher != nil {
+		pass = e.embedder
+	}
+	e.child.Attach(pass)
+}
+
+// IsAttached implements stack.LinkEndpoint.
+func (e *Endpoint) IsAttached() bool {
+	e.mu.RLock()
+	isAttached := e.dispatcher != nil
+	e.mu.RUnlock()
+	return isAttached
+}
+
+// MTU implements stack.LinkEndpoint.
+func (e *Endpoint) MTU() uint32 {
+	return e.child.MTU()
+}
+
+// SetMTU implements stack.LinkEndpoint.
+func (e *Endpoint) SetMTU(mtu uint32) {
+	e.child.SetMTU(mtu)
+}
+
+// Capabilities implements stack.LinkEndpoint.
+func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.child.Capabilities()
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.
+func (e *Endpoint) MaxHeaderLength() uint16 {
+	return e.child.MaxHeaderLength()
+}
+
+// LinkAddress implements stack.LinkEndpoint.
+func (e *Endpoint) LinkAddress() tcpip.LinkAddress {
+	return e.child.LinkAddress()
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.child.SetLinkAddress(addr)
+}
+
+// WritePackets implements stack.LinkEndpoint.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	return e.child.WritePackets(pkts)
+}
+
+// Wait implements stack.LinkEndpoint.
+func (e *Endpoint) Wait() {
+	e.child.Wait()
+}
+
+// GSOMaxSize implements stack.GSOEndpoint.
+func (e *Endpoint) GSOMaxSize() uint32 {
+	if e, ok := e.child.(stack.GSOEndpoint); ok {
+		return e.GSOMaxSize()
+	}
+	return 0
+}
+
+// SupportedGSO implements stack.GSOEndpoint.
+func (e *Endpoint) SupportedGSO() stack.SupportedGSO {
+	if e, ok := e.child.(stack.GSOEndpoint); ok {
+		return e.SupportedGSO()
+	}
+	return stack.GSONotSupported
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType
+func (e *Endpoint) ARPHardwareType() header.ARPHardwareType {
+	return e.child.ARPHardwareType()
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *Endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	e.child.AddHeader(pkt)
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	return e.child.ParseHeader(pkt)
+}
+
+// Close implements stack.LinkEndpoint.
+func (e *Endpoint) Close() {
+	e.child.Close()
+}
+
+// SetOnCloseAction implement stack.LinkEndpoints.
+func (e *Endpoint) SetOnCloseAction(action func()) {
+	e.child.SetOnCloseAction(action)
+}
+
+// Child returns the child endpoint.
+func (e *Endpoint) Child() stack.LinkEndpoint {
+	return e.child
+}

pkg/tcpip/link/nested/nested_state_autogen.go

@@ -0,0 +1,44 @@
+// automatically generated by stateify.
+
+package nested
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/nested.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"child",
+		"embedder",
+		"dispatcher",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.child)
+	stateSinkObject.Save(1, &e.embedder)
+	stateSinkObject.Save(2, &e.dispatcher)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.child)
+	stateSourceObject.Load(1, &e.embedder)
+	stateSourceObject.Load(2, &e.dispatcher)
+}
+
+func init() {
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/packetsocket/packetsocket.go

@@ -0,0 +1,62 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package packetsocket provides a link endpoint that enables delivery of
+// incoming and outgoing packets to any interested packet sockets.
+package packetsocket
+
+import (
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/nested"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+var (
+	_ stack.NetworkDispatcher = (*Endpoint)(nil)
+	_ stack.LinkEndpoint      = (*Endpoint)(nil)
+)
+
+// Endpoint is a link endpoint that enables delivery of incoming and outgoing
+// packets to any interested packet sockets.
+//
+// +stateify savable
+type Endpoint struct {
+	nested.Endpoint
+}
+
+// New creates a new packetsocket link endpoint wrapping a lower link endpoint.
+//
+// On ingress, the lower link endpoint must only deliver packets that have
+// a link-layer header set if one is required for the link.
+func New(lower stack.LinkEndpoint) stack.LinkEndpoint {
+	e := &Endpoint{}
+	e.Endpoint.Init(lower, e)
+	return e
+}
+
+// DeliverNetworkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.Endpoint.DeliverLinkPacket(protocol, pkt)
+
+	e.Endpoint.DeliverNetworkPacket(protocol, pkt)
+}
+
+// WritePackets implements stack.LinkEndpoint.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	for _, pkt := range pkts.AsSlice() {
+		e.Endpoint.DeliverLinkPacket(pkt.NetworkProtocolNumber, pkt)
+	}
+
+	return e.Endpoint.WritePackets(pkts)
+}

pkg/tcpip/link/packetsocket/packetsocket_state_autogen.go

@@ -0,0 +1,38 @@
+// automatically generated by stateify.
+
+package packetsocket
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/packetsocket.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"Endpoint",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.Endpoint)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.Endpoint)
+}
+
+func init() {
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/pipe/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package pipe
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/pipe/pipe.go

@@ -0,0 +1,154 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pipe provides the implementation of pipe-like data-link layer
+// endpoints. Such endpoints allow packets to be sent between two interfaces.
+package pipe
+
+import (
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+var _ stack.LinkEndpoint = (*Endpoint)(nil)
+
+// New returns both ends of a new pipe.
+func New(linkAddr1, linkAddr2 tcpip.LinkAddress, mtu uint32) (*Endpoint, *Endpoint) {
+	ep1 := &Endpoint{
+		linkAddr: linkAddr1,
+		mtu:      mtu,
+	}
+	ep2 := &Endpoint{
+		linkAddr: linkAddr2,
+		mtu:      mtu,
+	}
+	ep1.linked = ep2
+	ep2.linked = ep1
+	return ep1, ep2
+}
+
+// Endpoint is one end of a pipe.
+//
+// +stateify savable
+type Endpoint struct {
+	linked *Endpoint
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+	// +checklocks:mu
+	linkAddr tcpip.LinkAddress
+	// +checklocks:mu
+	mtu uint32
+}
+
+func (e *Endpoint) deliverPackets(pkts stack.PacketBufferList) {
+	e.linked.mu.RLock()
+	d := e.linked.dispatcher
+	e.linked.mu.RUnlock()
+	if d == nil {
+		return
+	}
+
+	for _, pkt := range pkts.AsSlice() {
+		// Create a fresh packet with pkt's payload but without struct fields
+		// or headers set so the next link protocol can properly set the link
+		// header.
+		newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: pkt.ToBuffer(),
+		})
+		d.DeliverNetworkPacket(pkt.NetworkProtocolNumber, newPkt)
+		newPkt.DecRef()
+	}
+}
+
+// WritePackets implements stack.LinkEndpoint.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	n := pkts.Len()
+	e.deliverPackets(pkts)
+	return n, nil
+}
+
+// Attach implements stack.LinkEndpoint.
+func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dispatcher = dispatcher
+}
+
+// IsAttached implements stack.LinkEndpoint.
+func (e *Endpoint) IsAttached() bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.dispatcher != nil
+}
+
+// Wait implements stack.LinkEndpoint.
+func (*Endpoint) Wait() {}
+
+// MTU implements stack.LinkEndpoint.
+func (e *Endpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+// SetMTU implements stack.LinkEndpoint.
+func (e *Endpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.
+func (*Endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return 0
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.
+func (*Endpoint) MaxHeaderLength() uint16 {
+	return 0
+}
+
+// LinkAddress implements stack.LinkEndpoint.
+func (e *Endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.linkAddr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.
+func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.linkAddr = addr
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.
+func (*Endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareNone
+}
+
+// AddHeader implements stack.LinkEndpoint.
+func (*Endpoint) AddHeader(*stack.PacketBuffer) {}
+
+// ParseHeader implements stack.LinkEndpoint.
+func (*Endpoint) ParseHeader(*stack.PacketBuffer) bool { return true }
+
+// Close implements stack.LinkEndpoint.
+func (e *Endpoint) Close() {}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (*Endpoint) SetOnCloseAction(func()) {}

pkg/tcpip/link/pipe/pipe_state_autogen.go

@@ -0,0 +1,47 @@
+// automatically generated by stateify.
+
+package pipe
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/pipe.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"linked",
+		"dispatcher",
+		"linkAddr",
+		"mtu",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.linked)
+	stateSinkObject.Save(1, &e.dispatcher)
+	stateSinkObject.Save(2, &e.linkAddr)
+	stateSinkObject.Save(3, &e.mtu)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.linked)
+	stateSourceObject.Load(1, &e.dispatcher)
+	stateSourceObject.Load(2, &e.linkAddr)
+	stateSourceObject.Load(3, &e.mtu)
+}
+
+func init() {
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/qdisc/fifo/dispatcher_mutex.go

@@ -0,0 +1,64 @@
+package fifo
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// Mutex is sync.Mutex with the correctness validator.
+type queueDispatcherMutex struct {
+	mu sync.Mutex
+}
+
+var queueDispatcherprefixIndex *locking.MutexClass
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var queueDispatcherlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type queueDispatcherlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *queueDispatcherMutex) Lock() {
+	locking.AddGLock(queueDispatcherprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *queueDispatcherMutex) NestedLock(i queueDispatcherlockNameIndex) {
+	locking.AddGLock(queueDispatcherprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *queueDispatcherMutex) Unlock() {
+	locking.DelGLock(queueDispatcherprefixIndex, -1)
+	m.mu.Unlock()
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *queueDispatcherMutex) NestedUnlock(i queueDispatcherlockNameIndex) {
+	locking.DelGLock(queueDispatcherprefixIndex, int(i))
+	m.mu.Unlock()
+}
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func queueDispatcherinitLockNames() {}
+
+func init() {
+	queueDispatcherinitLockNames()
+	queueDispatcherprefixIndex = locking.NewMutexClass(reflect.TypeOf(queueDispatcherMutex{}), queueDispatcherlockNames)
+}

pkg/tcpip/link/qdisc/fifo/fifo.go

@@ -0,0 +1,158 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fifo provides the implementation of FIFO queuing discipline that
+// queues all outbound packets and asynchronously dispatches them to the
+// lower link endpoint in the order that they were queued.
+package fifo
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/sleep"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+var _ stack.QueueingDiscipline = (*discipline)(nil)
+
+const (
+	// BatchSize is the number of packets to write in each syscall. It is 47
+	// because when GVisorGSO is in use then a single 65KB TCP segment can get
+	// split into 46 segments of 1420 bytes and a single 216 byte segment.
+	BatchSize = 47
+
+	qDiscClosed = 1
+)
+
+// discipline represents a QueueingDiscipline which implements a FIFO queue for
+// all outgoing packets. discipline can have 1 or more underlying
+// queueDispatchers. All outgoing packets are consistently hashed to a single
+// underlying queue using the PacketBuffer.Hash if set, otherwise all packets
+// are queued to the first queue to avoid reordering in case of missing hash.
+//
+// +stateify savable
+type discipline struct {
+	wg          sync.WaitGroup `state:"nosave"`
+	dispatchers []queueDispatcher
+
+	closed atomicbitops.Int32
+}
+
+// queueDispatcher is responsible for dispatching all outbound packets in its
+// queue. It will also smartly batch packets when possible and write them
+// through the lower LinkWriter.
+//
+// +stateify savable
+type queueDispatcher struct {
+	lower stack.LinkWriter
+
+	mu queueDispatcherMutex `state:"nosave"`
+	// +checklocks:mu
+	queue packetBufferCircularList
+
+	newPacketWaker sleep.Waker `state:"nosave"`
+	closeWaker     sleep.Waker `state:"nosave"`
+}
+
+// New creates a new fifo queuing discipline with the n queues with maximum
+// capacity of queueLen.
+//
+// +checklocksignore: we don't have to hold locks during initialization.
+func New(lower stack.LinkWriter, n int, queueLen int) stack.QueueingDiscipline {
+	d := &discipline{
+		dispatchers: make([]queueDispatcher, n),
+	}
+	// Create the required dispatchers
+	for i := range d.dispatchers {
+		qd := &d.dispatchers[i]
+		qd.lower = lower
+		qd.queue.init(queueLen)
+
+		d.wg.Add(1)
+		go func() {
+			defer d.wg.Done()
+			qd.dispatchLoop()
+		}()
+	}
+	return d
+}
+
+func (qd *queueDispatcher) dispatchLoop() {
+	s := sleep.Sleeper{}
+	s.AddWaker(&qd.newPacketWaker)
+	s.AddWaker(&qd.closeWaker)
+	defer s.Done()
+
+	var batch stack.PacketBufferList
+	for {
+		switch w := s.Fetch(true); w {
+		case &qd.newPacketWaker:
+		case &qd.closeWaker:
+			qd.mu.Lock()
+			for p := qd.queue.removeFront(); p != nil; p = qd.queue.removeFront() {
+				p.DecRef()
+			}
+			qd.queue.decRef()
+			qd.mu.Unlock()
+			return
+		default:
+			panic("unknown waker")
+		}
+		qd.mu.Lock()
+		for pkt := qd.queue.removeFront(); pkt != nil; pkt = qd.queue.removeFront() {
+			batch.PushBack(pkt)
+			if batch.Len() < BatchSize && !qd.queue.isEmpty() {
+				continue
+			}
+			qd.mu.Unlock()
+			_, _ = qd.lower.WritePackets(batch)
+			batch.Reset()
+			qd.mu.Lock()
+		}
+		qd.mu.Unlock()
+	}
+}
+
+// WritePacket implements stack.QueueingDiscipline.WritePacket.
+//
+// The packet must have the following fields populated:
+//   - pkt.EgressRoute
+//   - pkt.GSOOptions
+//   - pkt.NetworkProtocolNumber
+func (d *discipline) WritePacket(pkt *stack.PacketBuffer) tcpip.Error {
+	if d.closed.Load() == qDiscClosed {
+		return &tcpip.ErrClosedForSend{}
+	}
+	qd := &d.dispatchers[int(pkt.Hash)%len(d.dispatchers)]
+	qd.mu.Lock()
+	haveSpace := qd.queue.hasSpace()
+	if haveSpace {
+		qd.queue.pushBack(pkt.IncRef())
+	}
+	qd.mu.Unlock()
+	if !haveSpace {
+		return &tcpip.ErrNoBufferSpace{}
+	}
+	qd.newPacketWaker.Assert()
+	return nil
+}
+
+func (d *discipline) Close() {
+	d.closed.Store(qDiscClosed)
+	for i := range d.dispatchers {
+		d.dispatchers[i].closeWaker.Assert()
+	}
+	d.wg.Wait()
+}

pkg/tcpip/link/qdisc/fifo/fifo_state_autogen.go

@@ -0,0 +1,102 @@
+// automatically generated by stateify.
+
+package fifo
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (d *discipline) StateTypeName() string {
+	return "pkg/tcpip/link/qdisc/fifo.discipline"
+}
+
+func (d *discipline) StateFields() []string {
+	return []string{
+		"dispatchers",
+		"closed",
+	}
+}
+
+func (d *discipline) beforeSave() {}
+
+// +checklocksignore
+func (d *discipline) StateSave(stateSinkObject state.Sink) {
+	d.beforeSave()
+	stateSinkObject.Save(0, &d.dispatchers)
+	stateSinkObject.Save(1, &d.closed)
+}
+
+func (d *discipline) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (d *discipline) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &d.dispatchers)
+	stateSourceObject.Load(1, &d.closed)
+}
+
+func (qd *queueDispatcher) StateTypeName() string {
+	return "pkg/tcpip/link/qdisc/fifo.queueDispatcher"
+}
+
+func (qd *queueDispatcher) StateFields() []string {
+	return []string{
+		"lower",
+		"queue",
+	}
+}
+
+func (qd *queueDispatcher) beforeSave() {}
+
+// +checklocksignore
+func (qd *queueDispatcher) StateSave(stateSinkObject state.Sink) {
+	qd.beforeSave()
+	stateSinkObject.Save(0, &qd.lower)
+	stateSinkObject.Save(1, &qd.queue)
+}
+
+func (qd *queueDispatcher) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (qd *queueDispatcher) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &qd.lower)
+	stateSourceObject.Load(1, &qd.queue)
+}
+
+func (pl *packetBufferCircularList) StateTypeName() string {
+	return "pkg/tcpip/link/qdisc/fifo.packetBufferCircularList"
+}
+
+func (pl *packetBufferCircularList) StateFields() []string {
+	return []string{
+		"pbs",
+		"head",
+		"size",
+	}
+}
+
+func (pl *packetBufferCircularList) beforeSave() {}
+
+// +checklocksignore
+func (pl *packetBufferCircularList) StateSave(stateSinkObject state.Sink) {
+	pl.beforeSave()
+	stateSinkObject.Save(0, &pl.pbs)
+	stateSinkObject.Save(1, &pl.head)
+	stateSinkObject.Save(2, &pl.size)
+}
+
+func (pl *packetBufferCircularList) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (pl *packetBufferCircularList) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &pl.pbs)
+	stateSourceObject.Load(1, &pl.head)
+	stateSourceObject.Load(2, &pl.size)
+}
+
+func init() {
+	state.Register((*discipline)(nil))
+	state.Register((*queueDispatcher)(nil))
+	state.Register((*packetBufferCircularList)(nil))
+}

pkg/tcpip/link/qdisc/fifo/packet_buffer_circular_list.go

@@ -0,0 +1,93 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at //
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fifo
+
+import "github.com/sagernet/gvisor/pkg/tcpip/stack"
+
+// packetBufferCircularList is a slice-backed circular list. All operations are
+// O(1) unless otherwise noted. It only allocates once, during the call to
+// init().
+//
+// Users should call init() before using packetBufferCircularList.
+//
+// +stateify savable
+type packetBufferCircularList struct {
+	pbs  []*stack.PacketBuffer
+	head int
+	size int
+}
+
+// init initializes the list with the given size.
+func (pl *packetBufferCircularList) init(size int) {
+	pl.pbs = make([]*stack.PacketBuffer, size)
+}
+
+// length returns the number of elements in the list.
+//
+//go:nosplit
+func (pl *packetBufferCircularList) length() int {
+	return pl.size
+}
+
+// hasSpace returns whether there is space left in the list.
+//
+//go:nosplit
+func (pl *packetBufferCircularList) hasSpace() bool {
+	return pl.size < len(pl.pbs)
+}
+
+// isEmpty returns whether the list is empty.
+//
+//go:nosplit
+func (pl *packetBufferCircularList) isEmpty() bool {
+	return pl.size == 0
+}
+
+// pushBack inserts the PacketBuffer at the end of the list.
+//
+// Users must check beforehand that there is space via a call to hasSpace().
+// Failing to do so may clobber existing entries.
+//
+//go:nosplit
+func (pl *packetBufferCircularList) pushBack(pb *stack.PacketBuffer) {
+	next := (pl.head + pl.size) % len(pl.pbs)
+	pl.pbs[next] = pb
+	pl.size++
+}
+
+// removeFront returns the first element of the list or nil.
+//
+//go:nosplit
+func (pl *packetBufferCircularList) removeFront() *stack.PacketBuffer {
+	if pl.isEmpty() {
+		return nil
+	}
+	ret := pl.pbs[pl.head]
+	pl.pbs[pl.head] = nil
+	pl.head = (pl.head + 1) % len(pl.pbs)
+	pl.size--
+	return ret
+}
+
+// decRef decreases the reference count on each stack.PacketBuffer stored in
+// the list.
+//
+// NOTE: runs in O(n) time.
+//
+//go:nosplit
+func (pl *packetBufferCircularList) decRef() {
+	for i := 0; i < pl.size; i++ {
+		pl.pbs[(pl.head+i)%len(pl.pbs)].DecRef()
+	}
+}

pkg/tcpip/link/sharedmem/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package sharedmem
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/sharedmem/pipe/pipe.go

@@ -0,0 +1,80 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pipe implements a shared memory ring buffer on which a single reader
+// and a single writer can operate (read/write) concurrently. The ring buffer
+// allows for data of different sizes to be written, and preserves the boundary
+// of the written data.
+//
+// Example usage is as follows:
+//
+//	wb := t.Push(20)
+//	// Write data to wb.
+//	t.Flush()
+//
+//	rb := r.Pull()
+//	// Do something with data in rb.
+//	t.Flush()
+package pipe
+
+import (
+	"math"
+)
+
+const (
+	jump           uint64 = math.MaxUint32 + 1
+	offsetMask     uint64 = math.MaxUint32
+	revolutionMask uint64 = ^offsetMask
+
+	sizeOfSlotHeader        = 8 // sizeof(uint64)
+	slotFree         uint64 = 1 << 63
+	slotSizeMask     uint64 = math.MaxUint32
+)
+
+// payloadToSlotSize calculates the total size of a slot based on its payload
+// size. The  total size is the header size, plus the payload size, plus padding
+// if necessary to make the total size a multiple of sizeOfSlotHeader.
+func payloadToSlotSize(payloadSize uint64) uint64 {
+	s := sizeOfSlotHeader + payloadSize
+	return (s + sizeOfSlotHeader - 1) &^ (sizeOfSlotHeader - 1)
+}
+
+// slotToPayloadSize calculates the payload size of a slot based on the total
+// size of the slot. This is only meant to be used when creating slots that
+// don't carry information (e.g., free slots or wrap slots).
+func slotToPayloadSize(offset uint64) uint64 {
+	return offset - sizeOfSlotHeader
+}
+
+// pipe is a basic data structure used by both (transmit & receive) ends of a
+// pipe. Indices into this pipe are split into two fields: offset, which counts
+// the number of bytes from the beginning of the buffer, and revolution, which
+// counts the number of times the index has wrapped around.
+//
+// +stateify savable
+type pipe struct {
+	buffer []byte
+}
+
+// init initializes the pipe buffer such that its size is a multiple of the size
+// of the slot header.
+func (p *pipe) init(b []byte) {
+	p.buffer = b[:len(b)&^(sizeOfSlotHeader-1)]
+}
+
+// data returns a section of the buffer starting at the given index (which may
+// include revolution information) and with the given size.
+func (p *pipe) data(idx uint64, size uint64) []byte {
+	return p.buffer[(idx&offsetMask)+sizeOfSlotHeader:][:size]
+}

pkg/tcpip/link/sharedmem/pipe/pipe_state_autogen.go

@@ -0,0 +1,111 @@
+// automatically generated by stateify.
+
+package pipe
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (p *pipe) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem/pipe.pipe"
+}
+
+func (p *pipe) StateFields() []string {
+	return []string{
+		"buffer",
+	}
+}
+
+func (p *pipe) beforeSave() {}
+
+// +checklocksignore
+func (p *pipe) StateSave(stateSinkObject state.Sink) {
+	p.beforeSave()
+	stateSinkObject.Save(0, &p.buffer)
+}
+
+func (p *pipe) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (p *pipe) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &p.buffer)
+}
+
+func (r *Rx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem/pipe.Rx"
+}
+
+func (r *Rx) StateFields() []string {
+	return []string{
+		"p",
+		"tail",
+		"head",
+	}
+}
+
+func (r *Rx) beforeSave() {}
+
+// +checklocksignore
+func (r *Rx) StateSave(stateSinkObject state.Sink) {
+	r.beforeSave()
+	stateSinkObject.Save(0, &r.p)
+	stateSinkObject.Save(1, &r.tail)
+	stateSinkObject.Save(2, &r.head)
+}
+
+func (r *Rx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (r *Rx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &r.p)
+	stateSourceObject.Load(1, &r.tail)
+	stateSourceObject.Load(2, &r.head)
+}
+
+func (t *Tx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem/pipe.Tx"
+}
+
+func (t *Tx) StateFields() []string {
+	return []string{
+		"p",
+		"maxPayloadSize",
+		"head",
+		"tail",
+		"next",
+		"tailHeader",
+	}
+}
+
+func (t *Tx) beforeSave() {}
+
+// +checklocksignore
+func (t *Tx) StateSave(stateSinkObject state.Sink) {
+	t.beforeSave()
+	stateSinkObject.Save(0, &t.p)
+	stateSinkObject.Save(1, &t.maxPayloadSize)
+	stateSinkObject.Save(2, &t.head)
+	stateSinkObject.Save(3, &t.tail)
+	stateSinkObject.Save(4, &t.next)
+	stateSinkObject.Save(5, &t.tailHeader)
+}
+
+func (t *Tx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (t *Tx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &t.p)
+	stateSourceObject.Load(1, &t.maxPayloadSize)
+	stateSourceObject.Load(2, &t.head)
+	stateSourceObject.Load(3, &t.tail)
+	stateSourceObject.Load(4, &t.next)
+	stateSourceObject.Load(5, &t.tailHeader)
+}
+
+func init() {
+	state.Register((*pipe)(nil))
+	state.Register((*Rx)(nil))
+	state.Register((*Tx)(nil))
+}

pkg/tcpip/link/sharedmem/pipe/pipe_unsafe.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipe
+
+import (
+	"unsafe"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+)
+
+func (p *pipe) write(idx uint64, v uint64) {
+	ptr := (*uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0]))
+	*ptr = v
+}
+
+func (p *pipe) writeAtomic(idx uint64, v uint64) {
+	ptr := (*atomicbitops.Uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0]))
+	ptr.Store(v)
+}
+
+func (p *pipe) readAtomic(idx uint64) uint64 {
+	ptr := (*atomicbitops.Uint64)(unsafe.Pointer(&p.buffer[idx&offsetMask:][:8][0]))
+	return ptr.Load()
+}

pkg/tcpip/link/sharedmem/pipe/pipe_unsafe_state_autogen.go

@@ -0,0 +1,3 @@
+// automatically generated by stateify.
+
+package pipe

pkg/tcpip/link/sharedmem/pipe/rx.go

@@ -0,0 +1,108 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipe
+
+// Rx is the receive side of the shared memory ring buffer.
+//
+// +stateify savable
+type Rx struct {
+	p pipe
+
+	tail uint64
+	head uint64
+}
+
+// Init initializes the receive end of the pipe. In the initial state, the next
+// slot to be inspected is the very first one.
+func (r *Rx) Init(b []byte) {
+	r.p.init(b)
+	r.tail = 0xfffffffe * jump
+	r.head = r.tail
+}
+
+// Pull reads the next buffer from the pipe, returning nil if there isn't one
+// currently available.
+//
+// The returned slice is available until Flush() is next called. After that, it
+// must not be touched.
+func (r *Rx) Pull() []byte {
+	if r.head == r.tail+jump {
+		// We've already pulled the whole pipe.
+		return nil
+	}
+
+	header := r.p.readAtomic(r.head)
+	if header&slotFree != 0 {
+		// The next slot is free, we can't pull it yet.
+		return nil
+	}
+
+	payloadSize := header & slotSizeMask
+	newHead := r.head + payloadToSlotSize(payloadSize)
+	headWrap := (r.head & revolutionMask) | uint64(len(r.p.buffer))
+
+	// Check if this is a wrapping slot. If that's the case, it carries no
+	// data, so we just skip it and try again from the first slot.
+	if int64(newHead-headWrap) >= 0 {
+		// If newHead passes the tail, the pipe is either damaged or the
+		// RX view of the pipe has completely wrapped without an
+		// intervening flush.
+		if int64(newHead-(r.tail+jump)) > 0 {
+			return nil
+		}
+		// The pipe is damaged if newHead doesn't point to the start of
+		// the ring.
+		if newHead&offsetMask != 0 {
+			return nil
+		}
+
+		if r.tail == r.head {
+			// If this is the first pull since the last Flush()
+			// call, we flush the state so that the sender can use
+			// this space if it needs to.
+			r.p.writeAtomic(r.head, slotFree|slotToPayloadSize(newHead-r.head))
+			r.tail = newHead
+		}
+
+		r.head = newHead
+		return r.Pull()
+	}
+
+	// Grab the buffer before updating r.head.
+	b := r.p.data(r.head, payloadSize)
+	r.head = newHead
+	return b
+}
+
+// Flush tells the transmitter that all buffers pulled since the last Flush()
+// have been used, so the transmitter is free to used their slots for further
+// transmission.
+func (r *Rx) Flush() {
+	if r.head == r.tail {
+		return
+	}
+	r.p.writeAtomic(r.tail, slotFree|slotToPayloadSize(r.head-r.tail))
+	r.tail = r.head
+}
+
+// Abort unpulls any pulled buffers.
+func (r *Rx) Abort() {
+	r.head = r.tail
+}
+
+// Bytes returns the byte slice on which the pipe operates.
+func (r *Rx) Bytes() []byte {
+	return r.p.buffer
+}

pkg/tcpip/link/sharedmem/pipe/tx.go

@@ -0,0 +1,164 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipe
+
+// Tx is the transmit side of the shared memory ring buffer.
+//
+// +stateify savable
+type Tx struct {
+	p              pipe
+	maxPayloadSize uint64
+
+	head uint64
+	tail uint64
+	next uint64
+
+	tailHeader uint64
+}
+
+// Init initializes the transmit end of the pipe. In the initial state, the next
+// slot to be written is the very first one, and the transmitter has the whole
+// ring buffer available to it.
+func (t *Tx) Init(b []byte) {
+	t.p.init(b)
+	// maxPayloadSize excludes the header of the payload, and the header
+	// of the wrapping message.
+	t.maxPayloadSize = uint64(len(t.p.buffer)) - 2*sizeOfSlotHeader
+	t.tail = 0xfffffffe * jump
+	t.next = t.tail
+	t.head = t.tail + jump
+	t.p.write(t.tail, slotFree)
+}
+
+// Capacity determines how many records of the given size can be written to the
+// pipe before it fills up.
+func (t *Tx) Capacity(recordSize uint64) uint64 {
+	available := uint64(len(t.p.buffer)) - sizeOfSlotHeader
+	entryLen := payloadToSlotSize(recordSize)
+	return available / entryLen
+}
+
+// Push reserves "payloadSize" bytes for transmission in the pipe. The caller
+// populates the returned slice with the data to be transferred and enventually
+// calls Flush() to make the data visible to the reader, or Abort() to make the
+// pipe forget all Push() calls since the last Flush().
+//
+// The returned slice is available until Flush() or Abort() is next called.
+// After that, it must not be touched.
+func (t *Tx) Push(payloadSize uint64) []byte {
+	// Fail request if we know we will never have enough room.
+	if payloadSize > t.maxPayloadSize {
+		return nil
+	}
+
+	// True if TxPipe currently has a pushed message, i.e., it is not
+	// Flush()'ed.
+	messageAhead := t.next != t.tail
+	totalLen := payloadToSlotSize(payloadSize)
+	newNext := t.next + totalLen
+	nextWrap := (t.next & revolutionMask) | uint64(len(t.p.buffer))
+	if int64(newNext-nextWrap) >= 0 {
+		// The new buffer would overflow the pipe, so we push a wrapping
+		// slot, then try to add the actual slot to the front of the
+		// pipe.
+		newNext = (newNext & revolutionMask) + jump
+		if !t.reclaim(newNext) {
+			return nil
+		}
+		wrappingPayloadSize := slotToPayloadSize(newNext - t.next)
+		oldNext := t.next
+		t.next = newNext
+		if messageAhead {
+			t.p.write(oldNext, wrappingPayloadSize)
+		} else {
+			t.tailHeader = wrappingPayloadSize
+			t.Flush()
+		}
+		return t.Push(payloadSize)
+	}
+
+	// Check that we have enough room for the buffer.
+	if !t.reclaim(newNext) {
+		return nil
+	}
+
+	if messageAhead {
+		t.p.write(t.next, payloadSize)
+	} else {
+		t.tailHeader = payloadSize
+	}
+
+	// Grab the buffer before updating t.next.
+	b := t.p.data(t.next, payloadSize)
+	t.next = newNext
+
+	return b
+}
+
+// reclaim attempts to advance the head until at least newNext. If the head is
+// already at or beyond newNext, nothing happens and true is returned; otherwise
+// it tries to reclaim slots that have already been consumed by the receive end
+// of the pipe (they will be marked as free) and returns a boolean indicating
+// whether it was successful in reclaiming enough slots.
+func (t *Tx) reclaim(newNext uint64) bool {
+	for int64(newNext-t.head) > 0 {
+		// Can't reclaim if slot is not free.
+		header := t.p.readAtomic(t.head)
+		if header&slotFree == 0 {
+			return false
+		}
+
+		payloadSize := header & slotSizeMask
+		newHead := t.head + payloadToSlotSize(payloadSize)
+
+		// Check newHead is within bounds and valid.
+		if int64(newHead-t.tail) > int64(jump) || newHead&offsetMask >= uint64(len(t.p.buffer)) {
+			return false
+		}
+
+		t.head = newHead
+	}
+
+	return true
+}
+
+// Abort causes all Push() calls since the last Flush() to be forgotten and
+// therefore they will not be made visible to the receiver.
+func (t *Tx) Abort() {
+	t.next = t.tail
+}
+
+// Flush causes all buffers pushed since the last Flush() [or Abort(), whichever
+// is the most recent] to be made visible to the receiver.
+func (t *Tx) Flush() {
+	if t.next == t.tail {
+		// Nothing to do if there are no pushed buffers.
+		return
+	}
+
+	if t.next != t.head {
+		// The receiver will spin in t.next, so we must make sure that
+		// the slotFree bit is set.
+		t.p.write(t.next, slotFree)
+	}
+
+	t.p.writeAtomic(t.tail, t.tailHeader)
+	t.tail = t.next
+}
+
+// Bytes returns the byte slice on which the pipe operates.
+func (t *Tx) Bytes() []byte {
+	return t.p.buffer
+}

pkg/tcpip/link/sharedmem/queue/queue_state_autogen.go

@@ -0,0 +1,3 @@
+// automatically generated by stateify.
+
+package queue

pkg/tcpip/link/sharedmem/queue/rx.go

@@ -0,0 +1,226 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package queue provides the implementation of transmit and receive queues
+// based on shared memory ring buffers.
+package queue
+
+import (
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+)
+
+const (
+	// Offsets within a posted buffer.
+	postedOffset           = 0
+	postedSize             = 8
+	postedRemainingInGroup = 12
+	postedUserData         = 16
+	postedID               = 24
+
+	sizeOfPostedBuffer = 32
+
+	// Offsets within a received packet header.
+	consumedPacketSize     = 0
+	consumedPacketReserved = 4
+
+	sizeOfConsumedPacketHeader = 8
+
+	// Offsets within a consumed buffer.
+	consumedOffset   = 0
+	consumedSize     = 8
+	consumedUserData = 12
+	consumedID       = 20
+
+	sizeOfConsumedBuffer = 28
+
+	// The following are the allowed states of the shared data area.
+	// EventFDUinitialized is the value stored at the start of the shared data
+	// region when it hasn't been initialized.
+	EventFDUninitialized = 0
+	// EventFDDisabled is the value stored at the start of the shared data region
+	// when notifications using eventFD has been disabled.
+	EventFDDisabled = 1
+	// EventFDEnabled is the value stored at the start of the shared data region
+	// when eventFD should be notified as the peer might be blocked waiting on
+	// notifications.
+	EventFDEnabled = 2
+)
+
+// RxBuffer is the descriptor of a receive buffer.
+type RxBuffer struct {
+	Offset   uint64
+	Size     uint32
+	ID       uint64
+	UserData uint64
+}
+
+// Rx is a receive queue. It is implemented with one tx and one rx pipe: the tx
+// pipe is used to "post" buffers, while the rx pipe is used to receive packets
+// whose contents have been written to previously posted buffers.
+//
+// This struct is thread-compatible.
+type Rx struct {
+	tx                 pipe.Tx
+	rx                 pipe.Rx
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// Init initializes the receive queue with the given pipes, and shared state
+// pointer -- the latter is used to enable/disable eventfd notifications.
+func (r *Rx) Init(tx, rx []byte, sharedEventFDState *atomicbitops.Uint32) {
+	r.sharedEventFDState = sharedEventFDState
+	r.tx.Init(tx)
+	r.rx.Init(rx)
+}
+
+// EnableNotification updates the shared state such that the peer will notify
+// the eventfd when there are packets to be dequeued.
+func (r *Rx) EnableNotification() {
+	r.sharedEventFDState.Store(EventFDEnabled)
+}
+
+// DisableNotification updates the shared state such that the peer will not
+// notify the eventfd.
+func (r *Rx) DisableNotification() {
+	r.sharedEventFDState.Store(EventFDDisabled)
+}
+
+// PostedBuffersLimit returns the maximum number of buffers that can be posted
+// before the tx queue fills up.
+func (r *Rx) PostedBuffersLimit() uint64 {
+	return r.tx.Capacity(sizeOfPostedBuffer)
+}
+
+// PostBuffers makes the given buffers available for receiving data from the
+// peer. Once they are posted, the peer is free to write to them and will
+// eventually post them back for consumption.
+func (r *Rx) PostBuffers(buffers []RxBuffer) bool {
+	for i := range buffers {
+		b := r.tx.Push(sizeOfPostedBuffer)
+		if b == nil {
+			r.tx.Abort()
+			return false
+		}
+
+		pb := &buffers[i]
+		binary.LittleEndian.PutUint64(b[postedOffset:], pb.Offset)
+		binary.LittleEndian.PutUint32(b[postedSize:], pb.Size)
+		binary.LittleEndian.PutUint32(b[postedRemainingInGroup:], 0)
+		binary.LittleEndian.PutUint64(b[postedUserData:], pb.UserData)
+		binary.LittleEndian.PutUint64(b[postedID:], pb.ID)
+	}
+
+	r.tx.Flush()
+	return true
+}
+
+// Dequeue receives buffers that have been previously posted by PostBuffers()
+// and that have been filled by the peer and posted back.
+//
+// This is similar to append() in that new buffers are appended to "bufs", with
+// reallocation only if "bufs" doesn't have enough capacity.
+func (r *Rx) Dequeue(bufs []RxBuffer) ([]RxBuffer, uint32) {
+	for {
+		outBufs := bufs
+		// Pull the next descriptor from the rx pipe.
+		b := r.rx.Pull()
+		if b == nil {
+			return bufs, 0
+		}
+
+		if len(b) < sizeOfConsumedPacketHeader {
+			log.Warningf("Ignoring packet header: size (%v) is less than header size (%v)", len(b), sizeOfConsumedPacketHeader)
+			r.rx.Flush()
+			continue
+		}
+
+		totalDataSize := binary.LittleEndian.Uint32(b[consumedPacketSize:])
+
+		// Calculate the number of buffer descriptors and copy them
+		// over to the output.
+		count := (len(b) - sizeOfConsumedPacketHeader) / sizeOfConsumedBuffer
+		offset := sizeOfConsumedPacketHeader
+		buffersSize := uint32(0)
+		for i := count; i > 0; i-- {
+			s := binary.LittleEndian.Uint32(b[offset+consumedSize:])
+			buffersSize += s
+			if buffersSize < s {
+				// The buffer size overflows an unsigned 32-bit
+				// integer, so break out and force it to be
+				// ignored.
+				totalDataSize = 1
+				buffersSize = 0
+				break
+			}
+
+			outBufs = append(outBufs, RxBuffer{
+				Offset: binary.LittleEndian.Uint64(b[offset+consumedOffset:]),
+				Size:   s,
+				ID:     binary.LittleEndian.Uint64(b[offset+consumedID:]),
+			})
+
+			offset += sizeOfConsumedBuffer
+		}
+
+		r.rx.Flush()
+
+		if buffersSize < totalDataSize {
+			// The descriptor is corrupted, ignore it.
+			log.Warningf("Ignoring packet: actual data size (%v) less than expected size (%v)", buffersSize, totalDataSize)
+			continue
+		}
+
+		return outBufs, totalDataSize
+	}
+}
+
+// Bytes returns the byte slices on which the queue operates.
+func (r *Rx) Bytes() (tx, rx []byte) {
+	return r.tx.Bytes(), r.rx.Bytes()
+}
+
+// DecodeRxBufferHeader decodes the header of a buffer posted on an rx queue.
+func DecodeRxBufferHeader(b []byte) RxBuffer {
+	return RxBuffer{
+		Offset:   binary.LittleEndian.Uint64(b[postedOffset:]),
+		Size:     binary.LittleEndian.Uint32(b[postedSize:]),
+		ID:       binary.LittleEndian.Uint64(b[postedID:]),
+		UserData: binary.LittleEndian.Uint64(b[postedUserData:]),
+	}
+}
+
+// RxCompletionSize returns the number of bytes needed to encode an rx
+// completion containing "count" buffers.
+func RxCompletionSize(count int) uint64 {
+	return sizeOfConsumedPacketHeader + uint64(count)*sizeOfConsumedBuffer
+}
+
+// EncodeRxCompletion encodes an rx completion header.
+func EncodeRxCompletion(b []byte, size, reserved uint32) {
+	binary.LittleEndian.PutUint32(b[consumedPacketSize:], size)
+	binary.LittleEndian.PutUint32(b[consumedPacketReserved:], reserved)
+}
+
+// EncodeRxCompletionBuffer encodes the i-th rx completion buffer header.
+func EncodeRxCompletionBuffer(b []byte, i int, rxb RxBuffer) {
+	b = b[RxCompletionSize(i):]
+	binary.LittleEndian.PutUint64(b[consumedOffset:], rxb.Offset)
+	binary.LittleEndian.PutUint32(b[consumedSize:], rxb.Size)
+	binary.LittleEndian.PutUint64(b[consumedUserData:], rxb.UserData)
+	binary.LittleEndian.PutUint64(b[consumedID:], rxb.ID)
+}

pkg/tcpip/link/sharedmem/queue/tx.go

@@ -0,0 +1,161 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package queue
+
+import (
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+)
+
+const (
+	// Offsets within a packet header.
+	packetID       = 0
+	packetSize     = 8
+	packetReserved = 12
+
+	sizeOfPacketHeader = 16
+
+	// Offsets with a buffer descriptor
+	bufferOffset = 0
+	bufferSize   = 8
+
+	sizeOfBufferDescriptor = 12
+)
+
+// TxBuffer is the descriptor of a transmit buffer.
+type TxBuffer struct {
+	Next   *TxBuffer
+	Offset uint64
+	Size   uint32
+}
+
+// Tx is a transmit queue. It is implemented with one tx and one rx pipe: the
+// tx pipe is used to request the transmission of packets, while the rx pipe
+// is used to receive which transmissions have completed.
+//
+// This struct is thread-compatible.
+type Tx struct {
+	tx                 pipe.Tx
+	rx                 pipe.Rx
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// Init initializes the transmit queue with the given pipes.
+func (t *Tx) Init(tx, rx []byte, sharedEventFDState *atomicbitops.Uint32) {
+	t.tx.Init(tx)
+	t.rx.Init(rx)
+	t.sharedEventFDState = sharedEventFDState
+}
+
+// NotificationsEnabled returns true if eventFD should be used to notify the
+// peer of events (eg. packet transmit etc).
+func (t *Tx) NotificationsEnabled() bool {
+	// Notifications are considered enabled unless explicitly disabled.
+	return t.sharedEventFDState.Load() != EventFDDisabled
+}
+
+// Enqueue queues the given linked list of buffers for transmission as one
+// packet. While it is queued, the caller must not modify them.
+func (t *Tx) Enqueue(id uint64, totalDataLen, bufferCount uint32, buffer *TxBuffer) bool {
+	// Reserve room in the tx pipe.
+	totalLen := sizeOfPacketHeader + uint64(bufferCount)*sizeOfBufferDescriptor
+
+	b := t.tx.Push(totalLen)
+	if b == nil {
+		return false
+	}
+
+	// Initialize the packet and buffer descriptors.
+	binary.LittleEndian.PutUint64(b[packetID:], id)
+	binary.LittleEndian.PutUint32(b[packetSize:], totalDataLen)
+	binary.LittleEndian.PutUint32(b[packetReserved:], 0)
+
+	offset := sizeOfPacketHeader
+	for i := bufferCount; i != 0; i-- {
+		binary.LittleEndian.PutUint64(b[offset+bufferOffset:], buffer.Offset)
+		binary.LittleEndian.PutUint32(b[offset+bufferSize:], buffer.Size)
+		offset += sizeOfBufferDescriptor
+		buffer = buffer.Next
+	}
+
+	t.tx.Flush()
+
+	return true
+}
+
+// CompletedPacket returns the id of the last completed transmission. The
+// returned id, if any, refers to a value passed on a previous call to
+// Enqueue().
+func (t *Tx) CompletedPacket() (id uint64, ok bool) {
+	for {
+		b := t.rx.Pull()
+		if b == nil {
+			return 0, false
+		}
+
+		if len(b) != 8 {
+			t.rx.Flush()
+			log.Warningf("Ignoring completed packet: size (%v) is less than expected (%v)", len(b), 8)
+			continue
+		}
+
+		v := binary.LittleEndian.Uint64(b)
+
+		t.rx.Flush()
+
+		return v, true
+	}
+}
+
+// Bytes returns the byte slices on which the queue operates.
+func (t *Tx) Bytes() (tx, rx []byte) {
+	return t.tx.Bytes(), t.rx.Bytes()
+}
+
+// TxPacketInfo holds information about a packet sent on a tx queue.
+type TxPacketInfo struct {
+	ID          uint64
+	Size        uint32
+	Reserved    uint32
+	BufferCount int
+}
+
+// DecodeTxPacketHeader decodes the header of a packet sent over a tx queue.
+func DecodeTxPacketHeader(b []byte) TxPacketInfo {
+	return TxPacketInfo{
+		ID:          binary.LittleEndian.Uint64(b[packetID:]),
+		Size:        binary.LittleEndian.Uint32(b[packetSize:]),
+		Reserved:    binary.LittleEndian.Uint32(b[packetReserved:]),
+		BufferCount: (len(b) - sizeOfPacketHeader) / sizeOfBufferDescriptor,
+	}
+}
+
+// DecodeTxBufferHeader decodes the header of the i-th buffer of a packet sent
+// over a tx queue.
+func DecodeTxBufferHeader(b []byte, i int) TxBuffer {
+	b = b[sizeOfPacketHeader+i*sizeOfBufferDescriptor:]
+	return TxBuffer{
+		Offset: binary.LittleEndian.Uint64(b[bufferOffset:]),
+		Size:   binary.LittleEndian.Uint32(b[bufferSize:]),
+	}
+}
+
+// EncodeTxCompletion encodes a tx completion header.
+func EncodeTxCompletion(b []byte, id uint64) {
+	binary.LittleEndian.PutUint64(b, id)
+}

pkg/tcpip/link/sharedmem/queuepair.go

@@ -0,0 +1,220 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// DefaultQueueDataSize is the size of the shared memory data region that
+	// holds the scatter/gather buffers.
+	DefaultQueueDataSize = 1 << 20 // 1MiB
+
+	// DefaultQueuePipeSize is the size of the pipe that holds the packet descriptors.
+	//
+	// Assuming each packet data is approximately 1280 bytes (IPv6 Minimum MTU)
+	// then we can hold approximately 1024*1024/1280 ~ 819 packets in the data
+	// area. Which means the pipe needs to be big enough to hold 819
+	// descriptors.
+	//
+	// Each descriptor is approximately 8 (slot descriptor in pipe) +
+	// 16 (packet descriptor) + 12 (for buffer descriptor) assuming each packet is
+	// stored in exactly 1 buffer descriptor (see queue/tx.go and pipe/tx.go.)
+	//
+	// Which means we need approximately 36*819 ~ 29 KiB to store all packet
+	// descriptors. We could go with a 32 KiB pipe but to give it some slack in
+	// how the upper layer may make use of the scatter gather buffers we double
+	// this to hold enough descriptors.
+	DefaultQueuePipeSize = 64 << 10 // 64KiB
+
+	// DefaultSharedDataSize is the size of the sharedData region used to
+	// enable/disable notifications.
+	DefaultSharedDataSize = 4 << 10 // 4KiB
+
+	// DefaultBufferSize is the size of each individual buffer that the data
+	// region is broken down into to hold packet data. Should be larger than
+	// 1500 + 14 (Ethernet header) + 10 (VirtIO header) to fit each packet
+	// in a single buffer.
+	DefaultBufferSize = 2048
+
+	// DefaultTmpDir is the path used to create the memory files if a path
+	// is not provided.
+	DefaultTmpDir = "/dev/shm"
+)
+
+// A QueuePair represents a pair of TX/RX queues.
+type QueuePair struct {
+	// txCfg is the QueueConfig to be used for transmit queue.
+	txCfg QueueConfig
+
+	// rxCfg is the QueueConfig to be used for receive queue.
+	rxCfg QueueConfig
+}
+
+// QueueOptions allows queue specific configuration to be specified when
+// creating a QueuePair.
+type QueueOptions struct {
+	// SharedMemPath is the path to use to create the shared memory backing
+	// files for the queue.
+	//
+	// If unspecified it defaults to "/dev/shm".
+	SharedMemPath string
+}
+
+// NewQueuePair creates a shared memory QueuePair.
+func NewQueuePair(opts QueueOptions) (*QueuePair, error) {
+	txCfg, err := createQueueFDs(opts.SharedMemPath, queueSizes{
+		dataSize:       DefaultQueueDataSize,
+		txPipeSize:     DefaultQueuePipeSize,
+		rxPipeSize:     DefaultQueuePipeSize,
+		sharedDataSize: DefaultSharedDataSize,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tx queue: %s", err)
+	}
+
+	rxCfg, err := createQueueFDs(opts.SharedMemPath, queueSizes{
+		dataSize:       DefaultQueueDataSize,
+		txPipeSize:     DefaultQueuePipeSize,
+		rxPipeSize:     DefaultQueuePipeSize,
+		sharedDataSize: DefaultSharedDataSize,
+	})
+	if err != nil {
+		closeFDs(txCfg)
+		return nil, fmt.Errorf("failed to create rx queue: %s", err)
+	}
+
+	return &QueuePair{
+		txCfg: txCfg,
+		rxCfg: rxCfg,
+	}, nil
+}
+
+// Close closes underlying tx/rx queue fds.
+func (q *QueuePair) Close() {
+	closeFDs(q.txCfg)
+	closeFDs(q.rxCfg)
+}
+
+// TXQueueConfig returns the QueueConfig for the receive queue.
+func (q *QueuePair) TXQueueConfig() QueueConfig {
+	return q.txCfg
+}
+
+// RXQueueConfig returns the QueueConfig for the transmit queue.
+func (q *QueuePair) RXQueueConfig() QueueConfig {
+	return q.rxCfg
+}
+
+type queueSizes struct {
+	dataSize       int64
+	txPipeSize     int64
+	rxPipeSize     int64
+	sharedDataSize int64
+}
+
+func createQueueFDs(sharedMemPath string, s queueSizes) (QueueConfig, error) {
+	success := false
+	var eventFD eventfd.Eventfd
+	var dataFD, txPipeFD, rxPipeFD, sharedDataFD int
+	defer func() {
+		if success {
+			return
+		}
+		closeFDs(QueueConfig{
+			EventFD:      eventFD,
+			DataFD:       dataFD,
+			TxPipeFD:     txPipeFD,
+			RxPipeFD:     rxPipeFD,
+			SharedDataFD: sharedDataFD,
+		})
+	}()
+	eventFD, err := eventfd.Create()
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("eventfd failed: %v", err)
+	}
+	dataFD, err = createFile(sharedMemPath, s.dataSize, false)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create dataFD: %s", err)
+	}
+	txPipeFD, err = createFile(sharedMemPath, s.txPipeSize, true)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create txPipeFD: %s", err)
+	}
+	rxPipeFD, err = createFile(sharedMemPath, s.rxPipeSize, true)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create rxPipeFD: %s", err)
+	}
+	sharedDataFD, err = createFile(sharedMemPath, s.sharedDataSize, false)
+	if err != nil {
+		return QueueConfig{}, fmt.Errorf("failed to create sharedDataFD: %s", err)
+	}
+	success = true
+	return QueueConfig{
+		EventFD:      eventFD,
+		DataFD:       dataFD,
+		TxPipeFD:     txPipeFD,
+		RxPipeFD:     rxPipeFD,
+		SharedDataFD: sharedDataFD,
+	}, nil
+}
+
+func createFile(sharedMemPath string, size int64, initQueue bool) (fd int, err error) {
+	tmpDir := DefaultTmpDir
+	if sharedMemPath != "" {
+		tmpDir = sharedMemPath
+	}
+	f, err := os.CreateTemp(tmpDir, "sharedmem_test")
+	if err != nil {
+		return -1, fmt.Errorf("TempFile failed: %v", err)
+	}
+	defer f.Close()
+	unix.Unlink(f.Name())
+
+	if initQueue {
+		// Write the "slot-free" flag in the initial queue.
+		if _, err := f.WriteAt([]byte{0, 0, 0, 0, 0, 0, 0, 0x80}, 0); err != nil {
+			return -1, fmt.Errorf("WriteAt failed: %v", err)
+		}
+	}
+
+	fd, err = unix.Dup(int(f.Fd()))
+	if err != nil {
+		return -1, fmt.Errorf("unix.Dup(%d) failed: %v", f.Fd(), err)
+	}
+
+	if err := unix.Ftruncate(fd, size); err != nil {
+		unix.Close(fd)
+		return -1, fmt.Errorf("ftruncate(%d, %d) failed: %v", fd, size, err)
+	}
+
+	return fd, nil
+}
+
+func closeFDs(c QueueConfig) {
+	unix.Close(c.DataFD)
+	c.EventFD.Close()
+	unix.Close(c.TxPipeFD)
+	unix.Close(c.RxPipeFD)
+	unix.Close(c.SharedDataFD)
+}

pkg/tcpip/link/sharedmem/rx.go

@@ -0,0 +1,152 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"golang.org/x/sys/unix"
+)
+
+// rx holds all state associated with an rx queue.
+type rx struct {
+	data       []byte
+	sharedData []byte
+	q          queue.Rx
+	eventFD    eventfd.Eventfd
+}
+
+// init initializes all state needed by the rx queue based on the information
+// provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (r *rx) init(mtu uint32, c *QueueConfig) error {
+	// Map in all buffers.
+	txPipe, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+
+	rxPipe, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		return err
+	}
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		return err
+	}
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		unix.Munmap(data)
+		return err
+	}
+
+	// Duplicate the eventFD so that caller can close it but we can still
+	// use it.
+	efd, err := c.EventFD.Dup()
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		unix.Munmap(data)
+		unix.Munmap(sharedData)
+		return err
+	}
+
+	// Initialize state based on buffers.
+	r.q.Init(txPipe, rxPipe, sharedDataPointer(sharedData))
+	r.data = data
+	r.eventFD = efd
+	r.sharedData = sharedData
+
+	return nil
+}
+
+// cleanup releases all resources allocated during init() except r.eventFD. It
+// must only be called if init() has previously succeeded.
+func (r *rx) cleanup() {
+	a, b := r.q.Bytes()
+	unix.Munmap(a)
+	unix.Munmap(b)
+
+	unix.Munmap(r.data)
+	unix.Munmap(r.sharedData)
+}
+
+// notify writes to the tx.eventFD to indicate to the peer that there is data to
+// be read.
+func (r *rx) notify() {
+	r.eventFD.Notify()
+}
+
+// postAndReceive posts the provided buffers (if any), and then tries to read
+// from the receive queue.
+//
+// Capacity permitting, it reuses the posted buffer slice to store the buffers
+// that were read as well.
+//
+// This function will block if there aren't any available packets.
+func (r *rx) postAndReceive(b []queue.RxBuffer, stopRequested *atomicbitops.Uint32) ([]queue.RxBuffer, uint32) {
+	// Post the buffers first. If we cannot post, sleep until we can. We
+	// never post more than will fit concurrently, so it's safe to wait
+	// until enough room is available.
+	if len(b) != 0 && !r.q.PostBuffers(b) {
+		r.q.EnableNotification()
+		for !r.q.PostBuffers(b) {
+			r.eventFD.Wait()
+			if stopRequested.Load() != 0 {
+				r.q.DisableNotification()
+				return nil, 0
+			}
+		}
+		r.q.DisableNotification()
+	}
+
+	// Read the next set of descriptors.
+	b, n := r.q.Dequeue(b[:0])
+	if len(b) != 0 {
+		return b, n
+	}
+
+	// Data isn't immediately available. Enable eventfd notifications.
+	r.q.EnableNotification()
+	for {
+		b, n = r.q.Dequeue(b)
+		if len(b) != 0 {
+			break
+		}
+
+		// Wait for notification.
+		r.eventFD.Wait()
+		if stopRequested.Load() != 0 {
+			r.q.DisableNotification()
+			return nil, 0
+		}
+	}
+	r.q.DisableNotification()
+
+	return b, n
+}

pkg/tcpip/link/sharedmem/server_endpoint_mutex.go

@@ -0,0 +1,96 @@
+package sharedmem
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type serverEndpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var serverEndpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type serverEndpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *serverEndpointRWMutex) Lock() {
+	locking.AddGLock(serverEndpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *serverEndpointRWMutex) NestedLock(i serverEndpointlockNameIndex) {
+	locking.AddGLock(serverEndpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *serverEndpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(serverEndpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *serverEndpointRWMutex) NestedUnlock(i serverEndpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(serverEndpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RLock() {
+	locking.AddGLock(serverEndpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(serverEndpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *serverEndpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *serverEndpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var serverEndpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func serverEndpointinitLockNames() {}
+
+func init() {
+	serverEndpointinitLockNames()
+	serverEndpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(serverEndpointRWMutex{}), serverEndpointlockNames)
+}

pkg/tcpip/link/sharedmem/server_rx.go

@@ -0,0 +1,162 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/cleanup"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"golang.org/x/sys/unix"
+)
+
+// +stateify savable
+type serverRx struct {
+	// packetPipe represents the receive end of the pipe that carries the packet
+	// descriptors sent by the client.
+	packetPipe pipe.Rx
+
+	// completionPipe represents the transmit end of the pipe that will carry
+	// completion notifications from the server to the client.
+	completionPipe pipe.Tx
+
+	// data represents the buffer area where the packet payload is held.
+	data []byte
+
+	// eventFD is used to notify the peer when transmission is completed.
+	eventFD eventfd.Eventfd
+
+	// sharedData the memory region to use to enable/disable notifications.
+	sharedData []byte
+
+	// sharedEventFDState is the memory region in sharedData used to enable
+	// disable notifications on eventFD.
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// init initializes all state needed by the serverTx queue based on the
+// information provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (s *serverRx) init(c *QueueConfig) error {
+	// Map in all buffers.
+	packetPipeMem, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu := cleanup.Make(func() { unix.Munmap(packetPipeMem) })
+	defer cu.Clean()
+
+	completionPipeMem, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(completionPipeMem) })
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(data) })
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(sharedData) })
+
+	// Duplicate the eventFD so that caller can close it but we can still
+	// use it.
+	efd, err := c.EventFD.Dup()
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { efd.Close() })
+
+	s.packetPipe.Init(packetPipeMem)
+	s.completionPipe.Init(completionPipeMem)
+	s.data = data
+	s.eventFD = efd
+	s.sharedData = sharedData
+	s.sharedEventFDState = sharedDataPointer(sharedData)
+
+	cu.Release()
+	return nil
+}
+
+func (s *serverRx) cleanup() {
+	unix.Munmap(s.packetPipe.Bytes())
+	unix.Munmap(s.completionPipe.Bytes())
+	unix.Munmap(s.data)
+	unix.Munmap(s.sharedData)
+	s.eventFD.Close()
+}
+
+// EnableNotification updates the shared state such that the peer will notify
+// the eventfd when there are packets to be dequeued.
+func (s *serverRx) EnableNotification() {
+	s.sharedEventFDState.Store(queue.EventFDEnabled)
+}
+
+// DisableNotification updates the shared state such that the peer will not
+// notify the eventfd.
+func (s *serverRx) DisableNotification() {
+	s.sharedEventFDState.Store(queue.EventFDDisabled)
+}
+
+// completionNotificationSize is size in bytes of a completion notification sent
+// on the completion queue after a transmitted packet has been handled.
+const completionNotificationSize = 8
+
+// receive receives a single packet from the packetPipe.
+func (s *serverRx) receive() *buffer.View {
+	desc := s.packetPipe.Pull()
+	if desc == nil {
+		return nil
+	}
+
+	pktInfo := queue.DecodeTxPacketHeader(desc)
+	contents := buffer.NewView(int(pktInfo.Size))
+	toCopy := pktInfo.Size
+	for i := 0; i < pktInfo.BufferCount; i++ {
+		txBuf := queue.DecodeTxBufferHeader(desc, i)
+		if txBuf.Size <= toCopy {
+			contents.Write(s.data[txBuf.Offset:][:txBuf.Size])
+			toCopy -= txBuf.Size
+			continue
+		}
+		contents.Write(s.data[txBuf.Offset:][:toCopy])
+		break
+	}
+
+	// Flush to let peer know that slots queued for transmission have been handled
+	// and its free to reuse the slots.
+	s.packetPipe.Flush()
+	// Encode packet completion.
+	b := s.completionPipe.Push(completionNotificationSize)
+	queue.EncodeTxCompletion(b, pktInfo.ID)
+	s.completionPipe.Flush()
+	return contents
+}
+
+func (s *serverRx) waitForPackets() {
+	s.eventFD.Wait()
+}

pkg/tcpip/link/sharedmem/server_tx.go

@@ -0,0 +1,194 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/cleanup"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/pipe"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"golang.org/x/sys/unix"
+)
+
+// serverTx represents the server end of the sharedmem queue and is used to send
+// packets to the peer in the buffers posted by the peer in the fillPipe.
+//
+// +stateify savable
+type serverTx struct {
+	// fillPipe represents the receive end of the pipe that carries the RxBuffers
+	// posted by the peer.
+	fillPipe pipe.Rx
+
+	// completionPipe represents the transmit end of the pipe that carries the
+	// descriptors for filled RxBuffers.
+	completionPipe pipe.Tx
+
+	// data represents the buffer area where the packet payload is held.
+	data []byte
+
+	// eventFD is used to notify the peer when fill requests are fulfilled.
+	eventFD eventfd.Eventfd
+
+	// sharedData the memory region to use to enable/disable notifications.
+	sharedData []byte
+
+	// sharedEventFDState is the memory region in sharedData used to enable/disable
+	// notifications on eventFD.
+	sharedEventFDState *atomicbitops.Uint32
+}
+
+// init initializes all tstate needed by the serverTx queue based on the
+// information provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (s *serverTx) init(c *QueueConfig) error {
+	// Map in all buffers.
+	fillPipeMem, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu := cleanup.Make(func() { unix.Munmap(fillPipeMem) })
+	defer cu.Clean()
+
+	completionPipeMem, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(completionPipeMem) })
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(data) })
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { unix.Munmap(sharedData) })
+
+	// Duplicate the eventFD so that caller can close it but we can still
+	// use it.
+	efd, err := c.EventFD.Dup()
+	if err != nil {
+		return err
+	}
+	cu.Add(func() { efd.Close() })
+
+	cu.Release()
+
+	s.fillPipe.Init(fillPipeMem)
+	s.completionPipe.Init(completionPipeMem)
+	s.data = data
+	s.eventFD = efd
+	s.sharedData = sharedData
+	s.sharedEventFDState = sharedDataPointer(sharedData)
+
+	return nil
+}
+
+func (s *serverTx) cleanup() {
+	unix.Munmap(s.fillPipe.Bytes())
+	unix.Munmap(s.completionPipe.Bytes())
+	unix.Munmap(s.data)
+	unix.Munmap(s.sharedData)
+	s.eventFD.Close()
+}
+
+// acquireBuffers acquires enough buffers to hold all the data in views or
+// returns nil if not enough buffers are currently available.
+func (s *serverTx) acquireBuffers(pktBuffer buffer.Buffer, buffers []queue.RxBuffer) (acquiredBuffers []queue.RxBuffer) {
+	acquiredBuffers = buffers[:0]
+	wantBytes := int(pktBuffer.Size())
+	for wantBytes > 0 {
+		var b []byte
+		if b = s.fillPipe.Pull(); b == nil {
+			s.fillPipe.Abort()
+			return nil
+		}
+		rxBuffer := queue.DecodeRxBufferHeader(b)
+		acquiredBuffers = append(acquiredBuffers, rxBuffer)
+		wantBytes -= int(rxBuffer.Size)
+	}
+	return acquiredBuffers
+}
+
+// fillPacket copies the data in the provided views into buffers pulled from the
+// fillPipe and returns a slice of RxBuffers that contain the copied data as
+// well as the total number of bytes copied.
+//
+// To avoid allocations the filledBuffers are appended to the buffers slice
+// which will be grown as required. This method takes ownership of pktBuffer.
+func (s *serverTx) fillPacket(pktBuffer buffer.Buffer, buffers []queue.RxBuffer) (filledBuffers []queue.RxBuffer, totalCopied uint32) {
+	bufs := s.acquireBuffers(pktBuffer, buffers)
+	if bufs == nil {
+		pktBuffer.Release()
+		return nil, 0
+	}
+	br := pktBuffer.AsBufferReader()
+	defer br.Close()
+
+	for i := 0; br.Len() > 0 && i < len(bufs); i++ {
+		buf := bufs[i]
+		copied, err := br.Read(s.data[buf.Offset:][:buf.Size])
+		buf.Size = uint32(copied)
+		// Copy the packet into the posted buffer.
+		totalCopied += bufs[i].Size
+		if err != nil {
+			return bufs, totalCopied
+		}
+	}
+	return bufs, totalCopied
+}
+
+func (s *serverTx) transmit(pkt *stack.PacketBuffer) bool {
+	buffers := make([]queue.RxBuffer, 8)
+	buffers, totalCopied := s.fillPacket(pkt.ToBuffer(), buffers)
+	if totalCopied == 0 {
+		// drop the packet as not enough buffers were probably available
+		// to send.
+		return false
+	}
+	b := s.completionPipe.Push(queue.RxCompletionSize(len(buffers)))
+	if b == nil {
+		return false
+	}
+	queue.EncodeRxCompletion(b, totalCopied, 0 /* reserved */)
+	for i := 0; i < len(buffers); i++ {
+		queue.EncodeRxCompletionBuffer(b, i, buffers[i])
+	}
+	s.completionPipe.Flush()
+	s.fillPipe.Flush()
+	return true
+}
+
+func (s *serverTx) notificationsEnabled() bool {
+	// notifications are considered to be enabled unless explicitly disabled.
+	return s.sharedEventFDState.Load() != queue.EventFDDisabled
+}
+
+func (s *serverTx) notify() {
+	if s.notificationsEnabled() {
+		s.eventFD.Notify()
+	}
+}

pkg/tcpip/link/sharedmem/sharedmem.go

@@ -0,0 +1,559 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+// Package sharedmem provides the implementation of data-link layer endpoints
+// backed by shared memory.
+//
+// Shared memory endpoints can be used in the networking stack by calling New()
+// to create a new endpoint, and then passing it as an argument to
+// Stack.CreateNIC().
+package sharedmem
+
+import (
+	"fmt"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// QueueConfig holds all the file descriptors needed to describe a tx or rx
+// queue over shared memory. It is used when creating new shared memory
+// endpoints to describe tx and rx queues.
+//
+// +stateify savable
+type QueueConfig struct {
+	// DataFD is a file descriptor for the file that contains the data to
+	// be transmitted via this queue. Descriptors contain offsets within
+	// this file.
+	DataFD int
+
+	// EventFD is a file descriptor for the event that is signaled when
+	// data is becomes available in this queue.
+	EventFD eventfd.Eventfd
+
+	// TxPipeFD is a file descriptor for the tx pipe associated with the
+	// queue.
+	TxPipeFD int
+
+	// RxPipeFD is a file descriptor for the rx pipe associated with the
+	// queue.
+	RxPipeFD int
+
+	// SharedDataFD is a file descriptor for the file that contains shared
+	// state between the two ends of the queue. This data specifies, for
+	// example, whether EventFD signaling is enabled or disabled.
+	SharedDataFD int
+}
+
+// FDs returns the FD's in the QueueConfig as a slice of ints. This must
+// be used in conjunction with QueueConfigFromFDs to ensure the order
+// of FDs matches when reconstructing the config when serialized or sent
+// as part of control messages.
+func (q *QueueConfig) FDs() []int {
+	return []int{q.DataFD, q.EventFD.FD(), q.TxPipeFD, q.RxPipeFD, q.SharedDataFD}
+}
+
+// QueueConfigFromFDs constructs a QueueConfig out of a slice of ints where each
+// entry represents an file descriptor. The order of FDs in the slice must be in
+// the order specified below for the config to be valid. QueueConfig.FDs()
+// should be used when the config needs to be serialized or sent as part of a
+// control message to ensure the correct order.
+func QueueConfigFromFDs(fds []int) (QueueConfig, error) {
+	if len(fds) != 5 {
+		return QueueConfig{}, fmt.Errorf("insufficient number of fds: len(fds): %d, want: 5", len(fds))
+	}
+	return QueueConfig{
+		DataFD:       fds[0],
+		EventFD:      eventfd.Wrap(fds[1]),
+		TxPipeFD:     fds[2],
+		RxPipeFD:     fds[3],
+		SharedDataFD: fds[4],
+	}, nil
+}
+
+// Options specify the details about the sharedmem endpoint to be created.
+//
+// +stateify savable
+type Options struct {
+	// MTU is the mtu to use for this endpoint.
+	MTU uint32
+
+	// BufferSize is the size of each scatter/gather buffer that will hold packet
+	// data.
+	//
+	// NOTE: This directly determines number of packets that can be held in
+	// the ring buffer at any time. This does not have to be sized to the MTU as
+	// the shared memory queue design allows usage of more than one buffer to be
+	// used to make up a given packet.
+	BufferSize uint32
+
+	// LinkAddress is the link address for this endpoint (required).
+	LinkAddress tcpip.LinkAddress
+
+	// TX is the transmit queue configuration for this shared memory endpoint.
+	TX QueueConfig
+
+	// RX is the receive queue configuration for this shared memory endpoint.
+	RX QueueConfig
+
+	// PeerFD is the fd for the connected peer which can be used to detect
+	// peer disconnects.
+	PeerFD int
+
+	// OnClosed is a function that is called when the endpoint is being closed
+	// (probably due to peer going away)
+	OnClosed func(err tcpip.Error)
+
+	// TXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityTXChecksumOffload.
+	TXChecksumOffload bool
+
+	// RXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityRXChecksumOffload.
+	RXChecksumOffload bool
+
+	// VirtioNetHeaderRequired if true, indicates that all outbound packets should have
+	// a virtio header and inbound packets should have a virtio header as well.
+	VirtioNetHeaderRequired bool
+
+	// GSOMaxSize is the maximum GSO packet size. It is zero if GSO is
+	// disabled. Note that only gVisor GSO is supported, not host GSO.
+	GSOMaxSize uint32
+}
+
+var (
+	_ stack.LinkEndpoint = (*endpoint)(nil)
+	_ stack.GSOEndpoint  = (*endpoint)(nil)
+)
+
+// +stateify savable
+type endpoint struct {
+	// bufferSize is the size of each individual buffer.
+	// bufferSize is immutable.
+	bufferSize uint32
+
+	// peerFD is an fd to the peer that can be used to detect when the
+	// peer is gone.
+	// peerFD is immutable.
+	peerFD int
+
+	// caps holds the endpoint capabilities.
+	caps stack.LinkEndpointCapabilities
+
+	// hdrSize is the size of the link layer header if any.
+	// hdrSize is immutable.
+	hdrSize uint32
+
+	// gSOMaxSize is the maximum GSO packet size. It is zero if GSO is
+	// disabled. Note that only gVisor GSO is supported, not host GSO.
+	// gsoMaxSize is immutable.
+	gsoMaxSize uint32
+
+	// virtioNetHeaderRequired if true indicates that a virtio header is expected
+	// in all inbound/outbound packets.
+	virtioNetHeaderRequired bool
+
+	// rx is the receive queue.
+	rx rx
+
+	// stopRequested  determines whether the worker goroutines should stop.
+	stopRequested atomicbitops.Uint32
+
+	// Wait group used to indicate that all workers have stopped.
+	completed sync.WaitGroup
+
+	// onClosed is a function to be called when the FD's peer (if any) closes
+	// its end of the communication pipe.
+	// TODO(b/341946753): Restore when netstack is savable.
+	onClosed func(tcpip.Error) `state:"nosave"`
+
+	// mu protects the following fields.
+	mu endpointRWMutex `state:"nosave"`
+
+	// tx is the transmit queue.
+	// +checklocks:mu
+	tx tx
+
+	// workerStarted specifies whether the worker goroutine was started.
+	// +checklocks:mu
+	workerStarted bool
+
+	// addr is the local address of this endpoint.
+	//
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+	// mtu (maximum transmission unit) is the maximum size of a packet.
+	// +checklocks:mu
+	mtu uint32
+}
+
+// New creates a new shared-memory-based endpoint. Buffers will be broken up
+// into buffers of "bufferSize" bytes.
+//
+// In order to release all resources held by the returned endpoint, Close()
+// must be called followed by Wait().
+func New(opts Options) (stack.LinkEndpoint, error) {
+	e := &endpoint{
+		mtu:                     opts.MTU,
+		bufferSize:              opts.BufferSize,
+		addr:                    opts.LinkAddress,
+		peerFD:                  opts.PeerFD,
+		onClosed:                opts.OnClosed,
+		virtioNetHeaderRequired: opts.VirtioNetHeaderRequired,
+		gsoMaxSize:              opts.GSOMaxSize,
+	}
+
+	if err := e.tx.init(opts.BufferSize, &opts.TX); err != nil {
+		return nil, err
+	}
+
+	if err := e.rx.init(opts.BufferSize, &opts.RX); err != nil {
+		e.tx.cleanup()
+		return nil, err
+	}
+
+	e.caps = stack.LinkEndpointCapabilities(0)
+	if opts.RXChecksumOffload {
+		e.caps |= stack.CapabilityRXChecksumOffload
+	}
+
+	if opts.TXChecksumOffload {
+		e.caps |= stack.CapabilityTXChecksumOffload
+	}
+
+	if opts.LinkAddress != "" {
+		e.hdrSize = header.EthernetMinimumSize
+		e.caps |= stack.CapabilityResolutionRequired
+	}
+
+	if opts.VirtioNetHeaderRequired {
+		e.hdrSize += header.VirtioNetHeaderSize
+	}
+
+	return e, nil
+}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (e *endpoint) SetOnCloseAction(func()) {}
+
+// Close frees most resources associated with the endpoint. Wait() must be
+// called after Close() in order to free the rest.
+func (e *endpoint) Close() {
+	// Tell dispatch goroutine to stop, then write to the eventfd so that
+	// it wakes up in case it's sleeping.
+	if e.stopRequested.Swap(1) == 1 {
+		// It is already closed.
+		return
+	}
+	e.rx.eventFD.Notify()
+
+	// Cleanup the queues inline if the worker hasn't started yet; we also
+	// know it won't start from now on because stopRequested is set to 1.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	workerPresent := e.workerStarted
+
+	if !workerPresent {
+		e.tx.cleanup()
+		e.rx.cleanup()
+	}
+}
+
+// Wait implements stack.LinkEndpoint.Wait. It waits until all workers have
+// stopped after a Close() call.
+func (e *endpoint) Wait() {
+	e.completed.Wait()
+	e.rx.eventFD.Close()
+}
+
+// Attach implements stack.LinkEndpoint.Attach. It launches the goroutine that
+// reads packets from the rx queue.
+func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	if dispatcher == nil {
+		e.Close()
+		return
+	}
+	e.mu.Lock()
+	if !e.workerStarted && e.stopRequested.Load() == 0 {
+		e.workerStarted = true
+		e.completed.Add(1)
+
+		// Spin up a goroutine to monitor for peer shutdown.
+		if e.peerFD >= 0 {
+			e.completed.Add(1)
+			go func() {
+				defer e.completed.Done()
+				b := make([]byte, 1)
+				// When sharedmem endpoint is in use the peerFD is never used for any data
+				// transfer and this Read should only return if the peer is shutting down.
+				_, errno := rawfile.BlockingRead(e.peerFD, b)
+				if e.onClosed != nil {
+					if errno == 0 {
+						e.onClosed(nil)
+					} else {
+						e.onClosed(tcpip.TranslateErrno(errno))
+					}
+				}
+			}()
+		}
+
+		// Link endpoints are not savable. When transportation endpoints
+		// are saved, they stop sending outgoing packets and all
+		// incoming packets are rejected.
+		go e.dispatchLoop(dispatcher) // S/R-SAFE: see above.
+	}
+	e.mu.Unlock()
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *endpoint) IsAttached() bool {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.workerStarted
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *endpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+func (e *endpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.caps
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It returns the
+// ethernet frame header size.
+func (e *endpoint) MaxHeaderLength() uint16 {
+	return uint16(e.hdrSize)
+}
+
+// LinkAddress implements stack.LinkEndpoint.LinkAddress. It returns the local
+// link address.
+func (e *endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.addr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.addr = addr
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return
+	}
+
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	eth.Encode(&header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	})
+}
+
+func (e *endpoint) parseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize)
+	return ok
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return true
+	}
+
+	return e.parseHeader(pkt)
+}
+
+func (e *endpoint) AddVirtioNetHeader(pkt *stack.PacketBuffer) {
+	virtio := header.VirtioNetHeader(pkt.VirtioNetHeader().Push(header.VirtioNetHeaderSize))
+	virtio.Encode(&header.VirtioNetHeaderFields{})
+}
+
+// +checklocks:e.mu
+func (e *endpoint) writePacketLocked(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	if e.virtioNetHeaderRequired {
+		e.AddVirtioNetHeader(pkt)
+	}
+
+	// Transmit the packet.
+	b := pkt.ToBuffer()
+	defer b.Release()
+	ok := e.tx.transmit(b)
+	if !ok {
+		return &tcpip.ErrWouldBlock{}
+	}
+
+	return nil
+}
+
+// WritePackets implements stack.LinkEndpoint.WritePackets.
+func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	n := 0
+	var err tcpip.Error
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for _, pkt := range pkts.AsSlice() {
+		if err = e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil {
+			break
+		}
+		n++
+	}
+	// WritePackets never returns an error if it successfully transmitted at least
+	// one packet.
+	if err != nil && n == 0 {
+		return 0, err
+	}
+	e.tx.notify()
+	return n, nil
+}
+
+// dispatchLoop reads packets from the rx queue in a loop and dispatches them
+// to the network stack.
+func (e *endpoint) dispatchLoop(d stack.NetworkDispatcher) {
+	// Post initial set of buffers.
+	limit := e.rx.q.PostedBuffersLimit()
+	if l := uint64(len(e.rx.data)) / uint64(e.bufferSize); limit > l {
+		limit = l
+	}
+	for i := uint64(0); i < limit; i++ {
+		b := queue.RxBuffer{
+			Offset: i * uint64(e.bufferSize),
+			Size:   e.bufferSize,
+			ID:     i,
+		}
+		if !e.rx.q.PostBuffers([]queue.RxBuffer{b}) {
+			log.Warningf("Unable to post %v-th buffer", i)
+		}
+	}
+
+	// Read in a loop until a stop is requested.
+	var rxb []queue.RxBuffer
+	for e.stopRequested.Load() == 0 {
+		var n uint32
+		rxb, n = e.rx.postAndReceive(rxb, &e.stopRequested)
+
+		// Copy data from the shared area to its own buffer, then
+		// prepare to repost the buffer.
+		v := buffer.NewView(int(n))
+		v.Grow(int(n))
+		offset := uint32(0)
+		for i := range rxb {
+			v.WriteAt(e.rx.data[rxb[i].Offset:][:rxb[i].Size], int(offset))
+			offset += rxb[i].Size
+
+			rxb[i].Size = e.bufferSize
+		}
+
+		pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: buffer.MakeWithView(v),
+		})
+
+		if e.virtioNetHeaderRequired {
+			_, ok := pkt.VirtioNetHeader().Consume(header.VirtioNetHeaderSize)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+		}
+
+		var proto tcpip.NetworkProtocolNumber
+		e.mu.RLock()
+		addrLen := len(e.addr)
+		e.mu.RUnlock()
+		if addrLen != 0 {
+			if !e.parseHeader(pkt) {
+				pkt.DecRef()
+				continue
+			}
+			proto = header.Ethernet(pkt.LinkHeader().Slice()).Type()
+		} else {
+			// We don't get any indication of what the packet is, so try to guess
+			// if it's an IPv4 or IPv6 packet.
+			// IP version information is at the first octet, so pulling up 1 byte.
+			h, ok := pkt.Data().PullUp(1)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+			switch header.IPVersion(h) {
+			case header.IPv4Version:
+				proto = header.IPv4ProtocolNumber
+			case header.IPv6Version:
+				proto = header.IPv6ProtocolNumber
+			default:
+				pkt.DecRef()
+				continue
+			}
+		}
+		// Send packet up the stack.
+		d.DeliverNetworkPacket(proto, pkt)
+		pkt.DecRef()
+	}
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	// Clean state.
+	e.tx.cleanup()
+	e.rx.cleanup()
+
+	e.completed.Done()
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType
+func (*endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareEther
+}
+
+// GSOMaxSize implements stack.GSOEndpoint.
+func (e *endpoint) GSOMaxSize() uint32 {
+	return e.gsoMaxSize
+}
+
+// SupportsGSO implements stack.GSOEndpoint.
+func (e *endpoint) SupportedGSO() stack.SupportedGSO {
+	return stack.GVisorGSOSupported
+}

pkg/tcpip/link/sharedmem/sharedmem_server.go

@@ -0,0 +1,399 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package sharedmem
+
+import (
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// +stateify savable
+type serverEndpoint struct {
+	// bufferSize is the size of each individual buffer.
+	// bufferSize is immutable.
+	bufferSize uint32
+
+	// rx is the receive queue.
+	rx serverRx
+
+	// stopRequested determines whether the worker goroutines should stop.
+	stopRequested atomicbitops.Uint32
+
+	// Wait group used to indicate that all workers have stopped.
+	completed sync.WaitGroup `state:"nosave"`
+
+	// peerFD is an fd to the peer that can be used to detect when the peer is
+	// gone.
+	// peerFD is immutable.
+	peerFD int
+
+	// caps holds the endpoint capabilities.
+	caps stack.LinkEndpointCapabilities
+
+	// hdrSize is the size of the link layer header if any.
+	// hdrSize is immutable.
+	hdrSize uint32
+
+	// virtioNetHeaderRequired if true indicates that a virtio header is expected
+	// in all inbound/outbound packets.
+	virtioNetHeaderRequired bool
+
+	// onClosed is a function to be called when the FD's peer (if any) closes its
+	// end of the communication pipe.
+	onClosed func(tcpip.Error) `state:"nosave"`
+
+	// mu protects the following fields.
+	mu serverEndpointRWMutex `state:"nosave"`
+
+	// tx is the transmit queue.
+	// +checklocks:mu
+	tx serverTx
+
+	// workerStarted specifies whether the worker goroutine was started.
+	// +checklocks:mu
+	workerStarted bool
+
+	// addr is the local address of this endpoint.
+	//
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+	// mtu (maximum transmission unit) is the maximum size of a packet.
+	// +checklocks:mu
+	mtu uint32
+}
+
+// NewServerEndpoint creates a new shared-memory-based endpoint. Buffers will be
+// broken up into buffers of "bufferSize" bytes.
+func NewServerEndpoint(opts Options) (stack.LinkEndpoint, error) {
+	e := &serverEndpoint{
+		mtu:        opts.MTU,
+		bufferSize: opts.BufferSize,
+		addr:       opts.LinkAddress,
+		peerFD:     opts.PeerFD,
+		onClosed:   opts.OnClosed,
+	}
+
+	if err := e.tx.init(&opts.RX); err != nil {
+		return nil, err
+	}
+
+	if err := e.rx.init(&opts.TX); err != nil {
+		e.tx.cleanup()
+		return nil, err
+	}
+
+	e.caps = stack.LinkEndpointCapabilities(0)
+	if opts.RXChecksumOffload {
+		e.caps |= stack.CapabilityRXChecksumOffload
+	}
+
+	if opts.TXChecksumOffload {
+		e.caps |= stack.CapabilityTXChecksumOffload
+	}
+
+	if opts.LinkAddress != "" {
+		e.hdrSize = header.EthernetMinimumSize
+		e.caps |= stack.CapabilityResolutionRequired
+	}
+
+	return e, nil
+}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (*serverEndpoint) SetOnCloseAction(func()) {}
+
+// Close frees all resources associated with the endpoint.
+func (e *serverEndpoint) Close() {
+	// Tell dispatch goroutine to stop, then write to the eventfd so that it wakes
+	// up in case it's sleeping.
+	e.stopRequested.Store(1)
+	e.rx.eventFD.Notify()
+
+	// Cleanup the queues inline if the worker hasn't started yet; we also know it
+	// won't start from now on because stopRequested is set to 1.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	workerPresent := e.workerStarted
+
+	if !workerPresent {
+		e.tx.cleanup()
+		e.rx.cleanup()
+	}
+}
+
+// Wait implements stack.LinkEndpoint.Wait. It waits until all workers have
+// stopped after a Close() call.
+func (e *serverEndpoint) Wait() {
+	e.completed.Wait()
+}
+
+// Attach implements stack.LinkEndpoint.Attach. It launches the goroutine that
+// reads packets from the rx queue.
+func (e *serverEndpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	if !e.workerStarted && e.stopRequested.Load() == 0 {
+		e.workerStarted = true
+		e.completed.Add(1)
+		if e.peerFD >= 0 {
+			e.completed.Add(1)
+			// Spin up a goroutine to monitor for peer shutdown.
+			go func() {
+				b := make([]byte, 1)
+				// When sharedmem endpoint is in use the peerFD is never used for any
+				// data transfer and this Read should only return if the peer is
+				// shutting down.
+				_, errno := rawfile.BlockingRead(e.peerFD, b)
+				if e.onClosed != nil {
+					if errno == 0 {
+						e.onClosed(nil)
+					} else {
+						e.onClosed(tcpip.TranslateErrno(errno))
+					}
+				}
+				e.completed.Done()
+			}()
+		}
+		// Link endpoints are not savable. When transportation endpoints are saved,
+		// they stop sending outgoing packets and all incoming packets are rejected.
+		go e.dispatchLoop(dispatcher) // S/R-SAFE: see above.
+	}
+	e.mu.Unlock()
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *serverEndpoint) IsAttached() bool {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.workerStarted
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *serverEndpoint) MTU() uint32 {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.mtu
+}
+
+func (e *serverEndpoint) SetMTU(mtu uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *serverEndpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.caps
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It returns the
+// ethernet frame header size.
+func (e *serverEndpoint) MaxHeaderLength() uint16 {
+	return uint16(e.hdrSize)
+}
+
+// LinkAddress implements stack.LinkEndpoint.LinkAddress. It returns the local
+// link address.
+func (e *serverEndpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.addr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *serverEndpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.addr = addr
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *serverEndpoint) AddHeader(pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return
+	}
+
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	eth.Encode(&header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	})
+}
+
+func (e *serverEndpoint) parseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize)
+	return ok
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *serverEndpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	// Add ethernet header if needed.
+	if len(e.addr) == 0 {
+		return true
+	}
+
+	return e.parseHeader(pkt)
+}
+
+func (e *serverEndpoint) AddVirtioNetHeader(pkt *stack.PacketBuffer) {
+	virtio := header.VirtioNetHeader(pkt.VirtioNetHeader().Push(header.VirtioNetHeaderSize))
+	virtio.Encode(&header.VirtioNetHeaderFields{})
+}
+
+// +checklocks:e.mu
+func (e *serverEndpoint) writePacketLocked(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	if e.virtioNetHeaderRequired {
+		e.AddVirtioNetHeader(pkt)
+	}
+
+	ok := e.tx.transmit(pkt)
+	if !ok {
+		return &tcpip.ErrWouldBlock{}
+	}
+
+	return nil
+}
+
+// WritePacket writes outbound packets to the file descriptor. If it is not
+// currently writable, the packet is dropped.
+// WritePacket implements stack.LinkEndpoint.WritePacket.
+func (e *serverEndpoint) WritePacket(_ stack.RouteInfo, _ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	// Transmit the packet.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if err := e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil {
+		return err
+	}
+	e.tx.notify()
+	return nil
+}
+
+// WritePackets implements stack.LinkEndpoint.WritePackets.
+func (e *serverEndpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	n := 0
+	var err tcpip.Error
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for _, pkt := range pkts.AsSlice() {
+		if err = e.writePacketLocked(pkt.EgressRoute, pkt.NetworkProtocolNumber, pkt); err != nil {
+			break
+		}
+		n++
+	}
+	// WritePackets never returns an error if it successfully transmitted at least
+	// one packet.
+	if err != nil && n == 0 {
+		return 0, err
+	}
+	e.tx.notify()
+	return n, nil
+}
+
+// dispatchLoop reads packets from the rx queue in a loop and dispatches them
+// to the network stack.
+func (e *serverEndpoint) dispatchLoop(d stack.NetworkDispatcher) {
+	for e.stopRequested.Load() == 0 {
+		b := e.rx.receive()
+		if b == nil {
+			e.rx.EnableNotification()
+			// Now pull again to make sure we didn't receive any packets
+			// while notifications were not enabled.
+			for {
+				b = e.rx.receive()
+				if b != nil {
+					// Disable notifications as we only need to be notified when we are going
+					// to block on eventFD. This should prevent the peer from needlessly
+					// writing to eventFD when this end is already awake and processing
+					// packets.
+					e.rx.DisableNotification()
+					break
+				}
+				e.rx.waitForPackets()
+			}
+		}
+		pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: buffer.MakeWithView(b),
+		})
+		if e.virtioNetHeaderRequired {
+			_, ok := pkt.VirtioNetHeader().Consume(header.VirtioNetHeaderSize)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+		}
+		var proto tcpip.NetworkProtocolNumber
+		e.mu.RLock()
+		addrLen := len(e.addr)
+		e.mu.RUnlock()
+		if addrLen != 0 {
+			if !e.parseHeader(pkt) {
+				pkt.DecRef()
+				continue
+			}
+			proto = header.Ethernet(pkt.LinkHeader().Slice()).Type()
+		} else {
+			// We don't get any indication of what the packet is, so try to guess
+			// if it's an IPv4 or IPv6 packet.
+			// IP version information is at the first octet, so pulling up 1 byte.
+			h, ok := pkt.Data().PullUp(1)
+			if !ok {
+				pkt.DecRef()
+				continue
+			}
+			switch header.IPVersion(h) {
+			case header.IPv4Version:
+				proto = header.IPv4ProtocolNumber
+			case header.IPv6Version:
+				proto = header.IPv6ProtocolNumber
+			default:
+				pkt.DecRef()
+				continue
+			}
+		}
+		// Send packet up the stack.
+		d.DeliverNetworkPacket(proto, pkt)
+		pkt.DecRef()
+	}
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	// Clean state.
+	e.tx.cleanup()
+	e.rx.cleanup()
+
+	e.completed.Done()
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType
+func (e *serverEndpoint) ARPHardwareType() header.ARPHardwareType {
+	if e.hdrSize > 0 {
+		return header.ARPHardwareEther
+	}
+	return header.ARPHardwareNone
+}

pkg/tcpip/link/sharedmem/sharedmem_state_autogen.go

@@ -0,0 +1,309 @@
+// automatically generated by stateify.
+
+//go:build linux && linux && linux && linux && linux && linux
+// +build linux,linux,linux,linux,linux,linux
+
+package sharedmem
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (s *serverRx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.serverRx"
+}
+
+func (s *serverRx) StateFields() []string {
+	return []string{
+		"packetPipe",
+		"completionPipe",
+		"data",
+		"eventFD",
+		"sharedData",
+		"sharedEventFDState",
+	}
+}
+
+func (s *serverRx) beforeSave() {}
+
+// +checklocksignore
+func (s *serverRx) StateSave(stateSinkObject state.Sink) {
+	s.beforeSave()
+	stateSinkObject.Save(0, &s.packetPipe)
+	stateSinkObject.Save(1, &s.completionPipe)
+	stateSinkObject.Save(2, &s.data)
+	stateSinkObject.Save(3, &s.eventFD)
+	stateSinkObject.Save(4, &s.sharedData)
+	stateSinkObject.Save(5, &s.sharedEventFDState)
+}
+
+func (s *serverRx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (s *serverRx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &s.packetPipe)
+	stateSourceObject.Load(1, &s.completionPipe)
+	stateSourceObject.Load(2, &s.data)
+	stateSourceObject.Load(3, &s.eventFD)
+	stateSourceObject.Load(4, &s.sharedData)
+	stateSourceObject.Load(5, &s.sharedEventFDState)
+}
+
+func (s *serverTx) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.serverTx"
+}
+
+func (s *serverTx) StateFields() []string {
+	return []string{
+		"fillPipe",
+		"completionPipe",
+		"data",
+		"eventFD",
+		"sharedData",
+		"sharedEventFDState",
+	}
+}
+
+func (s *serverTx) beforeSave() {}
+
+// +checklocksignore
+func (s *serverTx) StateSave(stateSinkObject state.Sink) {
+	s.beforeSave()
+	stateSinkObject.Save(0, &s.fillPipe)
+	stateSinkObject.Save(1, &s.completionPipe)
+	stateSinkObject.Save(2, &s.data)
+	stateSinkObject.Save(3, &s.eventFD)
+	stateSinkObject.Save(4, &s.sharedData)
+	stateSinkObject.Save(5, &s.sharedEventFDState)
+}
+
+func (s *serverTx) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (s *serverTx) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &s.fillPipe)
+	stateSourceObject.Load(1, &s.completionPipe)
+	stateSourceObject.Load(2, &s.data)
+	stateSourceObject.Load(3, &s.eventFD)
+	stateSourceObject.Load(4, &s.sharedData)
+	stateSourceObject.Load(5, &s.sharedEventFDState)
+}
+
+func (q *QueueConfig) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.QueueConfig"
+}
+
+func (q *QueueConfig) StateFields() []string {
+	return []string{
+		"DataFD",
+		"EventFD",
+		"TxPipeFD",
+		"RxPipeFD",
+		"SharedDataFD",
+	}
+}
+
+func (q *QueueConfig) beforeSave() {}
+
+// +checklocksignore
+func (q *QueueConfig) StateSave(stateSinkObject state.Sink) {
+	q.beforeSave()
+	stateSinkObject.Save(0, &q.DataFD)
+	stateSinkObject.Save(1, &q.EventFD)
+	stateSinkObject.Save(2, &q.TxPipeFD)
+	stateSinkObject.Save(3, &q.RxPipeFD)
+	stateSinkObject.Save(4, &q.SharedDataFD)
+}
+
+func (q *QueueConfig) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (q *QueueConfig) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &q.DataFD)
+	stateSourceObject.Load(1, &q.EventFD)
+	stateSourceObject.Load(2, &q.TxPipeFD)
+	stateSourceObject.Load(3, &q.RxPipeFD)
+	stateSourceObject.Load(4, &q.SharedDataFD)
+}
+
+func (o *Options) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.Options"
+}
+
+func (o *Options) StateFields() []string {
+	return []string{
+		"MTU",
+		"BufferSize",
+		"LinkAddress",
+		"TX",
+		"RX",
+		"PeerFD",
+		"OnClosed",
+		"TXChecksumOffload",
+		"RXChecksumOffload",
+		"VirtioNetHeaderRequired",
+		"GSOMaxSize",
+	}
+}
+
+func (o *Options) beforeSave() {}
+
+// +checklocksignore
+func (o *Options) StateSave(stateSinkObject state.Sink) {
+	o.beforeSave()
+	stateSinkObject.Save(0, &o.MTU)
+	stateSinkObject.Save(1, &o.BufferSize)
+	stateSinkObject.Save(2, &o.LinkAddress)
+	stateSinkObject.Save(3, &o.TX)
+	stateSinkObject.Save(4, &o.RX)
+	stateSinkObject.Save(5, &o.PeerFD)
+	stateSinkObject.Save(6, &o.OnClosed)
+	stateSinkObject.Save(7, &o.TXChecksumOffload)
+	stateSinkObject.Save(8, &o.RXChecksumOffload)
+	stateSinkObject.Save(9, &o.VirtioNetHeaderRequired)
+	stateSinkObject.Save(10, &o.GSOMaxSize)
+}
+
+func (o *Options) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (o *Options) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &o.MTU)
+	stateSourceObject.Load(1, &o.BufferSize)
+	stateSourceObject.Load(2, &o.LinkAddress)
+	stateSourceObject.Load(3, &o.TX)
+	stateSourceObject.Load(4, &o.RX)
+	stateSourceObject.Load(5, &o.PeerFD)
+	stateSourceObject.Load(6, &o.OnClosed)
+	stateSourceObject.Load(7, &o.TXChecksumOffload)
+	stateSourceObject.Load(8, &o.RXChecksumOffload)
+	stateSourceObject.Load(9, &o.VirtioNetHeaderRequired)
+	stateSourceObject.Load(10, &o.GSOMaxSize)
+}
+
+func (e *endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.endpoint"
+}
+
+func (e *endpoint) StateFields() []string {
+	return []string{
+		"bufferSize",
+		"peerFD",
+		"caps",
+		"hdrSize",
+		"gsoMaxSize",
+		"virtioNetHeaderRequired",
+		"rx",
+		"stopRequested",
+		"completed",
+		"tx",
+		"workerStarted",
+		"addr",
+		"mtu",
+	}
+}
+
+func (e *endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.bufferSize)
+	stateSinkObject.Save(1, &e.peerFD)
+	stateSinkObject.Save(2, &e.caps)
+	stateSinkObject.Save(3, &e.hdrSize)
+	stateSinkObject.Save(4, &e.gsoMaxSize)
+	stateSinkObject.Save(5, &e.virtioNetHeaderRequired)
+	stateSinkObject.Save(6, &e.rx)
+	stateSinkObject.Save(7, &e.stopRequested)
+	stateSinkObject.Save(8, &e.completed)
+	stateSinkObject.Save(9, &e.tx)
+	stateSinkObject.Save(10, &e.workerStarted)
+	stateSinkObject.Save(11, &e.addr)
+	stateSinkObject.Save(12, &e.mtu)
+}
+
+func (e *endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.bufferSize)
+	stateSourceObject.Load(1, &e.peerFD)
+	stateSourceObject.Load(2, &e.caps)
+	stateSourceObject.Load(3, &e.hdrSize)
+	stateSourceObject.Load(4, &e.gsoMaxSize)
+	stateSourceObject.Load(5, &e.virtioNetHeaderRequired)
+	stateSourceObject.Load(6, &e.rx)
+	stateSourceObject.Load(7, &e.stopRequested)
+	stateSourceObject.Load(8, &e.completed)
+	stateSourceObject.Load(9, &e.tx)
+	stateSourceObject.Load(10, &e.workerStarted)
+	stateSourceObject.Load(11, &e.addr)
+	stateSourceObject.Load(12, &e.mtu)
+}
+
+func (e *serverEndpoint) StateTypeName() string {
+	return "pkg/tcpip/link/sharedmem.serverEndpoint"
+}
+
+func (e *serverEndpoint) StateFields() []string {
+	return []string{
+		"bufferSize",
+		"rx",
+		"stopRequested",
+		"peerFD",
+		"caps",
+		"hdrSize",
+		"virtioNetHeaderRequired",
+		"tx",
+		"workerStarted",
+		"addr",
+		"mtu",
+	}
+}
+
+func (e *serverEndpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *serverEndpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.bufferSize)
+	stateSinkObject.Save(1, &e.rx)
+	stateSinkObject.Save(2, &e.stopRequested)
+	stateSinkObject.Save(3, &e.peerFD)
+	stateSinkObject.Save(4, &e.caps)
+	stateSinkObject.Save(5, &e.hdrSize)
+	stateSinkObject.Save(6, &e.virtioNetHeaderRequired)
+	stateSinkObject.Save(7, &e.tx)
+	stateSinkObject.Save(8, &e.workerStarted)
+	stateSinkObject.Save(9, &e.addr)
+	stateSinkObject.Save(10, &e.mtu)
+}
+
+func (e *serverEndpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *serverEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.bufferSize)
+	stateSourceObject.Load(1, &e.rx)
+	stateSourceObject.Load(2, &e.stopRequested)
+	stateSourceObject.Load(3, &e.peerFD)
+	stateSourceObject.Load(4, &e.caps)
+	stateSourceObject.Load(5, &e.hdrSize)
+	stateSourceObject.Load(6, &e.virtioNetHeaderRequired)
+	stateSourceObject.Load(7, &e.tx)
+	stateSourceObject.Load(8, &e.workerStarted)
+	stateSourceObject.Load(9, &e.addr)
+	stateSourceObject.Load(10, &e.mtu)
+}
+
+func init() {
+	state.Register((*serverRx)(nil))
+	state.Register((*serverTx)(nil))
+	state.Register((*QueueConfig)(nil))
+	state.Register((*Options)(nil))
+	state.Register((*endpoint)(nil))
+	state.Register((*serverEndpoint)(nil))
+}

pkg/tcpip/link/sharedmem/sharedmem_unsafe.go

@@ -0,0 +1,59 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sharedmem
+
+import (
+	"fmt"
+	"reflect"
+	"unsafe"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/memutil"
+	"golang.org/x/sys/unix"
+)
+
+// sharedDataPointer converts the shared data slice into a pointer so that it
+// can be used in atomic operations.
+func sharedDataPointer(sharedData []byte) *atomicbitops.Uint32 {
+	return (*atomicbitops.Uint32)(unsafe.Pointer(&sharedData[0:4][0]))
+}
+
+// getBuffer returns a memory region mapped to the full contents of the given
+// file descriptor.
+func getBuffer(fd int) ([]byte, error) {
+	var s unix.Stat_t
+	if err := unix.Fstat(fd, &s); err != nil {
+		return nil, err
+	}
+
+	// Check that size doesn't overflow an int.
+	if s.Size > int64(^uint(0)>>1) {
+		return nil, unix.EDOM
+	}
+
+	addr, err := memutil.MapFile(0 /* addr */, uintptr(s.Size), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_FILE, uintptr(fd), 0 /*offset*/)
+	if err != nil {
+		return nil, fmt.Errorf("failed to map memory for buffer fd: %d, error: %s", fd, err)
+	}
+
+	// Use unsafe to convert addr into a []byte.
+	var b []byte
+	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	hdr.Data = addr
+	hdr.Len = int(s.Size)
+	hdr.Cap = int(s.Size)
+
+	return b, nil
+}

pkg/tcpip/link/sharedmem/sharedmem_unsafe_state_autogen.go

@@ -0,0 +1,3 @@
+// automatically generated by stateify.
+
+package sharedmem

pkg/tcpip/link/sharedmem/tx.go

@@ -0,0 +1,279 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sharedmem
+
+import (
+	"math"
+
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/eventfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/sharedmem/queue"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	nilID = math.MaxUint64
+)
+
+// tx holds all state associated with a tx queue.
+type tx struct {
+	data         []byte
+	q            queue.Tx
+	ids          idManager
+	bufs         bufferManager
+	eventFD      eventfd.Eventfd
+	sharedData   []byte
+	sharedDataFD int
+}
+
+// init initializes all state needed by the tx queue based on the information
+// provided.
+//
+// The caller always retains ownership of all file descriptors passed in. The
+// queue implementation will duplicate any that it may need in the future.
+func (t *tx) init(bufferSize uint32, c *QueueConfig) error {
+	// Map in all buffers.
+	txPipe, err := getBuffer(c.TxPipeFD)
+	if err != nil {
+		return err
+	}
+
+	rxPipe, err := getBuffer(c.RxPipeFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		return err
+	}
+
+	data, err := getBuffer(c.DataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		return err
+	}
+
+	sharedData, err := getBuffer(c.SharedDataFD)
+	if err != nil {
+		unix.Munmap(txPipe)
+		unix.Munmap(rxPipe)
+		unix.Munmap(data)
+	}
+
+	// Initialize state based on buffers.
+	t.q.Init(txPipe, rxPipe, sharedDataPointer(sharedData))
+	t.ids.init()
+	t.bufs.init(0, len(data), int(bufferSize))
+	t.data = data
+	t.eventFD = c.EventFD
+	t.sharedDataFD = c.SharedDataFD
+	t.sharedData = sharedData
+
+	return nil
+}
+
+// cleanup releases all resources allocated during init(). It must only be
+// called if init() has previously succeeded.
+func (t *tx) cleanup() {
+	a, b := t.q.Bytes()
+	unix.Munmap(a)
+	unix.Munmap(b)
+	unix.Munmap(t.data)
+}
+
+// transmit sends a packet made of bufs. Returns a boolean that specifies
+// whether the packet was successfully transmitted.
+func (t *tx) transmit(transmitBuf buffer.Buffer) bool {
+	// Pull completions from the tx queue and add their buffers back to the
+	// pool so that we can reuse them.
+	for {
+		id, ok := t.q.CompletedPacket()
+		if !ok {
+			break
+		}
+
+		if buf := t.ids.remove(id); buf != nil {
+			t.bufs.free(buf)
+		}
+	}
+
+	bSize := t.bufs.entrySize
+	total := uint32(transmitBuf.Size())
+	bufCount := (total + bSize - 1) / bSize
+
+	// Allocate enough buffers to hold all the data.
+	var buf *queue.TxBuffer
+	for i := bufCount; i != 0; i-- {
+		b := t.bufs.alloc()
+		if b == nil {
+			// Failed to get all buffers. Return to the pool
+			// whatever we had managed to get.
+			if buf != nil {
+				t.bufs.free(buf)
+			}
+			return false
+		}
+		b.Next = buf
+		buf = b
+	}
+
+	// Copy data into allocated buffers.
+	nBuf := buf
+	var dBuf []byte
+	transmitBuf.Apply(func(v *buffer.View) {
+		for v.Size() > 0 {
+			if len(dBuf) == 0 {
+				dBuf = t.data[nBuf.Offset:][:nBuf.Size]
+				nBuf = nBuf.Next
+			}
+			n := copy(dBuf, v.AsSlice())
+			v.TrimFront(n)
+			dBuf = dBuf[n:]
+		}
+	})
+
+	// Get an id for this packet and send it out.
+	id := t.ids.add(buf)
+	if !t.q.Enqueue(id, total, bufCount, buf) {
+		t.ids.remove(id)
+		t.bufs.free(buf)
+		return false
+	}
+
+	return true
+}
+
+// notify writes to the tx.eventFD to indicate to the peer that there is data to
+// be read.
+func (t *tx) notify() {
+	if t.q.NotificationsEnabled() {
+		t.eventFD.Notify()
+	}
+}
+
+// idDescriptor is used by idManager to either point to a tx buffer (in case
+// the ID is assigned) or to the next free element (if the id is not assigned).
+type idDescriptor struct {
+	buf      *queue.TxBuffer
+	nextFree uint64
+}
+
+// idManager is a manager of tx buffer identifiers. It assigns unique IDs to
+// tx buffers that are added to it; the IDs can only be reused after they have
+// been removed.
+//
+// The ID assignments are stored so that the tx buffers can be retrieved from
+// the IDs previously assigned to them.
+type idManager struct {
+	// ids is a slice containing all tx buffers. The ID is the index into
+	// this slice.
+	ids []idDescriptor
+
+	// freeList a list of free IDs.
+	freeList uint64
+}
+
+// init initializes the id manager.
+func (m *idManager) init() {
+	m.freeList = nilID
+}
+
+// add assigns an ID to the given tx buffer.
+func (m *idManager) add(b *queue.TxBuffer) uint64 {
+	if i := m.freeList; i != nilID {
+		// There is an id available in the free list, just use it.
+		m.ids[i].buf = b
+		m.freeList = m.ids[i].nextFree
+		return i
+	}
+
+	// We need to expand the id descriptor.
+	m.ids = append(m.ids, idDescriptor{buf: b})
+	return uint64(len(m.ids) - 1)
+}
+
+// remove retrieves the tx buffer associated with the given ID, and removes the
+// ID from the assigned table so that it can be reused in the future.
+func (m *idManager) remove(i uint64) *queue.TxBuffer {
+	if i >= uint64(len(m.ids)) {
+		return nil
+	}
+
+	desc := &m.ids[i]
+	b := desc.buf
+	if b == nil {
+		// The provided id is not currently assigned.
+		return nil
+	}
+
+	desc.buf = nil
+	desc.nextFree = m.freeList
+	m.freeList = i
+
+	return b
+}
+
+// bufferManager manages a buffer region broken up into smaller, equally sized
+// buffers. Smaller buffers can be allocated and freed.
+type bufferManager struct {
+	freeList  *queue.TxBuffer
+	curOffset uint64
+	limit     uint64
+	entrySize uint32
+}
+
+// init initializes the buffer manager.
+func (b *bufferManager) init(initialOffset, size, entrySize int) {
+	b.freeList = nil
+	b.curOffset = uint64(initialOffset)
+	b.limit = uint64(initialOffset + size/entrySize*entrySize)
+	b.entrySize = uint32(entrySize)
+}
+
+// alloc allocates a buffer from the manager, if one is available.
+func (b *bufferManager) alloc() *queue.TxBuffer {
+	if b.freeList != nil {
+		// There is a descriptor ready for reuse in the free list.
+		d := b.freeList
+		b.freeList = d.Next
+		d.Next = nil
+		return d
+	}
+
+	if b.curOffset < b.limit {
+		// There is room available in the never-used range, so create
+		// a new descriptor for it.
+		d := &queue.TxBuffer{
+			Offset: b.curOffset,
+			Size:   b.entrySize,
+		}
+		b.curOffset += uint64(b.entrySize)
+		return d
+	}
+
+	return nil
+}
+
+// free returns all buffers in the list to the buffer manager so that they can
+// be reused.
+func (b *bufferManager) free(d *queue.TxBuffer) {
+	// Find the last buffer in the list.
+	last := d
+	for last.Next != nil {
+		last = last.Next
+	}
+
+	// Push list onto free list.
+	last.Next = b.freeList
+	b.freeList = d
+}

pkg/tcpip/link/sniffer/pcap.go

@@ -0,0 +1,85 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sniffer
+
+import (
+	"encoding"
+	"encoding/binary"
+	"time"
+
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+type pcapHeader struct {
+	// MagicNumber is the file magic number.
+	MagicNumber uint32
+
+	// VersionMajor is the major version number.
+	VersionMajor uint16
+
+	// VersionMinor is the minor version number.
+	VersionMinor uint16
+
+	// Thiszone is the GMT to local correction.
+	Thiszone int32
+
+	// Sigfigs is the accuracy of timestamps.
+	Sigfigs uint32
+
+	// Snaplen is the max length of captured packets, in octets.
+	Snaplen uint32
+
+	// Network is the data link type.
+	Network uint32
+}
+
+var _ encoding.BinaryMarshaler = (*pcapPacket)(nil)
+
+type pcapPacket struct {
+	timestamp     time.Time
+	packet        *stack.PacketBuffer
+	maxCaptureLen int
+}
+
+func (p *pcapPacket) MarshalBinary() ([]byte, error) {
+	pkt := trimmedClone(p.packet)
+	defer pkt.DecRef()
+	packetSize := pkt.Size()
+	captureLen := p.maxCaptureLen
+	if packetSize < captureLen {
+		captureLen = packetSize
+	}
+	b := make([]byte, 16+captureLen)
+	binary.LittleEndian.PutUint32(b[0:4], uint32(p.timestamp.Unix()))
+	binary.LittleEndian.PutUint32(b[4:8], uint32(p.timestamp.Nanosecond()/1000))
+	binary.LittleEndian.PutUint32(b[8:12], uint32(captureLen))
+	binary.LittleEndian.PutUint32(b[12:16], uint32(packetSize))
+	w := tcpip.SliceWriter(b[16:])
+	for _, v := range pkt.AsSlices() {
+		if captureLen == 0 {
+			break
+		}
+		if len(v) > captureLen {
+			v = v[:captureLen]
+		}
+		n, err := w.Write(v)
+		if err != nil {
+			panic(err)
+		}
+		captureLen -= n
+	}
+	return b, nil
+}

pkg/tcpip/link/sniffer/sniffer.go

@@ -0,0 +1,399 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package sniffer provides the implementation of data-link layer endpoints that
+// wrap another endpoint and logs inbound and outbound packets.
+//
+// Sniffer endpoints can be used in the networking stack by calling New(eID) to
+// create a new endpoint, where eID is the ID of the endpoint being wrapped,
+// and then passing it as an argument to Stack.CreateNIC().
+package sniffer
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/log"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/header/parse"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/nested"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// LogPackets is a flag used to enable or disable packet logging via the log
+// package. Valid values are 0 or 1.
+var LogPackets atomicbitops.Uint32 = atomicbitops.FromUint32(1)
+
+// Endpoint is used to sniff and log network traffic.
+//
+// +stateify savable
+type Endpoint struct {
+	nested.Endpoint
+	writer     io.Writer
+	maxPCAPLen uint32
+	logPrefix  string
+}
+
+var (
+	_ stack.GSOEndpoint       = (*Endpoint)(nil)
+	_ stack.LinkEndpoint      = (*Endpoint)(nil)
+	_ stack.NetworkDispatcher = (*Endpoint)(nil)
+)
+
+// A Direction indicates whether the packing is being sent or received.
+type Direction int
+
+const (
+	// DirectionSend indicates a sent packet.
+	DirectionSend = iota
+	// DirectionRecv indicates a received packet.
+	DirectionRecv
+)
+
+func (dr Direction) String() string {
+	switch dr {
+	case DirectionSend:
+		return "send"
+	case DirectionRecv:
+		return "recv"
+	default:
+		panic(fmt.Sprintf("invalid Direction %d", dr))
+	}
+}
+
+// New creates a new sniffer link-layer endpoint. It wraps around another
+// endpoint and logs packets and they traverse the endpoint.
+func New(lower stack.LinkEndpoint) *Endpoint {
+	return NewWithPrefix(lower, "")
+}
+
+// NewWithPrefix creates a new sniffer link-layer endpoint. It wraps around
+// another endpoint and logs packets prefixed with logPrefix as they traverse
+// the endpoint.
+//
+// logPrefix is prepended to the log line without any separators.
+// E.g. logPrefix = "NIC:en0/" will produce log lines like
+// "NIC:en0/send udp [...]".
+func NewWithPrefix(lower stack.LinkEndpoint, logPrefix string) *Endpoint {
+	sniffer := &Endpoint{logPrefix: logPrefix}
+	sniffer.Endpoint.Init(lower, sniffer)
+	return sniffer
+}
+
+func zoneOffset() (int32, error) {
+	date := time.Date(0, 0, 0, 0, 0, 0, 0, time.Local)
+	_, offset := date.Zone()
+	return int32(offset), nil
+}
+
+func writePCAPHeader(w io.Writer, maxLen uint32) error {
+	offset, err := zoneOffset()
+	if err != nil {
+		return err
+	}
+	return binary.Write(w, binary.LittleEndian, pcapHeader{
+		// From https://wiki.wireshark.org/Development/LibpcapFileFormat
+		MagicNumber: 0xa1b2c3d4,
+
+		VersionMajor: 2,
+		VersionMinor: 4,
+		Thiszone:     offset,
+		Sigfigs:      0,
+		Snaplen:      maxLen,
+		Network:      101, // LINKTYPE_RAW
+	})
+}
+
+// NewWithWriter creates a new sniffer link-layer endpoint. It wraps around
+// another endpoint and logs packets as they traverse the endpoint.
+//
+// Each packet is written to writer in the pcap format in a single Write call
+// without synchronization. A sniffer created with this function will not emit
+// packets using the standard log package.
+//
+// snapLen is the maximum amount of a packet to be saved. Packets with a length
+// less than or equal to snapLen will be saved in their entirety. Longer
+// packets will be truncated to snapLen.
+func NewWithWriter(lower stack.LinkEndpoint, writer io.Writer, snapLen uint32) (*Endpoint, error) {
+	if err := writePCAPHeader(writer, snapLen); err != nil {
+		return nil, err
+	}
+	sniffer := &Endpoint{
+		writer:     writer,
+		maxPCAPLen: snapLen,
+	}
+	sniffer.Endpoint.Init(lower, sniffer)
+	return sniffer, nil
+}
+
+// DeliverNetworkPacket implements the stack.NetworkDispatcher interface. It is
+// called by the link-layer endpoint being wrapped when a packet arrives, and
+// logs the packet before forwarding to the actual dispatcher.
+func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.DumpPacket(DirectionRecv, protocol, pkt, nil)
+	e.Endpoint.DeliverNetworkPacket(protocol, pkt)
+}
+
+// DumpPacket logs a packet, depending on configuration, to stderr and/or a
+// pcap file. ts is an optional timestamp for the packet.
+func (e *Endpoint) DumpPacket(dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer, ts *time.Time) {
+	if LogPackets.Load() == 1 {
+		LogPacket(e.logPrefix, dir, protocol, pkt)
+	}
+	if e.writer != nil {
+		packet := pcapPacket{
+			packet:        pkt,
+			maxCaptureLen: int(e.maxPCAPLen),
+		}
+		if ts == nil {
+			packet.timestamp = time.Now()
+		} else {
+			packet.timestamp = *ts
+		}
+		b, err := packet.MarshalBinary()
+		if err != nil {
+			panic(err)
+		}
+		if _, err := e.writer.Write(b); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// WritePackets implements the stack.LinkEndpoint interface. It is called by
+// higher-level protocols to write packets; it just logs the packet and
+// forwards the request to the lower endpoint.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	for _, pkt := range pkts.AsSlice() {
+		e.DumpPacket(DirectionSend, pkt.NetworkProtocolNumber, pkt, nil)
+	}
+	return e.Endpoint.WritePackets(pkts)
+}
+
+// LogPacket logs a packet to stdout.
+func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	// Figure out the network layer info.
+	var transProto uint8
+	var src tcpip.Address
+	var dst tcpip.Address
+	var size uint16
+	var id uint32
+	var fragmentOffset uint16
+	var moreFragments bool
+
+	clone := trimmedClone(pkt)
+	defer clone.DecRef()
+	switch protocol {
+	case header.IPv4ProtocolNumber:
+		if ok := parse.IPv4(clone); !ok {
+			return
+		}
+
+		ipv4 := header.IPv4(clone.NetworkHeader().Slice())
+		fragmentOffset = ipv4.FragmentOffset()
+		moreFragments = ipv4.Flags()&header.IPv4FlagMoreFragments == header.IPv4FlagMoreFragments
+		src = ipv4.SourceAddress()
+		dst = ipv4.DestinationAddress()
+		transProto = ipv4.Protocol()
+		size = ipv4.TotalLength() - uint16(ipv4.HeaderLength())
+		id = uint32(ipv4.ID())
+
+	case header.IPv6ProtocolNumber:
+		proto, fragID, fragOffset, fragMore, ok := parse.IPv6(clone)
+		if !ok {
+			return
+		}
+
+		ipv6 := header.IPv6(clone.NetworkHeader().Slice())
+		src = ipv6.SourceAddress()
+		dst = ipv6.DestinationAddress()
+		transProto = uint8(proto)
+		size = ipv6.PayloadLength()
+		id = fragID
+		moreFragments = fragMore
+		fragmentOffset = fragOffset
+
+	case header.ARPProtocolNumber:
+		if !parse.ARP(clone) {
+			return
+		}
+
+		arp := header.ARP(clone.NetworkHeader().Slice())
+		log.Infof(
+			"%s%s arp %s (%s) -> %s (%s) valid:%t",
+			prefix,
+			dir,
+			tcpip.AddrFromSlice(arp.ProtocolAddressSender()), tcpip.LinkAddress(arp.HardwareAddressSender()),
+			tcpip.AddrFromSlice(arp.ProtocolAddressTarget()), tcpip.LinkAddress(arp.HardwareAddressTarget()),
+			arp.IsValid(),
+		)
+		return
+	default:
+		log.Infof("%s%s unknown network protocol: %d", prefix, dir, protocol)
+		return
+	}
+
+	// Figure out the transport layer info.
+	transName := "unknown"
+	srcPort := uint16(0)
+	dstPort := uint16(0)
+	details := ""
+	switch tcpip.TransportProtocolNumber(transProto) {
+	case header.ICMPv4ProtocolNumber:
+		transName = "icmp"
+		hdr, ok := clone.Data().PullUp(header.ICMPv4MinimumSize)
+		if !ok {
+			break
+		}
+		icmp := header.ICMPv4(hdr)
+		icmpType := "unknown"
+		if fragmentOffset == 0 {
+			switch icmp.Type() {
+			case header.ICMPv4EchoReply:
+				icmpType = "echo reply"
+			case header.ICMPv4DstUnreachable:
+				icmpType = "destination unreachable"
+			case header.ICMPv4SrcQuench:
+				icmpType = "source quench"
+			case header.ICMPv4Redirect:
+				icmpType = "redirect"
+			case header.ICMPv4Echo:
+				icmpType = "echo"
+			case header.ICMPv4TimeExceeded:
+				icmpType = "time exceeded"
+			case header.ICMPv4ParamProblem:
+				icmpType = "param problem"
+			case header.ICMPv4Timestamp:
+				icmpType = "timestamp"
+			case header.ICMPv4TimestampReply:
+				icmpType = "timestamp reply"
+			case header.ICMPv4InfoRequest:
+				icmpType = "info request"
+			case header.ICMPv4InfoReply:
+				icmpType = "info reply"
+			}
+		}
+		log.Infof("%s%s %s %s -> %s %s len:%d id:%04x code:%d", prefix, dir, transName, src, dst, icmpType, size, id, icmp.Code())
+		return
+
+	case header.ICMPv6ProtocolNumber:
+		transName = "icmp"
+		hdr, ok := clone.Data().PullUp(header.ICMPv6MinimumSize)
+		if !ok {
+			break
+		}
+		icmp := header.ICMPv6(hdr)
+		icmpType := "unknown"
+		switch icmp.Type() {
+		case header.ICMPv6DstUnreachable:
+			icmpType = "destination unreachable"
+		case header.ICMPv6PacketTooBig:
+			icmpType = "packet too big"
+		case header.ICMPv6TimeExceeded:
+			icmpType = "time exceeded"
+		case header.ICMPv6ParamProblem:
+			icmpType = "param problem"
+		case header.ICMPv6EchoRequest:
+			icmpType = "echo request"
+		case header.ICMPv6EchoReply:
+			icmpType = "echo reply"
+		case header.ICMPv6RouterSolicit:
+			icmpType = "router solicit"
+		case header.ICMPv6RouterAdvert:
+			icmpType = "router advert"
+		case header.ICMPv6NeighborSolicit:
+			icmpType = "neighbor solicit"
+		case header.ICMPv6NeighborAdvert:
+			icmpType = "neighbor advert"
+		case header.ICMPv6RedirectMsg:
+			icmpType = "redirect message"
+		}
+		log.Infof("%s%s %s %s -> %s %s len:%d id:%04x code:%d", prefix, dir, transName, src, dst, icmpType, size, id, icmp.Code())
+		return
+
+	case header.UDPProtocolNumber:
+		transName = "udp"
+		if ok := parse.UDP(clone); !ok {
+			break
+		}
+
+		udp := header.UDP(clone.TransportHeader().Slice())
+		if fragmentOffset == 0 {
+			srcPort = udp.SourcePort()
+			dstPort = udp.DestinationPort()
+			details = fmt.Sprintf("xsum: 0x%x", udp.Checksum())
+			size -= header.UDPMinimumSize
+		}
+
+	case header.TCPProtocolNumber:
+		transName = "tcp"
+		if ok := parse.TCP(clone); !ok {
+			break
+		}
+
+		tcp := header.TCP(clone.TransportHeader().Slice())
+		if fragmentOffset == 0 {
+			offset := int(tcp.DataOffset())
+			if offset < header.TCPMinimumSize {
+				details += fmt.Sprintf("invalid packet: tcp data offset too small %d", offset)
+				break
+			}
+			if size := clone.Data().Size() + len(tcp); offset > size && !moreFragments {
+				details += fmt.Sprintf("invalid packet: tcp data offset %d larger than tcp packet length %d", offset, size)
+				break
+			}
+
+			srcPort = tcp.SourcePort()
+			dstPort = tcp.DestinationPort()
+			size -= uint16(offset)
+
+			// Initialize the TCP flags.
+			flags := tcp.Flags()
+			details = fmt.Sprintf("flags:%s seqnum:%d ack:%d win:%d xsum:0x%x", flags, tcp.SequenceNumber(), tcp.AckNumber(), tcp.WindowSize(), tcp.Checksum())
+			if flags&header.TCPFlagSyn != 0 {
+				details += fmt.Sprintf(" options:%+v", header.ParseSynOptions(tcp.Options(), flags&header.TCPFlagAck != 0))
+			} else {
+				details += fmt.Sprintf(" options:%+v", tcp.ParsedOptions())
+			}
+		}
+
+	default:
+		log.Infof("%s%s %s -> %s unknown transport protocol: %d", prefix, dir, src, dst, transProto)
+		return
+	}
+
+	if pkt.GSOOptions.Type != stack.GSONone {
+		details += fmt.Sprintf(" gso:%#v", pkt.GSOOptions)
+	}
+
+	log.Infof("%s%s %s %s:%d -> %s:%d len:%d id:0x%04x %s", prefix, dir, transName, src, srcPort, dst, dstPort, size, id, details)
+}
+
+// trimmedClone clones the packet buffer to not modify the original. It trims
+// anything before the network header.
+func trimmedClone(pkt *stack.PacketBuffer) *stack.PacketBuffer {
+	// We don't clone the original packet buffer so that the new packet buffer
+	// does not have any of its headers set.
+	//
+	// We trim the link headers from the cloned buffer as the sniffer doesn't
+	// handle link headers.
+	buf := pkt.ToBuffer()
+	buf.TrimFront(int64(len(pkt.VirtioNetHeader().Slice())))
+	buf.TrimFront(int64(len(pkt.LinkHeader().Slice())))
+	return stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf})
+}

pkg/tcpip/link/sniffer/sniffer_state_autogen.go

@@ -0,0 +1,47 @@
+// automatically generated by stateify.
+
+package sniffer
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/sniffer.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"Endpoint",
+		"writer",
+		"maxPCAPLen",
+		"logPrefix",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.Endpoint)
+	stateSinkObject.Save(1, &e.writer)
+	stateSinkObject.Save(2, &e.maxPCAPLen)
+	stateSinkObject.Save(3, &e.logPrefix)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.Endpoint)
+	stateSourceObject.Load(1, &e.writer)
+	stateSourceObject.Load(2, &e.maxPCAPLen)
+	stateSourceObject.Load(3, &e.logPrefix)
+}
+
+func init() {
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/stopfd/stopfd.go

@@ -0,0 +1,54 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+// Package stopfd provides an type that can be used to signal the stop of a dispatcher.
+package stopfd
+
+import (
+	"fmt"
+
+	"golang.org/x/sys/unix"
+)
+
+// StopFD is an eventfd used to signal the stop of a dispatcher.
+//
+// +stateify savable
+type StopFD struct {
+	EFD int
+}
+
+// New returns a new, initialized StopFD.
+func New() (StopFD, error) {
+	efd, err := unix.Eventfd(0, unix.EFD_NONBLOCK)
+	if err != nil {
+		return StopFD{EFD: -1}, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+	return StopFD{EFD: efd}, nil
+}
+
+// Stop writes to the eventfd and notifies the dispatcher to stop. It does not
+// block.
+func (sf *StopFD) Stop() {
+	increment := []byte{1, 0, 0, 0, 0, 0, 0, 0}
+	if n, err := unix.Write(sf.EFD, increment); n != len(increment) || err != nil {
+		// There are two possible errors documented in eventfd(2) for writing:
+		// 1. We are writing 8 bytes and not 0xffffffffffffff, thus no EINVAL.
+		// 2. stop is only supposed to be called once, it can't reach the limit,
+		// thus no EAGAIN.
+		panic(fmt.Sprintf("write(EFD) = (%d, %s), want (%d, nil)", n, err, len(increment)))
+	}
+}

pkg/tcpip/link/stopfd/stopfd_state_autogen.go

@@ -0,0 +1,41 @@
+// automatically generated by stateify.
+
+//go:build linux
+// +build linux
+
+package stopfd
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (sf *StopFD) StateTypeName() string {
+	return "pkg/tcpip/link/stopfd.StopFD"
+}
+
+func (sf *StopFD) StateFields() []string {
+	return []string{
+		"EFD",
+	}
+}
+
+func (sf *StopFD) beforeSave() {}
+
+// +checklocksignore
+func (sf *StopFD) StateSave(stateSinkObject state.Sink) {
+	sf.beforeSave()
+	stateSinkObject.Save(0, &sf.EFD)
+}
+
+func (sf *StopFD) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (sf *StopFD) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &sf.EFD)
+}
+
+func init() {
+	state.Register((*StopFD)(nil))
+}

pkg/tcpip/link/tun/device.go

@@ -0,0 +1,447 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tun
+
+import (
+	"fmt"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/context"
+	"github.com/sagernet/gvisor/pkg/errors/linuxerr"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/channel"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/packetsocket"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"github.com/sagernet/gvisor/pkg/waiter"
+)
+
+const (
+	// drivers/net/tun.c:tun_net_init()
+	defaultDevMtu = 1500
+
+	// Queue length for outbound packet, arriving at fd side for read. Overflow
+	// causes packet drops. gVisor implementation-specific.
+	defaultDevOutQueueLen = 1024
+)
+
+var zeroMAC [6]byte
+
+// Device is an opened /dev/net/tun device.
+//
+// +stateify savable
+type Device struct {
+	waiter.Queue
+
+	mu           deviceRWMutex `state:"nosave"`
+	endpoint     *tunEndpoint
+	notifyHandle *channel.NotificationHandle
+	flags        Flags
+}
+
+// Flags set properties of a Device
+//
+// +stateify savable
+type Flags struct {
+	TUN          bool
+	TAP          bool
+	NoPacketInfo bool
+	Exclusive    bool
+}
+
+// beforeSave is invoked by stateify.
+func (d *Device) beforeSave() {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	// TODO(b/110961832): Restore the device to stack. At this moment, the stack
+	// is not savable.
+	if d.endpoint != nil {
+		panic("/dev/net/tun does not support save/restore when a device is associated with it.")
+	}
+}
+
+func (d *Device) SetPersistent(v bool) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if d.endpoint == nil {
+		return linuxerr.EBADFD
+	}
+
+	d.endpoint.setPersistent(v)
+
+	return nil
+}
+
+// Release implements fs.FileOperations.Release.
+func (d *Device) Release(ctx context.Context) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	// Decrease refcount if there is an endpoint associated with this file.
+	if d.endpoint != nil {
+		d.endpoint.Drain()
+		d.endpoint.RemoveNotify(d.notifyHandle)
+		d.endpoint.DecRef(ctx)
+		d.endpoint = nil
+	}
+}
+
+// SetIff services TUNSETIFF ioctl(2) request.
+func (d *Device) SetIff(ctx context.Context, s *stack.Stack, name string, flags Flags) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if d.endpoint != nil {
+		return linuxerr.EINVAL
+	}
+
+	// Input validation.
+	if (flags.TAP && flags.TUN) || (!flags.TAP && !flags.TUN) {
+		return linuxerr.EINVAL
+	}
+
+	prefix := "tun"
+	if flags.TAP {
+		prefix = "tap"
+	}
+
+	linkCaps := stack.CapabilityNone
+	if flags.TAP {
+		linkCaps |= stack.CapabilityResolutionRequired
+	}
+
+	endpoint, err := attachOrCreateNIC(ctx, s, name, prefix, linkCaps, flags)
+	if err != nil {
+		return err
+	}
+
+	d.endpoint = endpoint
+	d.notifyHandle = d.endpoint.AddNotify(d)
+	d.flags = flags
+	return nil
+}
+
+func attachOrCreateNIC(ctx context.Context, s *stack.Stack, name, prefix string, linkCaps stack.LinkEndpointCapabilities, flags Flags) (*tunEndpoint, error) {
+	for {
+		// 1. Try to attach to an existing NIC.
+		if name != "" && !flags.Exclusive {
+			if linkEP := s.GetLinkEndpointByName(name); linkEP != nil {
+				packetEndpoint, ok := linkEP.(*packetsocket.Endpoint)
+				if !ok {
+					// Not a NIC created by tun device.
+					return nil, linuxerr.EOPNOTSUPP
+				}
+				endpoint, ok := packetEndpoint.Child().(*tunEndpoint)
+				if !ok {
+					// Not a NIC created by tun device.
+					return nil, linuxerr.EOPNOTSUPP
+				}
+				if !endpoint.TryIncRef() {
+					// Race detected: NIC got deleted in between.
+					continue
+				}
+				return endpoint, nil
+			}
+		}
+
+		// 2. Creating a new NIC.
+		id := s.NextNICID()
+		endpoint := &tunEndpoint{
+			Endpoint: channel.New(defaultDevOutQueueLen, defaultDevMtu, ""),
+			stack:    s,
+			nicID:    id,
+			name:     name,
+			isTap:    prefix == "tap",
+		}
+		endpoint.InitRefs()
+		endpoint.Endpoint.LinkEPCapabilities = linkCaps
+		if endpoint.name == "" {
+			endpoint.name = fmt.Sprintf("%s%d", prefix, id)
+		}
+		err := s.CreateNICWithOptions(endpoint.nicID, packetsocket.New(endpoint), stack.NICOptions{
+			Name: endpoint.name,
+		})
+		switch err.(type) {
+		case nil:
+			return endpoint, nil
+		case *tcpip.ErrDuplicateNICID:
+			endpoint.DecRef(ctx)
+			if !flags.Exclusive {
+				// Race detected: A NIC has been created in between.
+				continue
+			}
+			return nil, linuxerr.EEXIST
+		default:
+			endpoint.DecRef(ctx)
+			return nil, linuxerr.EINVAL
+		}
+	}
+}
+
+// MTU returns the tun endpoint MTU (maximum transmission unit).
+func (d *Device) MTU() (uint32, error) {
+	d.mu.RLock()
+	endpoint := d.endpoint
+	d.mu.RUnlock()
+	if endpoint == nil {
+		return 0, linuxerr.EBADFD
+	}
+	if !endpoint.IsAttached() {
+		return 0, linuxerr.EIO
+	}
+	return endpoint.MTU(), nil
+}
+
+// Write inject one inbound packet to the network interface.
+func (d *Device) Write(data *buffer.View) (int64, error) {
+	d.mu.RLock()
+	endpoint := d.endpoint
+	d.mu.RUnlock()
+	if endpoint == nil {
+		return 0, linuxerr.EBADFD
+	}
+	if !endpoint.IsAttached() {
+		return 0, linuxerr.EIO
+	}
+
+	dataLen := int64(data.Size())
+
+	// Packet information.
+	var pktInfoHdr PacketInfoHeader
+	if !d.flags.NoPacketInfo {
+		if dataLen < PacketInfoHeaderSize {
+			// Ignore bad packet.
+			return dataLen, nil
+		}
+		pktInfoHdrView := data.Clone()
+		defer pktInfoHdrView.Release()
+		pktInfoHdrView.CapLength(PacketInfoHeaderSize)
+		pktInfoHdr = PacketInfoHeader(pktInfoHdrView.AsSlice())
+		data.TrimFront(PacketInfoHeaderSize)
+	}
+
+	// Ethernet header (TAP only).
+	var ethHdr header.Ethernet
+	if d.flags.TAP {
+		if data.Size() < header.EthernetMinimumSize {
+			// Ignore bad packet.
+			return dataLen, nil
+		}
+		ethHdrView := data.Clone()
+		defer ethHdrView.Release()
+		ethHdrView.CapLength(header.EthernetMinimumSize)
+		ethHdr = header.Ethernet(ethHdrView.AsSlice())
+		data.TrimFront(header.EthernetMinimumSize)
+	}
+
+	// Try to determine network protocol number, default zero.
+	var protocol tcpip.NetworkProtocolNumber
+	switch {
+	case pktInfoHdr != nil:
+		protocol = pktInfoHdr.Protocol()
+	case ethHdr != nil:
+		protocol = ethHdr.Type()
+	case d.flags.TUN:
+		// TUN interface with IFF_NO_PI enabled, thus
+		// we need to determine protocol from version field
+		version := data.AsSlice()[0] >> 4
+		if version == 4 {
+			protocol = header.IPv4ProtocolNumber
+		} else if version == 6 {
+			protocol = header.IPv6ProtocolNumber
+		}
+	}
+
+	pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+		ReserveHeaderBytes: len(ethHdr),
+		Payload:            buffer.MakeWithView(data.Clone()),
+	})
+	defer pkt.DecRef()
+	copy(pkt.LinkHeader().Push(len(ethHdr)), ethHdr)
+	endpoint.InjectInbound(protocol, pkt)
+	return dataLen, nil
+}
+
+// Read reads one outgoing packet from the network interface.
+func (d *Device) Read() (*buffer.View, error) {
+	d.mu.RLock()
+	endpoint := d.endpoint
+	d.mu.RUnlock()
+	if endpoint == nil {
+		return nil, linuxerr.EBADFD
+	}
+
+	pkt := endpoint.Read()
+	if pkt == nil {
+		return nil, linuxerr.ErrWouldBlock
+	}
+	v := d.encodePkt(pkt)
+	pkt.DecRef()
+	return v, nil
+}
+
+// encodePkt encodes packet for fd side.
+func (d *Device) encodePkt(pkt *stack.PacketBuffer) *buffer.View {
+	var view *buffer.View
+
+	// Packet information.
+	if !d.flags.NoPacketInfo {
+		view = buffer.NewView(PacketInfoHeaderSize + pkt.Size())
+		view.Grow(PacketInfoHeaderSize)
+		hdr := PacketInfoHeader(view.AsSlice())
+		hdr.Encode(&PacketInfoFields{
+			Protocol: pkt.NetworkProtocolNumber,
+		})
+		pktView := pkt.ToView()
+		view.Write(pktView.AsSlice())
+		pktView.Release()
+	} else {
+		view = pkt.ToView()
+	}
+
+	return view
+}
+
+// Name returns the name of the attached network interface. Empty string if
+// unattached.
+func (d *Device) Name() string {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+	if d.endpoint != nil {
+		return d.endpoint.name
+	}
+	return ""
+}
+
+// Flags returns the flags set for d. Zero value if unset.
+func (d *Device) Flags() Flags {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+	return d.flags
+}
+
+// Readiness implements watier.Waitable.Readiness.
+func (d *Device) Readiness(mask waiter.EventMask) waiter.EventMask {
+	if mask&waiter.ReadableEvents != 0 {
+		d.mu.RLock()
+		endpoint := d.endpoint
+		d.mu.RUnlock()
+		if endpoint != nil && endpoint.NumQueued() == 0 {
+			mask &= ^waiter.ReadableEvents
+		}
+	}
+	return mask & (waiter.ReadableEvents | waiter.WritableEvents)
+}
+
+// WriteNotify implements channel.Notification.WriteNotify.
+func (d *Device) WriteNotify() {
+	d.Notify(waiter.ReadableEvents)
+}
+
+// tunEndpoint is the link endpoint for the NIC created by the tun device.
+//
+// It is ref-counted as multiple opening files can attach to the same NIC.
+// The last owner is responsible for deleting the NIC.
+//
+// +stateify savable
+type tunEndpoint struct {
+	tunEndpointRefs
+	*channel.Endpoint
+
+	stack      *stack.Stack
+	nicID      tcpip.NICID
+	name       string
+	isTap      bool
+	persistent atomicbitops.Bool
+	closed     atomicbitops.Bool
+
+	mu            endpointMutex `state:"nosave"`
+	onCloseAction func()        `state:"nosave"`
+}
+
+func (e *tunEndpoint) setPersistent(v bool) {
+	old := e.persistent.Swap(v)
+	if old == v {
+		return
+	}
+	if v {
+		e.IncRef()
+	} else {
+		e.DecRef(context.Background())
+	}
+}
+
+func (e *tunEndpoint) Close() {
+	if e.closed.Swap(true) {
+		return
+	}
+
+	if e.persistent.Load() {
+		e.DecRef(context.Background())
+	}
+	e.mu.Lock()
+	action := e.onCloseAction
+	e.onCloseAction = nil
+	e.mu.Unlock()
+	if action != nil {
+		action()
+	}
+	e.Endpoint.Close()
+}
+
+// SetOnCloseAction implements stack.LinkEndpoint.
+func (e *tunEndpoint) SetOnCloseAction(action func()) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.onCloseAction = action
+}
+
+// DecRef decrements refcount of e, removing NIC if it reaches 0.
+func (e *tunEndpoint) DecRef(ctx context.Context) {
+	e.tunEndpointRefs.DecRef(func() {
+		e.Close()
+	})
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (e *tunEndpoint) ARPHardwareType() header.ARPHardwareType {
+	if e.isTap {
+		return header.ARPHardwareEther
+	}
+	return header.ARPHardwareNone
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *tunEndpoint) AddHeader(pkt *stack.PacketBuffer) {
+	if !e.isTap {
+		return
+	}
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	eth.Encode(&header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	})
+}
+
+// MaxHeaderLength returns the maximum size of the link layer header.
+func (e *tunEndpoint) MaxHeaderLength() uint16 {
+	if e.isTap {
+		return header.EthernetMinimumSize
+	}
+	return 0
+}

pkg/tcpip/link/tun/device_mutex.go

@@ -0,0 +1,96 @@
+package tun
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type deviceRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var devicelockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type devicelockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *deviceRWMutex) Lock() {
+	locking.AddGLock(deviceprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *deviceRWMutex) NestedLock(i devicelockNameIndex) {
+	locking.AddGLock(deviceprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *deviceRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(deviceprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *deviceRWMutex) NestedUnlock(i devicelockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(deviceprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *deviceRWMutex) RLock() {
+	locking.AddGLock(deviceprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *deviceRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(deviceprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *deviceRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *deviceRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *deviceRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var deviceprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func deviceinitLockNames() {}
+
+func init() {
+	deviceinitLockNames()
+	deviceprefixIndex = locking.NewMutexClass(reflect.TypeOf(deviceRWMutex{}), devicelockNames)
+}

pkg/tcpip/link/tun/endpoint_mutex.go

@@ -0,0 +1,64 @@
+package tun
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// Mutex is sync.Mutex with the correctness validator.
+type endpointMutex struct {
+	mu sync.Mutex
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointMutex) Unlock() {
+	locking.DelGLock(endpointprefixIndex, -1)
+	m.mu.Unlock()
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointMutex) NestedUnlock(i endpointlockNameIndex) {
+	locking.DelGLock(endpointprefixIndex, int(i))
+	m.mu.Unlock()
+}
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointMutex{}), endpointlockNames)
+}

pkg/tcpip/link/tun/protocol.go

@@ -0,0 +1,56 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tun
+
+import (
+	"encoding/binary"
+
+	"github.com/sagernet/gvisor/pkg/tcpip"
+)
+
+const (
+	// PacketInfoHeaderSize is the size of the packet information header.
+	PacketInfoHeaderSize = 4
+
+	offsetFlags    = 0
+	offsetProtocol = 2
+)
+
+// PacketInfoFields contains fields sent through the wire if IFF_NO_PI flag is
+// not set.
+type PacketInfoFields struct {
+	Flags    uint16
+	Protocol tcpip.NetworkProtocolNumber
+}
+
+// PacketInfoHeader is the wire representation of the packet information sent if
+// IFF_NO_PI flag is not set.
+type PacketInfoHeader []byte
+
+// Encode encodes f into h.
+func (h PacketInfoHeader) Encode(f *PacketInfoFields) {
+	binary.BigEndian.PutUint16(h[offsetFlags:][:2], f.Flags)
+	binary.BigEndian.PutUint16(h[offsetProtocol:][:2], uint16(f.Protocol))
+}
+
+// Flags returns the flag field in h.
+func (h PacketInfoHeader) Flags() uint16 {
+	return binary.BigEndian.Uint16(h[offsetFlags:])
+}
+
+// Protocol returns the protocol field in h.
+func (h PacketInfoHeader) Protocol() tcpip.NetworkProtocolNumber {
+	return tcpip.NetworkProtocolNumber(binary.BigEndian.Uint16(h[offsetProtocol:]))
+}

pkg/tcpip/link/tun/tun_endpoint_refs.go

@@ -0,0 +1,141 @@
+package tun
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/sagernet/gvisor/pkg/atomicbitops"
+	"github.com/sagernet/gvisor/pkg/refs"
+)
+
+// enableLogging indicates whether reference-related events should be logged (with
+// stack traces). This is false by default and should only be set to true for
+// debugging purposes, as it can generate an extremely large amount of output
+// and drastically degrade performance.
+const tunEndpointenableLogging = false
+
+// obj is used to customize logging. Note that we use a pointer to T so that
+// we do not copy the entire object when passed as a format parameter.
+var tunEndpointobj *tunEndpoint
+
+// Refs implements refs.RefCounter. It keeps a reference count using atomic
+// operations and calls the destructor when the count reaches zero.
+//
+// NOTE: Do not introduce additional fields to the Refs struct. It is used by
+// many filesystem objects, and we want to keep it as small as possible (i.e.,
+// the same size as using an int64 directly) to avoid taking up extra cache
+// space. In general, this template should not be extended at the cost of
+// performance. If it does not offer enough flexibility for a particular object
+// (example: b/187877947), we should implement the RefCounter/CheckedObject
+// interfaces manually.
+//
+// +stateify savable
+type tunEndpointRefs struct {
+	// refCount is composed of two fields:
+	//
+	//	[32-bit speculative references]:[32-bit real references]
+	//
+	// Speculative references are used for TryIncRef, to avoid a CompareAndSwap
+	// loop. See IncRef, DecRef and TryIncRef for details of how these fields are
+	// used.
+	refCount atomicbitops.Int64
+}
+
+// InitRefs initializes r with one reference and, if enabled, activates leak
+// checking.
+func (r *tunEndpointRefs) InitRefs() {
+	r.refCount.RacyStore(1)
+	refs.Register(r)
+}
+
+// RefType implements refs.CheckedObject.RefType.
+func (r *tunEndpointRefs) RefType() string {
+	return fmt.Sprintf("%T", tunEndpointobj)[1:]
+}
+
+// LeakMessage implements refs.CheckedObject.LeakMessage.
+func (r *tunEndpointRefs) LeakMessage() string {
+	return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs())
+}
+
+// LogRefs implements refs.CheckedObject.LogRefs.
+func (r *tunEndpointRefs) LogRefs() bool {
+	return tunEndpointenableLogging
+}
+
+// ReadRefs returns the current number of references. The returned count is
+// inherently racy and is unsafe to use without external synchronization.
+func (r *tunEndpointRefs) ReadRefs() int64 {
+	return r.refCount.Load()
+}
+
+// IncRef implements refs.RefCounter.IncRef.
+//
+//go:nosplit
+func (r *tunEndpointRefs) IncRef() {
+	v := r.refCount.Add(1)
+	if tunEndpointenableLogging {
+		refs.LogIncRef(r, v)
+	}
+	if v <= 1 {
+		panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType()))
+	}
+}
+
+// TryIncRef implements refs.TryRefCounter.TryIncRef.
+//
+// To do this safely without a loop, a speculative reference is first acquired
+// on the object. This allows multiple concurrent TryIncRef calls to distinguish
+// other TryIncRef calls from genuine references held.
+//
+//go:nosplit
+func (r *tunEndpointRefs) TryIncRef() bool {
+	const speculativeRef = 1 << 32
+	if v := r.refCount.Add(speculativeRef); int32(v) == 0 {
+
+		r.refCount.Add(-speculativeRef)
+		return false
+	}
+
+	v := r.refCount.Add(-speculativeRef + 1)
+	if tunEndpointenableLogging {
+		refs.LogTryIncRef(r, v)
+	}
+	return true
+}
+
+// DecRef implements refs.RefCounter.DecRef.
+//
+// Note that speculative references are counted here. Since they were added
+// prior to real references reaching zero, they will successfully convert to
+// real references. In other words, we see speculative references only in the
+// following case:
+//
+//	A: TryIncRef [speculative increase => sees non-negative references]
+//	B: DecRef [real decrease]
+//	A: TryIncRef [transform speculative to real]
+//
+//go:nosplit
+func (r *tunEndpointRefs) DecRef(destroy func()) {
+	v := r.refCount.Add(-1)
+	if tunEndpointenableLogging {
+		refs.LogDecRef(r, v)
+	}
+	switch {
+	case v < 0:
+		panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType()))
+
+	case v == 0:
+		refs.Unregister(r)
+
+		if destroy != nil {
+			destroy()
+		}
+	}
+}
+
+func (r *tunEndpointRefs) afterLoad(context.Context) {
+	if r.ReadRefs() > 0 {
+		refs.Register(r)
+	}
+}

pkg/tcpip/link/tun/tun_state_autogen.go

@@ -0,0 +1,152 @@
+// automatically generated by stateify.
+
+package tun
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (d *Device) StateTypeName() string {
+	return "pkg/tcpip/link/tun.Device"
+}
+
+func (d *Device) StateFields() []string {
+	return []string{
+		"Queue",
+		"endpoint",
+		"notifyHandle",
+		"flags",
+	}
+}
+
+// +checklocksignore
+func (d *Device) StateSave(stateSinkObject state.Sink) {
+	d.beforeSave()
+	stateSinkObject.Save(0, &d.Queue)
+	stateSinkObject.Save(1, &d.endpoint)
+	stateSinkObject.Save(2, &d.notifyHandle)
+	stateSinkObject.Save(3, &d.flags)
+}
+
+func (d *Device) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (d *Device) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &d.Queue)
+	stateSourceObject.Load(1, &d.endpoint)
+	stateSourceObject.Load(2, &d.notifyHandle)
+	stateSourceObject.Load(3, &d.flags)
+}
+
+func (f *Flags) StateTypeName() string {
+	return "pkg/tcpip/link/tun.Flags"
+}
+
+func (f *Flags) StateFields() []string {
+	return []string{
+		"TUN",
+		"TAP",
+		"NoPacketInfo",
+		"Exclusive",
+	}
+}
+
+func (f *Flags) beforeSave() {}
+
+// +checklocksignore
+func (f *Flags) StateSave(stateSinkObject state.Sink) {
+	f.beforeSave()
+	stateSinkObject.Save(0, &f.TUN)
+	stateSinkObject.Save(1, &f.TAP)
+	stateSinkObject.Save(2, &f.NoPacketInfo)
+	stateSinkObject.Save(3, &f.Exclusive)
+}
+
+func (f *Flags) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (f *Flags) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &f.TUN)
+	stateSourceObject.Load(1, &f.TAP)
+	stateSourceObject.Load(2, &f.NoPacketInfo)
+	stateSourceObject.Load(3, &f.Exclusive)
+}
+
+func (e *tunEndpoint) StateTypeName() string {
+	return "pkg/tcpip/link/tun.tunEndpoint"
+}
+
+func (e *tunEndpoint) StateFields() []string {
+	return []string{
+		"tunEndpointRefs",
+		"Endpoint",
+		"stack",
+		"nicID",
+		"name",
+		"isTap",
+		"persistent",
+		"closed",
+	}
+}
+
+func (e *tunEndpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *tunEndpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.tunEndpointRefs)
+	stateSinkObject.Save(1, &e.Endpoint)
+	stateSinkObject.Save(2, &e.stack)
+	stateSinkObject.Save(3, &e.nicID)
+	stateSinkObject.Save(4, &e.name)
+	stateSinkObject.Save(5, &e.isTap)
+	stateSinkObject.Save(6, &e.persistent)
+	stateSinkObject.Save(7, &e.closed)
+}
+
+func (e *tunEndpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *tunEndpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.tunEndpointRefs)
+	stateSourceObject.Load(1, &e.Endpoint)
+	stateSourceObject.Load(2, &e.stack)
+	stateSourceObject.Load(3, &e.nicID)
+	stateSourceObject.Load(4, &e.name)
+	stateSourceObject.Load(5, &e.isTap)
+	stateSourceObject.Load(6, &e.persistent)
+	stateSourceObject.Load(7, &e.closed)
+}
+
+func (r *tunEndpointRefs) StateTypeName() string {
+	return "pkg/tcpip/link/tun.tunEndpointRefs"
+}
+
+func (r *tunEndpointRefs) StateFields() []string {
+	return []string{
+		"refCount",
+	}
+}
+
+func (r *tunEndpointRefs) beforeSave() {}
+
+// +checklocksignore
+func (r *tunEndpointRefs) StateSave(stateSinkObject state.Sink) {
+	r.beforeSave()
+	stateSinkObject.Save(0, &r.refCount)
+}
+
+// +checklocksignore
+func (r *tunEndpointRefs) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &r.refCount)
+	stateSourceObject.AfterLoad(func() { r.afterLoad(ctx) })
+}
+
+func init() {
+	state.Register((*Device)(nil))
+	state.Register((*Flags)(nil))
+	state.Register((*tunEndpoint)(nil))
+	state.Register((*tunEndpointRefs)(nil))
+}

pkg/tcpip/link/tun/tun_unsafe.go

@@ -0,0 +1,65 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+// Package tun contains methods to open TAP and TUN devices.
+package tun
+
+import (
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// Open opens the specified TUN device, sets it to non-blocking mode, and
+// returns its file descriptor.
+func Open(name string) (int, error) {
+	return open(name, unix.IFF_TUN|unix.IFF_NO_PI)
+}
+
+// OpenTAP opens the specified TAP device, sets it to non-blocking mode, and
+// returns its file descriptor.
+func OpenTAP(name string) (int, error) {
+	return open(name, unix.IFF_TAP|unix.IFF_NO_PI)
+}
+
+func open(name string, flags uint16) (int, error) {
+	fd, err := unix.Open("/dev/net/tun", unix.O_RDWR, 0)
+	if err != nil {
+		return -1, err
+	}
+
+	var ifr struct {
+		name  [16]byte
+		flags uint16
+		_     [22]byte
+	}
+
+	copy(ifr.name[:], name)
+	ifr.flags = flags
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), unix.TUNSETIFF, uintptr(unsafe.Pointer(&ifr)))
+	if errno != 0 {
+		unix.Close(fd)
+		return -1, errno
+	}
+
+	if err = unix.SetNonblock(fd, true); err != nil {
+		unix.Close(fd)
+		return -1, err
+	}
+
+	return fd, nil
+}

pkg/tcpip/link/tun/tun_unsafe_state_autogen.go

@@ -0,0 +1,6 @@
+// automatically generated by stateify.
+
+//go:build linux
+// +build linux
+
+package tun

pkg/tcpip/link/veth/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package veth
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/veth/veth.go

@@ -0,0 +1,258 @@
+// Copyright 2024 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package veth provides the implementation of virtual ethernet device pair.
+package veth
+
+import (
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+// DefaultBacklogSize is the default size of a veth device's buffer.
+const DefaultBacklogSize = 1000
+
+var (
+	_ stack.LinkEndpoint = (*Endpoint)(nil)
+	_ stack.GSOEndpoint  = (*Endpoint)(nil)
+)
+
+// +stateify savable
+type veth struct {
+	mu           vethRWMutex `state:"nosave"`
+	closed       bool
+	backlogQueue chan vethPacket `state:"nosave"`
+	mtu          uint32
+	endpoints    [2]Endpoint
+}
+
+func (v *veth) close() {
+	v.mu.Lock()
+	closed := v.closed
+	v.closed = true
+	v.mu.Unlock()
+	if closed {
+		return
+	}
+
+	for i := range v.endpoints {
+		e := &v.endpoints[i]
+		e.mu.Lock()
+		action := e.onCloseAction
+		e.onCloseAction = nil
+		e.mu.Unlock()
+		if action != nil {
+			action()
+		}
+	}
+	close(v.backlogQueue)
+}
+
+// +stateify savable
+type vethPacket struct {
+	e        *Endpoint
+	protocol tcpip.NetworkProtocolNumber
+	pkt      *stack.PacketBuffer
+}
+
+// Endpoint is link layer endpoint that redirects packets to a pair veth endpoint.
+//
+// +stateify savable
+type Endpoint struct {
+	peer *Endpoint
+
+	veth *veth
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+	// linkAddr is the local address of this endpoint.
+	//
+	// +checklocks:mu
+	linkAddr tcpip.LinkAddress
+	// +checklocks:mu
+	onCloseAction func() `state:"nosave"`
+}
+
+// NewPair creates a new veth pair.
+func NewPair(mtu, backlogQueueSize uint32) (*Endpoint, *Endpoint) {
+	veth := veth{
+		backlogQueue: make(chan vethPacket, backlogQueueSize),
+		mtu:          mtu,
+		endpoints: [2]Endpoint{
+			{
+				linkAddr: tcpip.GetRandMacAddr(),
+			},
+			{
+				linkAddr: tcpip.GetRandMacAddr(),
+			},
+		},
+	}
+	a := &veth.endpoints[0]
+	b := &veth.endpoints[1]
+	a.peer = b
+	b.peer = a
+	a.veth = &veth
+	b.veth = &veth
+	go func() {
+		for t := range veth.backlogQueue {
+			t.e.InjectInbound(t.protocol, t.pkt)
+			t.pkt.DecRef()
+		}
+	}()
+	return a, b
+}
+
+// Close closes e. Further packet injections will return an error, and all pending
+// packets are discarded. Close may be called concurrently with WritePackets.
+func (e *Endpoint) Close() {
+	e.veth.close()
+}
+
+// InjectInbound injects an inbound packet. If the endpoint is not attached, the
+// packet is not delivered.
+func (e *Endpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverNetworkPacket(protocol, pkt)
+	}
+}
+
+// Attach saves the stack network-layer dispatcher for use later when packets
+// are injected.
+func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dispatcher = dispatcher
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *Endpoint) IsAttached() bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.dispatcher != nil
+}
+
+// MTU implements stack.LinkEndpoint.MTU.
+func (e *Endpoint) MTU() uint32 {
+	e.veth.mu.RLock()
+	defer e.veth.mu.RUnlock()
+	return e.veth.mtu
+}
+
+// SetMTU implements stack.LinkEndpoint.SetMTU.
+func (e *Endpoint) SetMTU(mtu uint32) {
+	e.veth.mu.Lock()
+	defer e.veth.mu.Unlock()
+	e.veth.mtu = mtu
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	// TODO(b/352384218): Enable CapabilityTXChecksumOffload.
+	return stack.CapabilityRXChecksumOffload | stack.CapabilitySaveRestore
+}
+
+// GSOMaxSize implements stack.GSOEndpoint.
+func (*Endpoint) GSOMaxSize() uint32 {
+	return stack.GVisorGSOMaxSize
+}
+
+// SupportedGSO implements stack.GSOEndpoint.
+func (e *Endpoint) SupportedGSO() stack.SupportedGSO {
+	return stack.GVisorGSOSupported
+}
+
+// MaxHeaderLength returns the maximum size of the link layer header. Given it
+// doesn't have a header, it just returns 0.
+func (*Endpoint) MaxHeaderLength() uint16 {
+	return 0
+}
+
+// LinkAddress returns the link address of this endpoint.
+func (e *Endpoint) LinkAddress() tcpip.LinkAddress {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.linkAddr
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress.
+func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.linkAddr = addr
+}
+
+// WritePackets stores outbound packets into the channel.
+// Multiple concurrent calls are permitted.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	e.veth.mu.RLock()
+	defer e.veth.mu.RUnlock()
+
+	if e.veth.closed {
+		return 0, nil
+	}
+
+	n := 0
+	for _, pkt := range pkts.AsSlice() {
+		// In order to properly loop back to the inbound side we must create a
+		// fresh packet that only contains the underlying payload with no headers
+		// or struct fields set. We must deep clone the payload to avoid
+		// two goroutines writing to the same buffer.
+		//
+		// TODO(b/240580913): Remove this once IP headers use reference counted
+		// views instead of raw byte slices.
+		payload := pkt.ToBuffer()
+		newPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+			Payload: payload.DeepClone(),
+		})
+		payload.Release()
+		select {
+		case (e.veth.backlogQueue) <- vethPacket{
+			e:        e.peer,
+			protocol: pkt.NetworkProtocolNumber,
+			pkt:      newPkt,
+		}:
+			n++
+		default:
+			newPkt.DecRef()
+			return n, &tcpip.ErrNoBufferSpace{}
+		}
+	}
+	return n, nil
+}
+
+// Wait implements stack.LinkEndpoint.Wait.
+func (*Endpoint) Wait() {}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (*Endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareNone
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *Endpoint) AddHeader(pkt *stack.PacketBuffer) {}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool { return true }
+
+// SetOnCloseAction implements stack.LinkEndpoint.
+func (e *Endpoint) SetOnCloseAction(action func()) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.onCloseAction = action
+}

pkg/tcpip/link/veth/veth_mutex.go

@@ -0,0 +1,96 @@
+package veth
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type vethRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var vethlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type vethlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *vethRWMutex) Lock() {
+	locking.AddGLock(vethprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *vethRWMutex) NestedLock(i vethlockNameIndex) {
+	locking.AddGLock(vethprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *vethRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(vethprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *vethRWMutex) NestedUnlock(i vethlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(vethprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *vethRWMutex) RLock() {
+	locking.AddGLock(vethprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *vethRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(vethprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *vethRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *vethRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *vethRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var vethprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func vethinitLockNames() {}
+
+func init() {
+	vethinitLockNames()
+	vethprefixIndex = locking.NewMutexClass(reflect.TypeOf(vethRWMutex{}), vethlockNames)
+}

pkg/tcpip/link/veth/veth_state_autogen.go

@@ -0,0 +1,111 @@
+// automatically generated by stateify.
+
+package veth
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (v *veth) StateTypeName() string {
+	return "pkg/tcpip/link/veth.veth"
+}
+
+func (v *veth) StateFields() []string {
+	return []string{
+		"closed",
+		"mtu",
+		"endpoints",
+	}
+}
+
+func (v *veth) beforeSave() {}
+
+// +checklocksignore
+func (v *veth) StateSave(stateSinkObject state.Sink) {
+	v.beforeSave()
+	stateSinkObject.Save(0, &v.closed)
+	stateSinkObject.Save(1, &v.mtu)
+	stateSinkObject.Save(2, &v.endpoints)
+}
+
+func (v *veth) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (v *veth) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &v.closed)
+	stateSourceObject.Load(1, &v.mtu)
+	stateSourceObject.Load(2, &v.endpoints)
+}
+
+func (v *vethPacket) StateTypeName() string {
+	return "pkg/tcpip/link/veth.vethPacket"
+}
+
+func (v *vethPacket) StateFields() []string {
+	return []string{
+		"e",
+		"protocol",
+		"pkt",
+	}
+}
+
+func (v *vethPacket) beforeSave() {}
+
+// +checklocksignore
+func (v *vethPacket) StateSave(stateSinkObject state.Sink) {
+	v.beforeSave()
+	stateSinkObject.Save(0, &v.e)
+	stateSinkObject.Save(1, &v.protocol)
+	stateSinkObject.Save(2, &v.pkt)
+}
+
+func (v *vethPacket) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (v *vethPacket) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &v.e)
+	stateSourceObject.Load(1, &v.protocol)
+	stateSourceObject.Load(2, &v.pkt)
+}
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/veth.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"peer",
+		"veth",
+		"dispatcher",
+		"linkAddr",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.peer)
+	stateSinkObject.Save(1, &e.veth)
+	stateSinkObject.Save(2, &e.dispatcher)
+	stateSinkObject.Save(3, &e.linkAddr)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.peer)
+	stateSourceObject.Load(1, &e.veth)
+	stateSourceObject.Load(2, &e.dispatcher)
+	stateSourceObject.Load(3, &e.linkAddr)
+}
+
+func init() {
+	state.Register((*veth)(nil))
+	state.Register((*vethPacket)(nil))
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/waitable/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package waitable
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/waitable/waitable.go

@@ -0,0 +1,196 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package waitable provides the implementation of data-link layer endpoints
+// that wrap other endpoints, and can wait for inflight calls to WritePacket or
+// DeliverNetworkPacket to finish (and new ones to be prevented).
+//
+// Waitable endpoints can be used in the networking stack by calling New(eID) to
+// create a new endpoint, where eID is the ID of the endpoint being wrapped,
+// and then passing it as an argument to Stack.CreateNIC().
+package waitable
+
+import (
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+)
+
+var (
+	_ stack.NetworkDispatcher = (*Endpoint)(nil)
+	_ stack.LinkEndpoint      = (*Endpoint)(nil)
+)
+
+// Endpoint is a waitable link-layer endpoint.
+//
+// +stateify savable
+type Endpoint struct {
+	dispatchGate sync.Gate
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checklocks:mu
+	dispatcher stack.NetworkDispatcher
+
+	writeGate sync.Gate
+	lower     stack.LinkEndpoint
+}
+
+// New creates a new waitable link-layer endpoint. It wraps around another
+// endpoint and allows the caller to block new write/dispatch calls and wait for
+// the inflight ones to finish before returning.
+func New(lower stack.LinkEndpoint) *Endpoint {
+	return &Endpoint{
+		lower: lower,
+	}
+}
+
+// DeliverNetworkPacket implements stack.NetworkDispatcher.DeliverNetworkPacket.
+// It is called by the link-layer endpoint being wrapped when a packet arrives,
+// and only forwards to the actual dispatcher if Wait or WaitDispatch haven't
+// been called.
+func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	if !e.dispatchGate.Enter() {
+		return
+	}
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverNetworkPacket(protocol, pkt)
+	}
+	e.dispatchGate.Leave()
+}
+
+// DeliverLinkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	if !e.dispatchGate.Enter() {
+		return
+	}
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverLinkPacket(protocol, pkt)
+	}
+	e.dispatchGate.Leave()
+}
+
+// Attach implements stack.LinkEndpoint.Attach. It saves the dispatcher and
+// registers with the lower endpoint as its dispatcher so that "e" is called
+// for inbound packets.
+func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.mu.Lock()
+	e.dispatcher = dispatcher
+	e.mu.Unlock()
+	e.lower.Attach(e)
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (e *Endpoint) IsAttached() bool {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	return e.dispatcher != nil
+}
+
+// MTU implements stack.LinkEndpoint.MTU. It just forwards the request to the
+// lower endpoint.
+func (e *Endpoint) MTU() uint32 {
+	return e.lower.MTU()
+}
+
+// SetMTU implements stack.LinkEndpoint.SetMTU. It just forwards the request to
+// the lower endpoint.
+func (e *Endpoint) SetMTU(mtu uint32) {
+	e.lower.SetMTU(mtu)
+}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities. It just forwards the
+// request to the lower endpoint.
+func (e *Endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return e.lower.Capabilities()
+}
+
+// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength. It just
+// forwards the request to the lower endpoint.
+func (e *Endpoint) MaxHeaderLength() uint16 {
+	return e.lower.MaxHeaderLength()
+}
+
+// LinkAddress implements stack.LinkEndpoint.LinkAddress. It just forwards the
+// request to the lower endpoint.
+func (e *Endpoint) LinkAddress() tcpip.LinkAddress {
+	return e.lower.LinkAddress()
+}
+
+// SetLinkAddress implements stack.LinkEndpoint.SetLinkAddress. It forwards the
+// request to the lower endpoint.
+func (e *Endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.lower.SetLinkAddress(addr)
+}
+
+// WritePackets implements stack.LinkEndpoint.WritePackets. It is called by
+// higher-level protocols to write packets. It only forwards packets to the
+// lower endpoint if Wait or WaitWrite haven't been called.
+func (e *Endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	if !e.writeGate.Enter() {
+		return pkts.Len(), nil
+	}
+
+	n, err := e.lower.WritePackets(pkts)
+	e.writeGate.Leave()
+	return n, err
+}
+
+// WaitWrite prevents new calls to WritePacket from reaching the lower endpoint,
+// and waits for inflight ones to finish before returning.
+func (e *Endpoint) WaitWrite() {
+	e.writeGate.Close()
+}
+
+// WaitDispatch prevents new calls to DeliverNetworkPacket from reaching the
+// actual dispatcher, and waits for inflight ones to finish before returning.
+func (e *Endpoint) WaitDispatch() {
+	e.dispatchGate.Close()
+}
+
+// Wait implements stack.LinkEndpoint.Wait.
+func (e *Endpoint) Wait() {}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (e *Endpoint) ARPHardwareType() header.ARPHardwareType {
+	return e.lower.ARPHardwareType()
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (e *Endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	e.lower.AddHeader(pkt)
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (e *Endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	return e.lower.ParseHeader(pkt)
+}
+
+// SetOnCloseAction implements stack.LinkEndpoint.SetOnCloseAction.
+func (e *Endpoint) SetOnCloseAction(action func()) {
+	e.lower.SetOnCloseAction(action)
+}
+
+// Close implements stack.LinkEndpoint.
+func (e *Endpoint) Close() {
+	e.lower.Close()
+}

pkg/tcpip/link/waitable/waitable_state_autogen.go

@@ -0,0 +1,47 @@
+// automatically generated by stateify.
+
+package waitable
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (e *Endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/waitable.Endpoint"
+}
+
+func (e *Endpoint) StateFields() []string {
+	return []string{
+		"dispatchGate",
+		"dispatcher",
+		"writeGate",
+		"lower",
+	}
+}
+
+func (e *Endpoint) beforeSave() {}
+
+// +checklocksignore
+func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
+	e.beforeSave()
+	stateSinkObject.Save(0, &e.dispatchGate)
+	stateSinkObject.Save(1, &e.dispatcher)
+	stateSinkObject.Save(2, &e.writeGate)
+	stateSinkObject.Save(3, &e.lower)
+}
+
+func (e *Endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &e.dispatchGate)
+	stateSourceObject.Load(1, &e.dispatcher)
+	stateSourceObject.Load(2, &e.writeGate)
+	stateSourceObject.Load(3, &e.lower)
+}
+
+func init() {
+	state.Register((*Endpoint)(nil))
+}

pkg/tcpip/link/xdp/endpoint.go

@@ -0,0 +1,421 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+// Package xdp provides link layer endpoints backed by AF_XDP sockets.
+package xdp
+
+import (
+	"fmt"
+
+	"github.com/sagernet/gvisor/pkg/buffer"
+	"github.com/sagernet/gvisor/pkg/rawfile"
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/tcpip"
+	"github.com/sagernet/gvisor/pkg/tcpip/header"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/qdisc/fifo"
+	"github.com/sagernet/gvisor/pkg/tcpip/link/stopfd"
+	"github.com/sagernet/gvisor/pkg/tcpip/stack"
+	"github.com/sagernet/gvisor/pkg/xdp"
+	"golang.org/x/sys/unix"
+)
+
+// TODO(b/240191988): Turn off GSO, GRO, and LRO. Limit veth MTU to 1500.
+
+// MTU is sized to ensure packets fit inside a 2048 byte XDP frame.
+const MTU = 1500
+
+var _ stack.LinkEndpoint = (*endpoint)(nil)
+
+// +stateify savable
+type endpoint struct {
+	// fd is the underlying AF_XDP socket.
+	fd int
+
+	// caps holds the endpoint capabilities.
+	caps stack.LinkEndpointCapabilities
+
+	// closed is a function to be called when the FD's peer (if any) closes
+	// its end of the communication pipe.
+	// TODO(b/341946753): Restore when netstack is savable.
+	closed func(tcpip.Error) `state:"nosave"`
+
+	mu endpointRWMutex `state:"nosave"`
+	// +checkloks:mu
+	networkDispatcher stack.NetworkDispatcher
+
+	// wg keeps track of running goroutines.
+	wg sync.WaitGroup `state:"nosave"`
+
+	// control is used to control the AF_XDP socket.
+	control *xdp.ControlBlock
+
+	// stopFD is used to stop the dispatch loop.
+	stopFD stopfd.StopFD
+
+	// addr is the address of the endpoint.
+	//
+	// +checklocks:mu
+	addr tcpip.LinkAddress
+}
+
+// Options specify the details about the fd-based endpoint to be created.
+type Options struct {
+	// FD is used to read/write packets.
+	FD int
+
+	// ClosedFunc is a function to be called when an endpoint's peer (if
+	// any) closes its end of the communication pipe.
+	ClosedFunc func(tcpip.Error)
+
+	// Address is the link address for this endpoint.
+	Address tcpip.LinkAddress
+
+	// SaveRestore if true, indicates that this NIC capability set should
+	// include CapabilitySaveRestore
+	SaveRestore bool
+
+	// DisconnectOk if true, indicates that this NIC capability set should
+	// include CapabilityDisconnectOk.
+	DisconnectOk bool
+
+	// TXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityTXChecksumOffload.
+	TXChecksumOffload bool
+
+	// RXChecksumOffload if true, indicates that this endpoints capability
+	// set should include CapabilityRXChecksumOffload.
+	RXChecksumOffload bool
+
+	// InterfaceIndex is the interface index of the underlying device.
+	InterfaceIndex int
+
+	// Bind is true when we're responsible for binding the AF_XDP socket to
+	// a device. When false, another process is expected to bind for us.
+	Bind bool
+
+	// GRO enables generic receive offload.
+	GRO bool
+}
+
+// New creates a new endpoint from an AF_XDP socket.
+func New(opts *Options) (stack.LinkEndpoint, error) {
+	caps := stack.CapabilityResolutionRequired
+	if opts.RXChecksumOffload {
+		caps |= stack.CapabilityRXChecksumOffload
+	}
+
+	if opts.TXChecksumOffload {
+		caps |= stack.CapabilityTXChecksumOffload
+	}
+
+	if opts.SaveRestore {
+		caps |= stack.CapabilitySaveRestore
+	}
+
+	if opts.DisconnectOk {
+		caps |= stack.CapabilityDisconnectOk
+	}
+
+	if err := unix.SetNonblock(opts.FD, true); err != nil {
+		return nil, fmt.Errorf("unix.SetNonblock(%v) failed: %v", opts.FD, err)
+	}
+
+	ep := &endpoint{
+		fd:     opts.FD,
+		caps:   caps,
+		closed: opts.ClosedFunc,
+		addr:   opts.Address,
+	}
+
+	stopFD, err := stopfd.New()
+	if err != nil {
+		return nil, err
+	}
+	ep.stopFD = stopFD
+
+	// Use a 2MB UMEM to match the PACKET_MMAP dispatcher. There will be
+	// 1024 UMEM frames, and each queue will have 512 descriptors. Having
+	// fewer descriptors than frames prevents RX and TX from starving each
+	// other.
+	// TODO(b/240191988): Consider different numbers of descriptors for
+	// different queues.
+	const (
+		frameSize = 2048
+		umemSize  = 1 << 21
+		nFrames   = umemSize / frameSize
+	)
+	xdpOpts := xdp.Opts{
+		NFrames:      nFrames,
+		FrameSize:    frameSize,
+		NDescriptors: nFrames / 2,
+		Bind:         opts.Bind,
+	}
+	ep.control, err = xdp.NewFromSocket(opts.FD, uint32(opts.InterfaceIndex), 0 /* queueID */, xdpOpts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AF_XDP dispatcher: %v", err)
+	}
+
+	ep.control.UMEM.Lock()
+	defer ep.control.UMEM.Unlock()
+
+	ep.control.Fill.FillAll(&ep.control.UMEM)
+
+	return ep, nil
+}
+
+// Attach launches the goroutine that reads packets from the file descriptor and
+// dispatches them via the provided dispatcher. If one is already attached,
+// then nothing happens.
+//
+// Attach implements stack.LinkEndpoint.Attach.
+func (ep *endpoint) Attach(networkDispatcher stack.NetworkDispatcher) {
+	ep.mu.Lock()
+	defer ep.mu.Unlock()
+	// nil means the NIC is being removed.
+	if networkDispatcher == nil && ep.IsAttached() {
+		ep.stopFD.Stop()
+		ep.Wait()
+		ep.networkDispatcher = nil
+		return
+	}
+	if networkDispatcher != nil && ep.networkDispatcher == nil {
+		ep.networkDispatcher = networkDispatcher
+		// Link endpoints are not savable. When transportation endpoints are
+		// saved, they stop sending outgoing packets and all incoming packets
+		// are rejected.
+		ep.wg.Add(1)
+		go func() { // S/R-SAFE: See above.
+			defer ep.wg.Done()
+			for {
+				cont, err := ep.dispatch()
+				if err != nil || !cont {
+					if ep.closed != nil {
+						ep.closed(err)
+					}
+					return
+				}
+			}
+		}()
+	}
+}
+
+// IsAttached implements stack.LinkEndpoint.IsAttached.
+func (ep *endpoint) IsAttached() bool {
+	ep.mu.RLock()
+	defer ep.mu.RUnlock()
+	return ep.networkDispatcher != nil
+}
+
+// MTU implements stack.LinkEndpoint.MTU. It returns the value initialized
+// during construction.
+func (ep *endpoint) MTU() uint32 {
+	return MTU
+}
+
+// SetMTU implements stack.LinkEndpoint.SetMTU. It has no impact.
+func (*endpoint) SetMTU(uint32) {}
+
+// Capabilities implements stack.LinkEndpoint.Capabilities.
+func (ep *endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return ep.caps
+}
+
+// MaxHeaderLength returns the maximum size of the link-layer header.
+func (ep *endpoint) MaxHeaderLength() uint16 {
+	return uint16(header.EthernetMinimumSize)
+}
+
+// LinkAddress returns the link address of this endpoint.
+func (ep *endpoint) LinkAddress() tcpip.LinkAddress {
+	ep.mu.RLock()
+	defer ep.mu.RUnlock()
+	return ep.addr
+}
+
+// SetLinkAddress implemens stack.LinkEndpoint.SetLinkAddress
+func (ep *endpoint) SetLinkAddress(addr tcpip.LinkAddress) {
+	ep.mu.Lock()
+	defer ep.mu.Unlock()
+	ep.addr = addr
+}
+
+// Wait implements stack.LinkEndpoint.Wait. It waits for the endpoint to stop
+// reading from its FD.
+func (ep *endpoint) Wait() {
+	ep.wg.Wait()
+}
+
+// AddHeader implements stack.LinkEndpoint.AddHeader.
+func (ep *endpoint) AddHeader(pkt *stack.PacketBuffer) {
+	// Add ethernet header if needed.
+	eth := header.Ethernet(pkt.LinkHeader().Push(header.EthernetMinimumSize))
+	eth.Encode(&header.EthernetFields{
+		SrcAddr: pkt.EgressRoute.LocalLinkAddress,
+		DstAddr: pkt.EgressRoute.RemoteLinkAddress,
+		Type:    pkt.NetworkProtocolNumber,
+	})
+}
+
+// ParseHeader implements stack.LinkEndpoint.ParseHeader.
+func (ep *endpoint) ParseHeader(pkt *stack.PacketBuffer) bool {
+	_, ok := pkt.LinkHeader().Consume(header.EthernetMinimumSize)
+	return ok
+}
+
+// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType.
+func (ep *endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareEther
+}
+
+// WritePackets writes outbound packets to the underlying file descriptors. If
+// one is not currently writable, the packet is dropped.
+//
+// Each packet in pkts should have the following fields populated:
+//   - pkt.EgressRoute
+//   - pkt.NetworkProtocolNumber
+//
+// The following should not be populated, as GSO is not supported with XDP.
+//   - pkt.GSOOptions
+func (ep *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	// We expect to be called via fifo, which imposes a limit of
+	// fifo.BatchSize.
+	var preallocatedBatch [fifo.BatchSize]unix.XDPDesc
+	batch := preallocatedBatch[:0]
+
+	ep.control.UMEM.Lock()
+
+	ep.control.Completion.FreeAll(&ep.control.UMEM)
+
+	// Reserve TX queue descriptors and umem buffers
+	nReserved, index := ep.control.TX.Reserve(&ep.control.UMEM, uint32(pkts.Len()))
+	if nReserved == 0 {
+		ep.control.UMEM.Unlock()
+		return 0, &tcpip.ErrNoBufferSpace{}
+	}
+
+	// Allocate UMEM space. In order to release the UMEM lock as soon as
+	// possible we allocate up-front.
+	for _, pkt := range pkts.AsSlice() {
+		batch = append(batch, unix.XDPDesc{
+			Addr: ep.control.UMEM.AllocFrame(),
+			Len:  uint32(pkt.Size()),
+		})
+	}
+
+	for i, pkt := range pkts.AsSlice() {
+		// Copy packets into UMEM frame.
+		frame := ep.control.UMEM.Get(batch[i])
+		offset := 0
+		var view *buffer.View
+		views, pktOffset := pkt.AsViewList()
+		for view = views.Front(); view != nil && pktOffset >= view.Size(); view = view.Next() {
+			pktOffset -= view.Size()
+		}
+		offset += copy(frame[offset:], view.AsSlice()[pktOffset:])
+		for view = view.Next(); view != nil; view = view.Next() {
+			offset += copy(frame[offset:], view.AsSlice())
+		}
+		ep.control.TX.Set(index+uint32(i), batch[i])
+	}
+
+	// Notify the kernel that there're packets to write.
+	ep.control.TX.Notify()
+
+	// TODO(b/240191988): Explore more fine-grained locking. We shouldn't
+	// need to hold the UMEM lock for the whole duration of packet copying.
+	ep.control.UMEM.Unlock()
+
+	return pkts.Len(), nil
+}
+
+func (ep *endpoint) dispatch() (bool, tcpip.Error) {
+	var views []*buffer.View
+
+	for {
+		stopped, errno := rawfile.BlockingPollUntilStopped(ep.stopFD.EFD, ep.fd, unix.POLLIN|unix.POLLERR)
+		if errno != 0 {
+			if errno == unix.EINTR {
+				continue
+			}
+			return !stopped, tcpip.TranslateErrno(errno)
+		}
+		if stopped {
+			return true, nil
+		}
+
+		// Avoid the cost of the poll syscall if possible by peeking
+		// until there are no packets left.
+		for {
+			// We can receive multiple packets at once.
+			nReceived, rxIndex := ep.control.RX.Peek()
+
+			if nReceived == 0 {
+				break
+			}
+
+			// Reuse views to avoid allocating.
+			views = views[:0]
+
+			// Populate views quickly so that we can release frames
+			// back to the kernel.
+			ep.control.UMEM.Lock()
+			for i := uint32(0); i < nReceived; i++ {
+				// Copy packet bytes into a view and free up the
+				// buffer.
+				descriptor := ep.control.RX.Get(rxIndex + i)
+				data := ep.control.UMEM.Get(descriptor)
+				view := buffer.NewView(len(data))
+				view.Write(data)
+				views = append(views, view)
+				ep.control.UMEM.FreeFrame(descriptor.Addr)
+			}
+			ep.control.Fill.FillAll(&ep.control.UMEM)
+			ep.control.UMEM.Unlock()
+
+			// Process each packet.
+			ep.mu.RLock()
+			d := ep.networkDispatcher
+			ep.mu.RUnlock()
+			for i := uint32(0); i < nReceived; i++ {
+				view := views[i]
+				data := view.AsSlice()
+
+				netProto := header.Ethernet(data).Type()
+
+				// Wrap the packet in a PacketBuffer and send it up the stack.
+				pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+					Payload: buffer.MakeWithView(view),
+				})
+				// AF_XDP packets always have a link header.
+				if !ep.ParseHeader(pkt) {
+					panic("ParseHeader(_) must succeed")
+				}
+				d.DeliverNetworkPacket(netProto, pkt)
+				pkt.DecRef()
+			}
+			// Tell the kernel that we're done with these
+			// descriptors in the RX queue.
+			ep.control.RX.Release(nReceived)
+		}
+	}
+}
+
+// Close implements stack.LinkEndpoint.
+func (*endpoint) Close() {}
+
+// SetOnCloseAction implements stack.LinkEndpoint.
+func (*endpoint) SetOnCloseAction(func()) {}

pkg/tcpip/link/xdp/endpoint_mutex.go

@@ -0,0 +1,96 @@
+package xdp
+
+import (
+	"reflect"
+
+	"github.com/sagernet/gvisor/pkg/sync"
+	"github.com/sagernet/gvisor/pkg/sync/locking"
+)
+
+// RWMutex is sync.RWMutex with the correctness validator.
+type endpointRWMutex struct {
+	mu sync.RWMutex
+}
+
+// lockNames is a list of user-friendly lock names.
+// Populated in init.
+var endpointlockNames []string
+
+// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
+// referring to an index within lockNames.
+// Values are specified using the "consts" field of go_template_instance.
+type endpointlockNameIndex int
+
+// DO NOT REMOVE: The following function automatically replaced with lock index constants.
+// LOCK_NAME_INDEX_CONSTANTS
+const ()
+
+// Lock locks m.
+// +checklocksignore
+func (m *endpointRWMutex) Lock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.Lock()
+}
+
+// NestedLock locks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedLock(i endpointlockNameIndex) {
+	locking.AddGLock(endpointprefixIndex, int(i))
+	m.mu.Lock()
+}
+
+// Unlock unlocks m.
+// +checklocksignore
+func (m *endpointRWMutex) Unlock() {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// NestedUnlock unlocks m knowing that another lock of the same type is held.
+// +checklocksignore
+func (m *endpointRWMutex) NestedUnlock(i endpointlockNameIndex) {
+	m.mu.Unlock()
+	locking.DelGLock(endpointprefixIndex, int(i))
+}
+
+// RLock locks m for reading.
+// +checklocksignore
+func (m *endpointRWMutex) RLock() {
+	locking.AddGLock(endpointprefixIndex, -1)
+	m.mu.RLock()
+}
+
+// RUnlock undoes a single RLock call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlock() {
+	m.mu.RUnlock()
+	locking.DelGLock(endpointprefixIndex, -1)
+}
+
+// RLockBypass locks m for reading without executing the validator.
+// +checklocksignore
+func (m *endpointRWMutex) RLockBypass() {
+	m.mu.RLock()
+}
+
+// RUnlockBypass undoes a single RLockBypass call.
+// +checklocksignore
+func (m *endpointRWMutex) RUnlockBypass() {
+	m.mu.RUnlock()
+}
+
+// DowngradeLock atomically unlocks rw for writing and locks it for reading.
+// +checklocksignore
+func (m *endpointRWMutex) DowngradeLock() {
+	m.mu.DowngradeLock()
+}
+
+var endpointprefixIndex *locking.MutexClass
+
+// DO NOT REMOVE: The following function is automatically replaced.
+func endpointinitLockNames() {}
+
+func init() {
+	endpointinitLockNames()
+	endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
+}

pkg/tcpip/link/xdp/xdp_state_autogen.go

@@ -0,0 +1,56 @@
+// automatically generated by stateify.
+
+//go:build linux
+// +build linux
+
+package xdp
+
+import (
+	"context"
+
+	"github.com/sagernet/gvisor/pkg/state"
+)
+
+func (ep *endpoint) StateTypeName() string {
+	return "pkg/tcpip/link/xdp.endpoint"
+}
+
+func (ep *endpoint) StateFields() []string {
+	return []string{
+		"fd",
+		"caps",
+		"networkDispatcher",
+		"control",
+		"stopFD",
+		"addr",
+	}
+}
+
+func (ep *endpoint) beforeSave() {}
+
+// +checklocksignore
+func (ep *endpoint) StateSave(stateSinkObject state.Sink) {
+	ep.beforeSave()
+	stateSinkObject.Save(0, &ep.fd)
+	stateSinkObject.Save(1, &ep.caps)
+	stateSinkObject.Save(2, &ep.networkDispatcher)
+	stateSinkObject.Save(3, &ep.control)
+	stateSinkObject.Save(4, &ep.stopFD)
+	stateSinkObject.Save(5, &ep.addr)
+}
+
+func (ep *endpoint) afterLoad(context.Context) {}
+
+// +checklocksignore
+func (ep *endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &ep.fd)
+	stateSourceObject.Load(1, &ep.caps)
+	stateSourceObject.Load(2, &ep.networkDispatcher)
+	stateSourceObject.Load(3, &ep.control)
+	stateSourceObject.Load(4, &ep.stopFD)
+	stateSourceObject.Load(5, &ep.addr)
+}
+
+func init() {
+	state.Register((*endpoint)(nil))
+}