eburet-oss/gvisor-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

snapshot: sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 + SPEC 048 guard

Browse source 
Обновление снапшота с v0.0.0-20250811.0 на пин, которого требует
sing-box после мержа 235 коммитов (upstream d620bbbf2 "Update gvisor to
20260727.0"). Прежний снапшот был взят 2026-08-04 ровно с той версии,
на которой тогда стоял апстрим; разрыв возник 2026-08-05 вместе с его
бампом.

За год апстрим-gvisor изменил ~14 000 строк в 292 файлах. Значимое для
нас — сетевой стек: tcp/connect.go (PMTU-discovery + исправление
начального RTT/RTO: раньше задержка ACK внутри стека завышала стартовый
таймаут на несколько RTT), tcp/snd.go, tcp/rcv.go, stack/conntrack.go,
stack/packet_buffer.go. Всего 30 файлов в TCP и 37 в stack.

Баг SPEC 048 апстрим НЕ исправил — проверено по коду новой версии:
handleConnecting по-прежнему проверяет состояние endpoint'а, но не ep.h,
а performHandshake так же зануляет h и отпускает мьютекс до Close().
Поэтому guard перенесён (12 строк) вместе со своим тестом (45 строк).

Red/green проверен на новой базе: без guard'а тест падает с той же
nil-паникой, что в полевом крашдампе; с ним зелёный.
This commit is contained in:
Leadaxe 2026-08-05 14:53:31 +03:00
parent ffebe42860
commit 117243aa02
293 changed files with 16413 additions and 2842 deletions
Download patch file Download diff file Expand all files Collapse all files

32
pkg/tcpip/transport/icmp/endpoint_state.go
View file

@ -16,13 +16,11 @@ package icmp
import (
"context"
"fmt"
"time"
"github.com/sagernet/gvisor/pkg/log"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/tcpip/transport"
)
// saveReceivedAt is invoked by stateify.
@ -37,11 +35,7 @@ func (p *icmpPacket) loadReceivedAt(_ context.Context, nsec int64) {
// afterLoad is invoked by stateify.
func (e *endpoint) afterLoad(ctx context.Context) {
if e.stack.IsSaveRestoreEnabled() {
e.stack.RegisterRestoredEndpoint(e)
} else {
stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e)
}
e.stack.RegisterRestoredEndpoint(e)
}
// beforeSave is invoked by stateify.
@ -59,31 +53,7 @@ func (e *endpoint) Restore(s *stack.Stack) {
}
e.thaw()
if e.stack.IsSaveRestoreEnabled() {
e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
return
}
e.stack = s
e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
e.mu.Lock()
defer e.mu.Unlock()
switch state := e.net.State(); state {
case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed:
case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected:
var err tcpip.Error
info := e.net.Info()
info.ID.LocalPort = e.ident
info.ID, err = e.registerWithStack(info.NetProto, info.ID)
if err != nil {
panic(fmt.Sprintf("e.registerWithStack(%d, %#v): %s", info.NetProto, info.ID, err))
}
e.ident = info.ID.LocalPort
default:
panic(fmt.Sprintf("unhandled state = %s", state))
}
}
// Resume implements tcpip.ResumableEndpoint.Resume.

4
pkg/tcpip/transport/icmp/icmp_state_autogen.go
View file

@ -29,8 +29,8 @@ func (p *icmpPacket) beforeSave() {}
// +checklocksignore
func (p *icmpPacket) StateSave(stateSinkObject state.Sink) {
p.beforeSave()
var receivedAtValue int64
receivedAtValue = p.saveReceivedAt()
receivedAtValue := p.saveReceivedAt()
_ = (int64)(receivedAtValue)
stateSinkObject.SaveValue(4, receivedAtValue)
stateSinkObject.Save(0, &p.icmpPacketEntry)
stateSinkObject.Save(1, &p.senderAddress)

137
pkg/tcpip/transport/internal/network/endpoint.go
View file

@ -66,13 +66,16 @@ type Endpoint struct {
// TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6.
// +checklocks:mu
multicastAddr tcpip.Address
// TODO(https://gvisor.dev/issue/6389): Use different fields for IPv4/IPv6.
// +checklocks:mu
multicastNICID tcpip.NICID
// +checklocks:mu
ipv6MulticastNICID tcpip.NICID
// +checklocks:mu
ipv4TOS uint8
// +checklocks:mu
ipv6TClass uint8
// +checklocks:mu
pmtud tcpip.PMTUDStrategy
// Lock ordering: mu > infoMu.
infoMu sync.RWMutex `state:"nosave"`
@ -181,7 +184,11 @@ func (e *Endpoint) Close() {
}
for mem := range e.multicastMemberships {
e.stack.LeaveGroup(e.netProto, mem.nicID, mem.multicastAddr)
proto, err := e.multicastNetProto(mem.multicastAddr)
if err != nil {
panic("non multicast address in an existing membership")
}
e.stack.LeaveGroup(proto, mem.nicID, mem.multicastAddr)
}
e.multicastMemberships = nil
@ -229,6 +236,7 @@ type WriteContext struct {
route *stack.Route
ttl uint8
tos uint8
df bool
}
func (c *WriteContext) MTU() uint32 {
@ -274,33 +282,33 @@ func (c *WriteContext) TryNewPacketBuffer(reserveHdrBytes int, data buffer.Buffe
if !e.hasSendSpaceRLocked() {
return nil
}
return c.newPacketBufferLocked(reserveHdrBytes, data)
mark := e.ops.GetMark()
return c.newPacketBufferLocked(reserveHdrBytes, data, mark)
}
// TryNewPacketBufferFromPayloader returns a new packet buffer iff the endpoint's send buffer
// TryNewPacketBufferFromPayloader returns a new packet buffer if the endpoint's send buffer
// is not full. Otherwise, data from `payloader` isn't read.
//
// If this method returns nil, the caller should wait for the endpoint to become
// writable.
func (c *WriteContext) TryNewPacketBufferFromPayloader(reserveHdrBytes int, payloader tcpip.Payloader) *stack.PacketBuffer {
func (c *WriteContext) TryNewPacketBufferFromPayloader(reserveHdrBytes int, payloader tcpip.Payloader) (*stack.PacketBuffer, tcpip.Error) {
e := c.e
e.sendBufferSizeInUseMu.Lock()
defer e.sendBufferSizeInUseMu.Unlock()
if !e.hasSendSpaceRLocked() {
return nil
return nil, &tcpip.ErrWouldBlock{}
}
var data buffer.Buffer
if _, err := data.WriteFromReader(payloader, int64(payloader.Len())); err != nil {
data.Release()
return nil
return nil, &tcpip.ErrBadBuffer{}
}
return c.newPacketBufferLocked(reserveHdrBytes, data)
mark := e.ops.GetMark()
return c.newPacketBufferLocked(reserveHdrBytes, data, mark), nil
}
// +checklocks:c.e.sendBufferSizeInUseMu
func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Buffer) *stack.PacketBuffer {
func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Buffer, mark uint32) *stack.PacketBuffer {
e := c.e
// Note that we allow oversubscription - if there is any space at all in the
// send buffer, we accept the full packet which may be larger than the space
@ -323,6 +331,7 @@ func (c *WriteContext) newPacketBufferLocked(reserveHdrBytes int, data buffer.Bu
return stack.NewPacketBuffer(stack.PacketBufferOptions{
ReserveHeaderBytes: reserveHdrBytes,
Payload: data,
Mark: mark,
OnRelease: func() {
e.sendBufferSizeInUseMu.Lock()
if got := e.sendBufferSizeInUse; got < pktSize {
@ -360,6 +369,7 @@ func (c *WriteContext) WritePacket(pkt *stack.PacketBuffer, headerIncluded bool)
Protocol: c.e.transProto,
TTL: c.ttl,
TOS: c.tos,
DF: c.df,
ExperimentOptionValue: expOptVal,
}, pkt)
@ -433,8 +443,8 @@ func (e *Endpoint) AcquireContextForWrite(opts tcpip.WriteOptions) (WriteContext
route := e.connectedRoute
to := opts.To
info := e.Info()
switch {
case to == nil:
switch to {
case nil:
// If the user doesn't specify a destination, they should have
// connected to another address.
if e.State() != transport.DatagramEndpointStateConnected {
@ -566,11 +576,28 @@ func (e *Endpoint) AcquireContextForWrite(opts tcpip.WriteOptions) (WriteContext
panic(fmt.Sprintf("invalid protocol number = %d", netProto))
}
// Set the DF (Don't Fragment) bit based on the PMTUD strategy,
// matching TCP behavior in connect.go.
// Note: In gVisor, WANT and DO are treated identically (both set DF).
// Linux kernel differentiates them (WANT allows local fragmentation,
// DO returns EMSGSIZE), but gVisor's IPv4 layer always allows local
// fragmentation for locally-generated packets regardless of DF
// (see gvisor.dev/issue/5919).
//
// PROBE also sets DF, matching Linux ip_dont_fragment(). In Linux,
// PROBE differs from DO only in that it ignores incoming ICMP
// "Fragmentation Needed" messages (i.e. does not update the cached
// route PMTU). Since gVisor does not implement ICMP-based PMTU
// feedback for transport sockets, PROBE and DO are functionally
// equivalent here.
df := e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo || e.pmtud == tcpip.PMTUDiscoveryProbe
return WriteContext{
e: e,
route: route,
ttl: ttl,
tos: tos,
df: df,
}, nil
}
@ -613,7 +640,7 @@ func (e *Endpoint) connectRouteRLocked(nicID tcpip.NICID, localAddr tcpip.Addres
localAddr = tcpip.Address{}
}
if header.IsV4MulticastAddress(addr.Addr) || header.IsV6MulticastAddress(addr.Addr) {
if header.IsV4MulticastAddress(addr.Addr) {
if nicID == 0 {
nicID = e.multicastNICID
}
@ -621,6 +648,9 @@ func (e *Endpoint) connectRouteRLocked(nicID tcpip.NICID, localAddr tcpip.Addres
localAddr = e.multicastAddr
}
}
if header.IsV6MulticastAddress(addr.Addr) && nicID == 0 {
nicID = e.ipv6MulticastNICID
}
}
// Find a route to the desired destination.
@ -835,9 +865,18 @@ func (e *Endpoint) GetRemoteAddress() (tcpip.FullAddress, bool) {
func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error {
switch opt {
case tcpip.MTUDiscoverOption:
// Return not supported if the value is not disabling path
// MTU discovery.
if tcpip.PMTUDStrategy(v) != tcpip.PMTUDiscoveryDont {
// Store PMTU discovery settings. The DF bit on outgoing
// packets is set accordingly in AcquireContextForWrite.
// PROBE is accepted alongside DO/WANT/DONT. In Linux,
// PROBE sets DF but ignores ICMP-based PMTU updates;
// since gVisor lacks ICMP PMTU feedback, it behaves
// identically to DO.
switch tcpip.PMTUDStrategy(v) {
case tcpip.PMTUDiscoveryWant, tcpip.PMTUDiscoveryDont, tcpip.PMTUDiscoveryDo, tcpip.PMTUDiscoveryProbe:
e.mu.Lock()
e.pmtud = tcpip.PMTUDStrategy(v)
e.mu.Unlock()
default:
return &tcpip.ErrNotSupported{}
}
@ -865,6 +904,18 @@ func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error {
e.mu.Lock()
e.ipv6TClass = uint8(v)
e.mu.Unlock()
case tcpip.IPv6MulticastInterfaceOption:
if v != 0 && !e.stack.CheckNIC(tcpip.NICID(v)) {
return &tcpip.ErrUnknownNICID{}
}
e.mu.Lock()
defer e.mu.Unlock()
nic := tcpip.NICID(v)
if info := e.Info(); info.BindNICID != 0 && info.BindNICID != nic {
return &tcpip.ErrInvalidEndpointState{}
}
e.ipv6MulticastNICID = nic
}
return nil
@ -874,8 +925,10 @@ func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error {
func (e *Endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) {
switch opt {
case tcpip.MTUDiscoverOption:
// The only supported setting is path MTU discovery disabled.
return int(tcpip.PMTUDiscoveryDont), nil
e.mu.Lock()
v := int(e.pmtud)
e.mu.Unlock()
return v, nil
case tcpip.MulticastTTLOption:
e.mu.Lock()
@ -907,11 +960,30 @@ func (e *Endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, tcpip.Error) {
e.mu.RUnlock()
return v, nil
case tcpip.IPv6MulticastInterfaceOption:
e.mu.RLock()
v := int(e.ipv6MulticastNICID)
e.mu.RUnlock()
return v, nil
default:
return -1, &tcpip.ErrUnknownProtocolOption{}
}
}
// multicastNetProto returns the network protocol of a given multicast address.
// Returns an error if the address is not a multicast address.
func (e *Endpoint) multicastNetProto(addr tcpip.Address) (tcpip.NetworkProtocolNumber, tcpip.Error) {
switch {
case header.IsV4MulticastAddress(addr):
return header.IPv4ProtocolNumber, nil
case header.IsV6MulticastAddress(addr):
return header.IPv6ProtocolNumber, nil
default:
return 0, &tcpip.ErrInvalidOptionValue{}
}
}
// SetSockOpt sets the socket option.
func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error {
switch v := opt.(type) {
@ -952,21 +1024,23 @@ func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error {
e.multicastAddr = addr
case *tcpip.AddMembershipOption:
if !(header.IsV4MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv4ProtocolNumber) && !(header.IsV6MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv6ProtocolNumber) {
return &tcpip.ErrInvalidOptionValue{}
// Allowing IP_ADD_MEMBERSHIP on an ipv6 socket matches Linux behavior:
// https://github.com/torvalds/linux/blob/cec1e6e5d1a/net/ipv6/ipv6_sockglue.c#L964
proto, err := e.multicastNetProto(v.MulticastAddr)
if err != nil {
return err
}
nicID := v.NIC
if v.InterfaceAddr.Unspecified() {
if nicID == 0 {
if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, e.netProto, false /* multicastLoop */); err == nil {
if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, proto, false /* multicastLoop */); err == nil {
nicID = r.NICID()
r.Release()
}
}
} else {
nicID = e.stack.CheckLocalAddress(nicID, e.netProto, v.InterfaceAddr)
nicID = e.stack.CheckLocalAddress(nicID, proto, v.InterfaceAddr)
}
if nicID == 0 {
return &tcpip.ErrUnknownDevice{}
@ -981,27 +1055,28 @@ func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error {
return &tcpip.ErrPortInUse{}
}
if err := e.stack.JoinGroup(e.netProto, nicID, v.MulticastAddr); err != nil {
if err := e.stack.JoinGroup(proto, nicID, v.MulticastAddr); err != nil {
return err
}
e.multicastMemberships[memToInsert] = struct{}{}
case *tcpip.RemoveMembershipOption:
if !(header.IsV4MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv4ProtocolNumber) && !(header.IsV6MulticastAddress(v.MulticastAddr) && e.netProto == header.IPv6ProtocolNumber) {
return &tcpip.ErrInvalidOptionValue{}
proto, err := e.multicastNetProto(v.MulticastAddr)
if err != nil {
return err
}
nicID := v.NIC
if v.InterfaceAddr.Unspecified() {
if nicID == 0 {
if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, e.netProto, false /* multicastLoop */); err == nil {
if r, err := e.stack.FindRoute(0, tcpip.Address{}, v.MulticastAddr, proto, false /* multicastLoop */); err == nil {
nicID = r.NICID()
r.Release()
}
}
} else {
nicID = e.stack.CheckLocalAddress(nicID, e.netProto, v.InterfaceAddr)
nicID = e.stack.CheckLocalAddress(nicID, proto, v.InterfaceAddr)
}
if nicID == 0 {
return &tcpip.ErrUnknownDevice{}
@ -1016,7 +1091,7 @@ func (e *Endpoint) SetSockOpt(opt tcpip.SettableSocketOption) tcpip.Error {
return &tcpip.ErrBadLocalAddress{}
}
if err := e.stack.LeaveGroup(e.netProto, nicID, v.MulticastAddr); err != nil {
if err := e.stack.LeaveGroup(proto, nicID, v.MulticastAddr); err != nil {
return err
}

8
pkg/tcpip/transport/internal/network/endpoint_state.go
View file

@ -29,8 +29,12 @@ func (e *Endpoint) Resume(s *stack.Stack) error {
e.stack = s
for m := range e.multicastMemberships {
if err := e.stack.JoinGroup(e.netProto, m.nicID, m.multicastAddr); err != nil {
return fmt.Errorf("e.stack.JoinGroup(%d, %d, %s): %s", e.netProto, m.nicID, m.multicastAddr, err)
proto, err := e.multicastNetProto(m.multicastAddr)
if err != nil {
return fmt.Errorf("non multicast address in an existing membership during Resume: %s", err)
}
if err := e.stack.JoinGroup(proto, m.nicID, m.multicastAddr); err != nil {
return fmt.Errorf("e.stack.JoinGroup(%d, %d, %s): %s", proto, m.nicID, m.multicastAddr, err)
}
}

22
pkg/tcpip/transport/internal/network/network_state_autogen.go
View file

@ -29,8 +29,10 @@ func (e *Endpoint) StateFields() []string {
"multicastTTL",
"multicastAddr",
"multicastNICID",
"ipv6MulticastNICID",
"ipv4TOS",
"ipv6TClass",
"pmtud",
"info",
"state",
}
@ -56,10 +58,12 @@ func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
stateSinkObject.Save(12, &e.multicastTTL)
stateSinkObject.Save(13, &e.multicastAddr)
stateSinkObject.Save(14, &e.multicastNICID)
stateSinkObject.Save(15, &e.ipv4TOS)
stateSinkObject.Save(16, &e.ipv6TClass)
stateSinkObject.Save(17, &e.info)
stateSinkObject.Save(18, &e.state)
stateSinkObject.Save(15, &e.ipv6MulticastNICID)
stateSinkObject.Save(16, &e.ipv4TOS)
stateSinkObject.Save(17, &e.ipv6TClass)
stateSinkObject.Save(18, &e.pmtud)
stateSinkObject.Save(19, &e.info)
stateSinkObject.Save(20, &e.state)
}
func (e *Endpoint) afterLoad(context.Context) {}
@ -81,10 +85,12 @@ func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source
stateSourceObject.Load(12, &e.multicastTTL)
stateSourceObject.Load(13, &e.multicastAddr)
stateSourceObject.Load(14, &e.multicastNICID)
stateSourceObject.Load(15, &e.ipv4TOS)
stateSourceObject.Load(16, &e.ipv6TClass)
stateSourceObject.Load(17, &e.info)
stateSourceObject.Load(18, &e.state)
stateSourceObject.Load(15, &e.ipv6MulticastNICID)
stateSourceObject.Load(16, &e.ipv4TOS)
stateSourceObject.Load(17, &e.ipv6TClass)
stateSourceObject.Load(18, &e.pmtud)
stateSourceObject.Load(19, &e.info)
stateSourceObject.Load(20, &e.state)
}
func (m *multicastMembership) StateTypeName() string {

9
pkg/tcpip/transport/packet/endpoint.go
View file

@ -267,11 +267,12 @@ func (ep *endpoint) Write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tc
}
payloadSz := payload.Size()
mark := ep.ops.GetMark()
if err := func() tcpip.Error {
if ep.cooked {
return ep.stack.WritePacketToRemote(nicID, remote, proto, payload)
return ep.stack.WritePacketToRemoteWithMark(nicID, remote, proto, payload, mark)
}
return ep.stack.WriteRawPacket(nicID, proto, payload)
return ep.stack.WriteRawPacketWithMark(nicID, proto, payload, mark)
}(); err != nil {
return 0, err
}
@ -456,14 +457,14 @@ func (ep *endpoint) UpdateLastError(err tcpip.Error) {
// GetSockOpt implements tcpip.Endpoint.GetSockOpt.
func (ep *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) tcpip.Error {
switch opt.(type) {
switch opt := opt.(type) {
case *tcpip.TpacketStats:
ep.packetMmapMu.RLock()
defer ep.packetMmapMu.RUnlock()
if ep.packetMMapEp == nil {
return nil
}
*(opt.(*tcpip.TpacketStats)) = ep.packetMMapEp.Stats()
*opt = ep.packetMMapEp.Stats()
return nil
default:
return &tcpip.ErrUnknownProtocolOption{}

2
pkg/tcpip/transport/packet/endpoint_mutex.go
View file

@ -92,5 +92,5 @@ func endpointinitLockNames() {}
func init() {
endpointinitLockNames()
endpointprefixIndex = locking.NewMutexClass(reflect.TypeOf(endpointRWMutex{}), endpointlockNames)
endpointprefixIndex = locking.NewMutexClass(reflect.TypeFor[endpointRWMutex](), endpointlockNames)
}

2
pkg/tcpip/transport/packet/endpoint_rcv_mutex.go
View file

@ -60,5 +60,5 @@ func rcvinitLockNames() {}
func init() {
rcvinitLockNames()
rcvprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvMutex{}), rcvlockNames)
rcvprefixIndex = locking.NewMutexClass(reflect.TypeFor[rcvMutex](), rcvlockNames)
}

10
pkg/tcpip/transport/packet/endpoint_state.go
View file

@ -16,9 +16,9 @@ package packet
import (
"context"
"fmt"
"time"
"github.com/sagernet/gvisor/pkg/log"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
)
@ -43,11 +43,6 @@ func (ep *endpoint) beforeSave() {
// afterLoad is invoked by stateify.
func (ep *endpoint) afterLoad(ctx context.Context) {
if !ep.stack.IsSaveRestoreEnabled() {
ep.mu.Lock()
ep.stack = stack.RestoreStackFromContext(ctx)
ep.mu.Unlock()
}
ep.stack.RegisterRestoredEndpoint(ep)
}
@ -58,7 +53,8 @@ func (ep *endpoint) Restore(_ *stack.Stack) {
ep.ops.InitHandler(ep, ep.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
if err := ep.stack.RegisterPacketEndpoint(ep.boundNIC, ep.boundNetProto, ep); err != nil {
panic(fmt.Sprintf("RegisterPacketEndpoint(%d, %d, _): %s", ep.boundNIC, ep.boundNetProto, err))
log.Warningf("RegisterPacketEndpoint(%d, %d, _) failed during restore with error: %s", ep.boundNIC, ep.boundNetProto, err)
return
}
ep.rcvMu.Lock()

2
pkg/tcpip/transport/packet/last_error_mutex.go
View file

@ -60,5 +60,5 @@ func lastErrorinitLockNames() {}
func init() {
lastErrorinitLockNames()
lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeOf(lastErrorMutex{}), lastErrorlockNames)
lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeFor[lastErrorMutex](), lastErrorlockNames)
}

2
pkg/tcpip/transport/packet/packet_mmap_mutex.go
View file

@ -92,5 +92,5 @@ func packetMmapinitLockNames() {}
func init() {
packetMmapinitLockNames()
packetMmapprefixIndex = locking.NewMutexClass(reflect.TypeOf(packetMmapRWMutex{}), packetMmaplockNames)
packetMmapprefixIndex = locking.NewMutexClass(reflect.TypeFor[packetMmapRWMutex](), packetMmaplockNames)
}

4
pkg/tcpip/transport/packet/packet_state_autogen.go
View file

@ -27,8 +27,8 @@ func (p *packet) beforeSave() {}
// +checklocksignore
func (p *packet) StateSave(stateSinkObject state.Sink) {
p.beforeSave()
var receivedAtValue int64
receivedAtValue = p.saveReceivedAt()
receivedAtValue := p.saveReceivedAt()
_ = (int64)(receivedAtValue)
stateSinkObject.SaveValue(2, receivedAtValue)
stateSinkObject.Save(0, &p.packetEntry)
stateSinkObject.Save(1, &p.data)

19
pkg/tcpip/transport/raw/endpoint_state.go
View file

@ -35,11 +35,7 @@ func (p *rawPacket) loadReceivedAt(_ context.Context, nsec int64) {
// afterLoad is invoked by stateify.
func (e *endpoint) afterLoad(ctx context.Context) {
if e.stack.IsSaveRestoreEnabled() {
e.stack.RegisterRestoredEndpoint(e)
} else {
stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e)
}
e.stack.RegisterRestoredEndpoint(e)
}
// beforeSave is invoked by stateify.
@ -56,20 +52,7 @@ func (e *endpoint) Restore(s *stack.Stack) {
return
}
e.setReceiveDisabled(false)
if e.stack.IsSaveRestoreEnabled() {
e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
return
}
e.stack = s
e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
if e.associated {
netProto := e.net.NetProto()
if err := e.stack.RegisterRawTransportEndpoint(netProto, e.transProto, e); err != nil {
panic("RegisterRawTransportEndpoint failed during restore")
}
}
}
// Resume implements tcpip.ResumableEndpoint.Resume.

4
pkg/tcpip/transport/raw/raw_state_autogen.go
View file

@ -29,8 +29,8 @@ func (p *rawPacket) beforeSave() {}
// +checklocksignore
func (p *rawPacket) StateSave(stateSinkObject state.Sink) {
p.beforeSave()
var receivedAtValue int64
receivedAtValue = p.saveReceivedAt()
receivedAtValue := p.saveReceivedAt()
_ = (int64)(receivedAtValue)
stateSinkObject.SaveValue(2, receivedAtValue)
stateSinkObject.Save(0, &p.rawPacketEntry)
stateSinkObject.Save(1, &p.data)

4
pkg/tcpip/transport/tcp/accept.go
View file

@ -255,7 +255,7 @@ func (l *listenContext) startHandshake(s *segment, opts header.TCPSynOptions, qu
// Propagate any inheritable options from the listening endpoint
// to the newly created endpoint.
l.listenEP.propagateInheritableOptionsLocked(ep) // +checklocksforce
l.listenEP.propagateInheritableOptionsLocked(ep) // +checklocksforce:ep.mu
if !ep.reserveTupleLocked() {
ep.mu.Unlock()
@ -359,6 +359,7 @@ func (e *Endpoint) propagateInheritableOptionsLocked(n *Endpoint) {
n.boundBindToDevice = e.boundBindToDevice
n.boundPortFlags = e.boundPortFlags
n.userMSS = e.userMSS
n.ops.SetMark(e.ops.GetMark())
}
// reserveTupleLocked reserves an accepted endpoint's tuple.
@ -529,6 +530,7 @@ func (e *Endpoint) handleListenSegment(ctx *listenContext, s *segment) tcpip.Err
seq: cookie,
ack: s.sequenceNumber + 1,
rcvWnd: ctx.rcvWnd,
df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo || e.pmtud == tcpip.PMTUDiscoveryProbe,
expOptVal: e.getExperimentOptionValue(route),
}
if err := e.sendSynTCP(route, fields, synOpts); err != nil {

2
pkg/tcpip/transport/tcp/accept_mutex.go
View file

@ -60,5 +60,5 @@ func acceptinitLockNames() {}
func init() {
acceptinitLockNames()
acceptprefixIndex = locking.NewMutexClass(reflect.TypeOf(acceptMutex{}), acceptlockNames)
acceptprefixIndex = locking.NewMutexClass(reflect.TypeFor[acceptMutex](), acceptlockNames)
}

176
pkg/tcpip/transport/tcp/connect.go
View file

@ -371,6 +371,7 @@ func (h *handshake) synSentState(s *segment) tcpip.Error {
seq: h.iss,
ack: h.ackNum,
rcvWnd: h.rcvWnd,
df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe,
expOptVal: h.ep.getExperimentOptionValue(h.ep.route),
}, synOpts)
return nil
@ -458,6 +459,7 @@ func (h *handshake) synRcvdState(s *segment) tcpip.Error {
seq: h.iss,
ack: h.ackNum,
rcvWnd: h.rcvWnd,
df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe,
expOptVal: h.ep.getExperimentOptionValue(h.ep.route),
}, synOpts)
return nil
@ -555,6 +557,7 @@ func (h *handshake) processSegments() tcpip.Error {
// start sends the first SYN/SYN-ACK. It does not block, even if link address
// resolution is required.
// +checklocks:h.ep.mu
func (h *handshake) start() {
h.startTime = h.ep.stack.Clock().NowMonotonic()
h.ep.amss = calculateAdvertisedMSS(h.ep.userMSS, h.ep.route)
@ -595,6 +598,7 @@ func (h *handshake) start() {
seq: h.iss,
ack: h.ackNum,
rcvWnd: h.rcvWnd,
df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe,
expOptVal: h.ep.getExperimentOptionValue(h.ep.route),
}, synOpts)
}
@ -632,6 +636,7 @@ func (h *handshake) retransmitHandlerLocked() tcpip.Error {
seq: h.iss,
ack: h.ackNum,
rcvWnd: h.rcvWnd,
df: h.ep.pmtud == tcpip.PMTUDiscoveryWant || h.ep.pmtud == tcpip.PMTUDiscoveryDo || h.ep.pmtud == tcpip.PMTUDiscoveryProbe,
expOptVal: e.getExperimentOptionValue(e.route),
}, h.sendSYNOpts)
// If we have ever retransmitted the SYN-ACK or
@ -642,11 +647,10 @@ func (h *handshake) retransmitHandlerLocked() tcpip.Error {
return nil
}
// transitionToStateEstablisedLocked transitions the endpoint of the handshake
// transitionToStateEstablishedLocked transitions the endpoint of the handshake
// to an established state given the last segment received from peer. It also
// initializes sender/receiver.
// +checklocks:h.ep.mu
// +checklocksalias:h.ep.snd.ep.mu=h.ep.mu
func (h *handshake) transitionToStateEstablishedLocked(s *segment) {
// Stop the SYN retransmissions now that handshake is complete.
if h.retransmitTimer != nil {
@ -656,16 +660,20 @@ func (h *handshake) transitionToStateEstablishedLocked(s *segment) {
// Transfer handshake state to TCP connection. We disable
// receive window scaling if the peer doesn't support it
// (indicated by a negative send window scale).
h.ep.snd = newSender(h.ep, h.iss, h.ackNum-1, h.sndWnd, h.mss, h.sndWndScale)
initSender(h.ep, h.iss, h.ackNum-1, h.sndWnd, h.mss, h.sndWndScale)
now := h.ep.stack.Clock().NowMonotonic()
// Use the final handshake ACK's ingress time (s.rcvdTime) rather than the
// current clock to seed the initial RTT/RTO. If the ACK was delayed inside
// the stack before processing, the processing-time clock would inflate the
// initial RTO, which then persists for several RTTs.
rcvd := s.rcvdTime
var rtt time.Duration
if h.ep.SendTSOk && s.parsedOptions.TSEcr != 0 {
rtt = h.ep.elapsed(now, s.parsedOptions.TSEcr)
rtt = h.ep.elapsed(rcvd, s.parsedOptions.TSEcr)
}
if !h.sampleRTTWithTSOnly && rtt == 0 {
rtt = now.Sub(h.startTime)
rtt = rcvd.Sub(h.startTime)
}
if rtt > 0 {
@ -824,7 +832,10 @@ func (e *Endpoint) sendSynTCP(r *stack.Route, tf tcpFields, opts header.TCPSynOp
if r.NetProto() == header.IPv6ProtocolNumber && tf.expOptVal != 0 {
hdrSize += header.IPv6ExperimentHdrLength
}
p := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize})
p := stack.NewPacketBuffer(stack.PacketBufferOptions{
ReserveHeaderBytes: hdrSize,
Mark: e.ops.GetMark(),
})
defer p.DecRef()
if err := e.sendTCP(r, tf, p, stack.GSO{}); err != nil {
e.stats.SendErrors.SynSendToNetworkFailed.Increment()
@ -900,7 +911,10 @@ func sendTCPBatch(r *stack.Route, tf tcpFields, pkt *stack.PacketBuffer, gso sta
// Reserve extra bytes for the experiment option.
hdrSize += header.IPv6ExperimentHdrLength
}
splitPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize})
splitPkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
ReserveHeaderBytes: hdrSize,
Mark: pkt.Mark,
})
splitPkt.Data().ReadFromPacketData(pkt.Data(), packetSize)
pkt = splitPkt
}
@ -1007,9 +1021,10 @@ func (e *Endpoint) makeOptions(sackBlocks []header.SACKBlock) []byte {
// sendEmptyRaw sends a TCP segment with no payload to the endpoint's peer.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) sendEmptyRaw(flags header.TCPFlags, seq, ack seqnum.Value, rcvWnd seqnum.Size) tcpip.Error {
pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{})
pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
Mark: e.ops.GetMark(),
})
defer pkt.DecRef()
return e.sendRaw(pkt, flags, seq, ack, rcvWnd)
}
@ -1018,7 +1033,6 @@ func (e *Endpoint) sendEmptyRaw(flags header.TCPFlags, seq, ack seqnum.Value, rc
// ownership of pkt. pkt must not have any headers set.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) sendRaw(pkt *stack.PacketBuffer, flags header.TCPFlags, seq, ack seqnum.Value, rcvWnd seqnum.Size) tcpip.Error {
var sackBlocks []header.SACKBlock
if e.EndpointState() == StateEstablished && e.rcv.pendingRcvdSegments.Len() > 0 && (flags&header.TCPFlagAck != 0) {
@ -1033,21 +1047,21 @@ func (e *Endpoint) sendRaw(pkt *stack.PacketBuffer, flags header.TCPFlags, seq,
}
pkt.ReserveHeaderBytes(hdrSize)
return e.sendTCP(e.route, tcpFields{
id: e.TransportEndpointInfo.ID,
ttl: calculateTTL(e.route, e.ipv4TTL, e.ipv6HopLimit),
tos: e.sendTOS,
flags: flags,
seq: seq,
ack: ack,
rcvWnd: rcvWnd,
opts: options,
df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo,
id: e.TransportEndpointInfo.ID,
ttl: calculateTTL(e.route, e.ipv4TTL, e.ipv6HopLimit),
tos: e.sendTOS,
flags: flags,
seq: seq,
ack: ack,
rcvWnd: rcvWnd,
opts: options,
// PROBE sets DF like DO; see network/endpoint.go for details.
df: e.pmtud == tcpip.PMTUDiscoveryWant || e.pmtud == tcpip.PMTUDiscoveryDo || e.pmtud == tcpip.PMTUDiscoveryProbe,
expOptVal: expOptVal,
}, pkt, e.gso)
}
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) sendData(next *segment) {
// Initialize the next segment to write if it's currently nil.
if e.snd.writeNext == nil {
@ -1065,7 +1079,6 @@ func (e *Endpoint) sendData(next *segment) {
// error code and sends a RST if and only if the error is not ErrConnectionReset
// indicating that the connection is being reset due to receiving a RST.
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) resetConnectionLocked(err tcpip.Error) {
// Only send a reset if the connection is being aborted for a reason
// other than receiving a reset.
@ -1080,12 +1093,26 @@ func (e *Endpoint) resetConnectionLocked(err tcpip.Error) {
//
// See: https://www.snellman.net/blog/archive/2016-02-01-tcp-rst/ for more
// information.
sndWndEnd := e.snd.SndUna.Add(e.snd.SndWnd)
resetSeqNum := sndWndEnd
if !sndWndEnd.LessThan(e.snd.SndNxt) || e.snd.SndNxt.Size(sndWndEnd) < (1<<e.snd.SndWndScale) {
resetSeqNum = e.snd.SndNxt
//
// e.snd and e.rcv may be nil if the endpoint is in a handshake
// state (e.g. SynSent) where the sender and receiver have not
// yet been initialized. Per Linux behavior, use a sequence
// number of zero when no ACK has been received (snd is nil),
// and a receive window of zero when rcv is nil since the
// connection will be immediately terminated.
var resetSeqNum seqnum.Value
var ackNum seqnum.Value
if e.snd != nil {
sndWndEnd := e.snd.SndUna.Add(e.snd.SndWnd)
resetSeqNum = sndWndEnd
if !sndWndEnd.LessThan(e.snd.SndNxt) || e.snd.SndNxt.Size(sndWndEnd) < (1<<e.snd.SndWndScale) {
resetSeqNum = e.snd.SndNxt
}
}
e.sendEmptyRaw(header.TCPFlagAck|header.TCPFlagRst, resetSeqNum, e.rcv.RcvNxt, 0)
if e.rcv != nil {
ackNum = e.rcv.RcvNxt
}
e.sendEmptyRaw(header.TCPFlagAck|header.TCPFlagRst, resetSeqNum, ackNum, 0)
}
// Don't purge read queues here. If there's buffered data, it's still allowed
// to be read.
@ -1162,52 +1189,66 @@ func (e *Endpoint) drainClosingSegmentQueue() {
}
}
// handleReset processes an inbound segment carrying the RST flag.
//
// Acceptance follows RFC 5961 section 3.2:
// - If the segment sequence number is out of window, the segment is
// silently dropped.
// - If the segment sequence number is in window but not exactly equal
// to RCV.NXT, the implementation sends a challenge ACK and drops
// the segment.
// - Only an exact match against RCV.NXT causes the connection to be
// reset.
//
// This is stricter than RFC 793 page 37, which accepted any in-window RST.
// The strict-match rule defends against off-path blind RST injection.
// Linux has implemented it since version 3.6 (2012); see
// net/ipv4/tcp_input.c tcp_validate_incoming().
//
// +checklocks:e.mu
func (e *Endpoint) handleReset(s *segment) (ok bool, err tcpip.Error) {
if e.rcv.acceptable(s.sequenceNumber, 0) {
// RFC 793, page 37 states that "in all states
// except SYN-SENT, all reset (RST) segments are
// validated by checking their SEQ-fields." So
// we only process it if it's acceptable.
switch e.EndpointState() {
// In case of a RST in CLOSE-WAIT linux moves
// the socket to closed state with an error set
// to indicate EPIPE.
//
// Technically this seems to be at odds w/ RFC.
// As per https://tools.ietf.org/html/rfc793#section-2.7
// page 69 the behavior for a segment arriving
// w/ RST bit set in CLOSE-WAIT is inlined below.
//
// ESTABLISHED
// FIN-WAIT-1
// FIN-WAIT-2
// CLOSE-WAIT
// If the RST bit is set then, any outstanding RECEIVEs and
// SEND should receive "reset" responses. All segment queues
// should be flushed. Users should also receive an unsolicited
// general "connection reset" signal. Enter the CLOSED state,
// delete the TCB, and return.
case StateCloseWait:
e.transitionToStateCloseLocked()
e.hardError = &tcpip.ErrAborted{}
return false, nil
default:
// RFC 793, page 37 states that "in all states
// except SYN-SENT, all reset (RST) segments are
// validated by checking their SEQ-fields." So
// we only process it if it's acceptable.
return false, &tcpip.ErrConnectionReset{}
}
if !e.rcv.acceptable(s.sequenceNumber, 0) {
// Out of window. Silent drop.
return true, nil
}
if s.sequenceNumber != e.rcv.RcvNxt {
// In window but not an exact match. Send a challenge ACK and drop the
// segment per RFC 5961 section 3.2. The challenge ACK helper rate-limits
// challenge transmission per RFC 5961 section 7.
e.snd.maybeSendOutOfWindowAck(s)
return true, nil
}
switch e.EndpointState() {
// In case of a RST in CLOSE-WAIT linux moves the socket to closed state
// with an error set to indicate EPIPE.
//
// As per https://tools.ietf.org/html/rfc793#section-2.7 page 69 the
// behavior for a segment arriving w/ RST bit set in CLOSE-WAIT is
// inlined below.
//
// ESTABLISHED
// FIN-WAIT-1
// FIN-WAIT-2
// CLOSE-WAIT
//
// If the RST bit is set then, any outstanding RECEIVEs and SEND should
// receive "reset" responses. All segment queues should be flushed.
// Users should also receive an unsolicited general "connection reset"
// signal. Enter the CLOSED state, delete the TCB, and return.
case StateCloseWait:
e.transitionToStateCloseLocked()
e.hardError = &tcpip.ErrAborted{}
return false, nil
default:
return false, &tcpip.ErrConnectionReset{}
}
return true, nil
}
// handleSegments processes all inbound segments.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) handleSegmentsLocked() tcpip.Error {
sndUna := e.snd.SndUna
for i := 0; i < maxSegmentsPerWake; i++ {
@ -1262,8 +1303,6 @@ func (e *Endpoint) probeSegmentLocked() {
// if the connection should be terminated.
//
// +checklocks:e.mu
// +checklocksalias:e.rcv.ep.mu=e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) handleSegmentLocked(s *segment) (cont bool, err tcpip.Error) {
// Invoke the tcp probe if installed. The tcp probe function will update
// the TCPEndpointState after the segment is processed.
@ -1337,7 +1376,6 @@ func (e *Endpoint) handleSegmentLocked(s *segment) (cont bool, err tcpip.Error)
// keepalive packets periodically when the connection is idle. If we don't hear
// from the other side after a number of tries, we terminate the connection.
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) keepaliveTimerExpired() tcpip.Error {
userTimeout := e.userTimeout
@ -1379,7 +1417,6 @@ func (e *Endpoint) keepaliveTimerExpired() tcpip.Error {
// whether it is enabled for this endpoint.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) resetKeepaliveTimer(receivedData bool) {
e.keepalive.Lock()
defer e.keepalive.Unlock()
@ -1442,7 +1479,6 @@ func (e *Endpoint) handshakeFailed(err tcpip.Error) {
// handleTimeWaitSegments processes segments received during TIME_WAIT
// state.
// +checklocks:e.mu
// +checklocksalias:e.rcv.ep.mu=e.mu
func (e *Endpoint) handleTimeWaitSegments() (extendTimeWait bool, reuseTW func()) {
for i := 0; i < maxSegmentsPerWake; i++ {
s := e.segmentQueue.dequeue()

3
pkg/tcpip/transport/tcp/connect_unsafe.go
View file

@ -15,7 +15,6 @@
package tcp
import (
"reflect"
"unsafe"
)
@ -26,5 +25,5 @@ import (
func optionsToArray(options []byte) *[maxOptionSize]byte {
// Reslice to full capacity.
options = options[0:maxOptionSize]
return (*[maxOptionSize]byte)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&options)).Data))
return (*[maxOptionSize]byte)(unsafe.Pointer(&options[0]))
}

26
pkg/tcpip/transport/tcp/cubic.go
View file

@ -79,9 +79,14 @@ func newCubicCC(s *sender) *cubicState {
C: 0.4,
// By this point, the sender has initialized it's initial sequence
// number.
EndSeq: s.SndNxt,
LastRTT: effectivelyInfinity,
CurrRTT: effectivelyInfinity,
EndSeq: s.SndNxt,
LastRTT: effectivelyInfinity,
CurrRTT: effectivelyInfinity,
// LastAck/RoundStart are seeded here from processing time, but the
// HyStart ACK-train comparator that reads them is gated on
// LastRTT < effectivelyInfinity, which only becomes true after the
// first beginHyStartRound re-seeds both from an ACK's ingress time.
// So this initial processing-time seed is never compared.
LastAck: now,
RoundStart: now,
},
@ -122,12 +127,19 @@ func (c *cubicState) enterCongestionAvoidance() {
// here.
//
// +checklocks:c.s.ep.mu
func (c *cubicState) updateHyStart(rtt time.Duration) {
func (c *cubicState) updateHyStart(rtt time.Duration, ackTime tcpip.MonotonicTime) {
if rtt < 0 {
// negative indicates unknown
return
}
now := c.s.ep.stack.Clock().NowMonotonic()
// Use the ACK's ingress time (ackTime) rather than the current clock for
// HyStart's ACK-train timing. The ACK-train detector compares inter-ACK
// spacing against ackDelta (2ms) and the round duration against LastRTT/2;
// if ACKs are delayed and processed in a burst inside the stack (e.g.
// queued while the application held the endpoint lock during a Write), the
// processing clock would make distinct ACKs appear to arrive together,
// distorting both comparisons and potentially exiting slow start early.
now := ackTime
if c.EndSeq.LessThan(c.s.SndUna) {
c.beginHyStartRound(now)
}
@ -197,9 +209,9 @@ func (c *cubicState) updateSlowStart(packetsAcked int) int {
// Refer: https://tools.ietf.org/html/rfc8312#section-4
//
// +checklocks:c.s.ep.mu
func (c *cubicState) Update(packetsAcked int, rtt time.Duration) {
func (c *cubicState) Update(packetsAcked int, rtt time.Duration, ackTime tcpip.MonotonicTime) {
if c.s.Ssthresh == InitialSsthresh && c.s.SndCwnd < c.s.Ssthresh {
c.updateHyStart(rtt)
c.updateHyStart(rtt, ackTime)
}
if c.s.SndCwnd < c.s.Ssthresh {
packetsAcked = c.updateSlowStart(packetsAcked)

17
pkg/tcpip/transport/tcp/dispatcher.go
View file

@ -18,7 +18,9 @@ import (
"encoding/binary"
"fmt"
"math/rand"
"time"
"github.com/sagernet/gvisor/pkg/log"
"github.com/sagernet/gvisor/pkg/sleep"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/tcpip"
@ -26,6 +28,7 @@ import (
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/waiter"
"golang.org/x/time/rate"
)
// epQueue is a queue of endpoints.
@ -33,7 +36,7 @@ import (
// +stateify savable
type epQueue struct {
mu epQueueMutex `state:"nosave"`
list endpointList
list endpointList `state:"nosave"`
}
// enqueue adds e to the queue if the endpoint is not already on the queue.
@ -161,7 +164,7 @@ func handleConnecting(ep *Endpoint) {
return
}
// lx:end handshake-nil-guard
if err := ep.h.processSegments(); err != nil { // +checklocksforce:ep.h.ep.mu
if err := ep.h.processSegments(); err != nil {
// handshake failed. clean up the tcp endpoint and handshake
// state.
if lEP := ep.h.listenEP; lEP != nil {
@ -265,6 +268,8 @@ func handleTimeWait(ep *Endpoint) {
ep.mu.Unlock()
}
var warnRateLimiter = rate.NewLimiter(rate.Every(time.Second), 1)
// handleListen is responsible for TCP processing for an endpoint in LISTEN
// state.
func handleListen(ep *Endpoint) {
@ -286,9 +291,11 @@ func handleListen(ep *Endpoint) {
break
}
// TODO(gvisor.dev/issue/4690): Better handle errors instead of
// silently dropping.
_ = ep.handleListenSegment(ep.listenCtx, s)
if err := ep.handleListenSegment(ep.listenCtx, s); err != nil {
if warnRateLimiter.Allow() {
log.Warningf("tcp.Endpoint.handleListenSegment() failed for packet [nic=%d, source=%s, dest=%s, protocol=%d]: %v", s.pkt.NICID, s.pkt.Network().SourceAddress(), s.pkt.Network().DestinationAddress(), s.pkt.NetworkProtocolNumber, err)
}
}
s.DecRef()
}
}

2
pkg/tcpip/transport/tcp/dispatcher_mutex.go
View file

@ -60,5 +60,5 @@ func dispatcherinitLockNames() {}
func init() {
dispatcherinitLockNames()
dispatcherprefixIndex = locking.NewMutexClass(reflect.TypeOf(dispatcherMutex{}), dispatcherlockNames)
dispatcherprefixIndex = locking.NewMutexClass(reflect.TypeFor[dispatcherMutex](), dispatcherlockNames)
}

83
pkg/tcpip/transport/tcp/endpoint.go
View file

@ -341,6 +341,9 @@ func (sq *sndQueueInfo) CloneState(other *TCPSndBufState) {
// TODO(b/339664055): Checklocks should be used more extensively here. Coverage
// is currently sparse.
//
// +checklocksalias:snd.ep.mu=mu
// +checklocksalias:rcv.ep.mu=mu
// +checklocksalias:h.ep.mu=mu
// +stateify savable
type Endpoint struct {
TCPEndpointStateInner
@ -607,6 +610,16 @@ type Endpoint struct {
//
// +checklocks:mu
alsoBindToV4 bool
// terminateAtRestore indicates whether the endpoint must be terminated
// upon restore. This applies specifically when the snapshots are taken
// with the "save-resume" flag, ensuring that if such a snapshot is
// restored later, this endpoint is cleaned up. This flag is only set
// to true in beforeSave for external endpoints which do not have
// save-restore capability.
//
// +checklocks:mu
terminateAtRestore bool
}
// calculateAdvertisedMSS calculates the MSS to advertise.
@ -641,7 +654,6 @@ func (e *Endpoint) isOwnedByUser() bool {
// should not be holding the lock for long and spinning reduces latency as we
// avoid an expensive sleep/wakeup of the syscall goroutine).
// +checklocksacquire:e.mu
// +checklocksacquire:e.snd.ep.mu
func (e *Endpoint) LockUser() {
const iterations = 5
for i := 0; i < iterations; i++ {
@ -654,14 +666,14 @@ func (e *Endpoint) LockUser() {
if e.ownedByUser.Load() == 1 {
e.mu.Lock()
e.ownedByUser.Store(1)
return // +checklocksforce: this locks e.snd.ep.mu
return
}
// Spin but don't yield the processor since the lower half
// should yield the lock soon.
continue
}
e.ownedByUser.Store(1)
return // +checklocksforce: this locks e.snd.ep.mu
return
}
for i := 0; i < iterations; i++ {
@ -674,7 +686,7 @@ func (e *Endpoint) LockUser() {
if e.ownedByUser.Load() == 1 {
e.mu.Lock()
e.ownedByUser.Store(1)
return // +checklocksforce: this locks e.snd.ep.mu
return
}
// Spin but yield the processor since the lower half
// should yield the lock soon.
@ -682,7 +694,7 @@ func (e *Endpoint) LockUser() {
continue
}
e.ownedByUser.Store(1)
return // +checklocksforce: this locks e.snd.ep.mu
return
}
// Finally just give up and wait for the Lock.
@ -725,7 +737,6 @@ func (e *Endpoint) UnlockUser() {
// processor goroutine starts running before we release the lock here
// then it will fail to process as TryLock() will fail.
processor.queueEndpoint(e)
return
}
// StopWork halts packet processing. Only to be used in tests.
@ -760,10 +771,7 @@ func (e *Endpoint) AssertLockHeld(locked *Endpoint) {
// TODO(b/226403629): Remove this once checklocks understands TryLock.
// +checklocksacquire:e.mu
func (e *Endpoint) TryLock() bool {
if e.mu.TryLock() {
return true // +checklocksforce
}
return false // +checklocksignore
return e.mu.TryLock() // +checklocksforce: TryLock.
}
// setEndpointState updates the state of the endpoint to state atomically. This
@ -1015,7 +1023,6 @@ func (e *Endpoint) purgeReadQueue() {
}
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) purgeWriteQueue() {
if e.snd != nil {
e.sndQueueInfo.sndQueueMu.Lock()
@ -1358,7 +1365,7 @@ func (e *Endpoint) ModerateRecvBuf(copied int) {
// Send the update after unlocking rcvQueueMu as sending a segment acquires
// the lock to calculate the window to be sent.
if e.EndpointState().connected() && sendNonZeroWindowUpdate {
e.rcv.nonZeroWindow() // +checklocksforce:e.rcv.ep.mu
e.rcv.nonZeroWindow()
}
}
@ -1465,7 +1472,7 @@ func (e *Endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult
e.rcvQueueMu.Unlock()
if e.EndpointState().connected() && sendNonZeroWindowUpdate {
e.rcv.nonZeroWindow() // +checklocksforce:e.rcv.ep.mu
e.rcv.nonZeroWindow()
}
}
@ -1601,7 +1608,6 @@ func (e *Endpoint) readFromPayloader(p tcpip.Payloader, opts tcpip.WriteOptions,
// queueSegment reads data from the payloader and returns a segment to be sent.
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) queueSegment(p tcpip.Payloader, opts tcpip.WriteOptions) (*segment, int, tcpip.Error) {
e.sndQueueInfo.sndQueueMu.Lock()
defer e.sndQueueInfo.sndQueueMu.Unlock()
@ -1642,7 +1648,7 @@ func (e *Endpoint) queueSegment(p tcpip.Payloader, opts tcpip.WriteOptions) (*se
// Add data to the send queue.
size := int(buf.Size())
s := newOutgoingSegment(e.TransportEndpointInfo.ID, e.stack.Clock(), buf)
s := newOutgoingSegment(e.TransportEndpointInfo.ID, e.stack.Clock(), buf, e.ops.GetMark())
e.sndQueueInfo.SndBufUsed += size
e.snd.writeList.PushBack(s)
@ -1828,7 +1834,7 @@ func (e *Endpoint) OnSetReceiveBufferSize(rcvBufSz, oldSz int64) (newSz int64, p
e.LockUser()
defer e.UnlockUser()
if e.EndpointState().connected() && sendNonZeroWindowUpdate {
e.rcv.nonZeroWindow() // +checklocksforce:e.rcv.ep.mu
e.rcv.nonZeroWindow()
}
}
e.UnlockUser()
@ -1894,15 +1900,15 @@ func (e *Endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) tcpip.Error {
e.UnlockUser()
case tcpip.MTUDiscoverOption:
// PROBE is accepted alongside DO/WANT/DONT. In Linux,
// PROBE sets DF but ignores ICMP-based PMTU updates;
// since gVisor lacks ICMP PMTU feedback, it behaves
// identically to DO.
switch v := tcpip.PMTUDStrategy(v); v {
case tcpip.PMTUDiscoveryWant, tcpip.PMTUDiscoveryDont, tcpip.PMTUDiscoveryDo:
case tcpip.PMTUDiscoveryWant, tcpip.PMTUDiscoveryDont, tcpip.PMTUDiscoveryDo, tcpip.PMTUDiscoveryProbe:
e.LockUser()
e.pmtud = v
e.UnlockUser()
case tcpip.PMTUDiscoveryProbe:
// We don't support a way to ignore MTU updates; it's
// either on or it's off.
return &tcpip.ErrNotSupported{}
default:
return &tcpip.ErrNotSupported{}
}
@ -2391,7 +2397,6 @@ func (e *Endpoint) registerEndpoint(addr tcpip.FullAddress, netProto tcpip.Netwo
// connect connects the endpoint to its peer.
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) connect(addr tcpip.FullAddress, handshake bool) tcpip.Error {
connectingAddr := addr.Addr
@ -2540,7 +2545,6 @@ func (e *Endpoint) Shutdown(flags tcpip.ShutdownFlags) tcpip.Error {
}
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) shutdownLocked(flags tcpip.ShutdownFlags) tcpip.Error {
e.shutdownFlags |= flags
switch {
@ -2581,9 +2585,18 @@ func (e *Endpoint) shutdownLocked(flags tcpip.ShutdownFlags) tcpip.Error {
return nil
}
// Queue fin segment.
s := newOutgoingSegment(e.TransportEndpointInfo.ID, e.stack.Clock(), buffer.Buffer{})
// Queue FIN and transition to the closing state immediately,
// matching Linux tcp_close_state(): the FIN may be queued but
// not yet transmitted when the write queue is blocked.
s := newOutgoingSegment(e.TransportEndpointInfo.ID, e.stack.Clock(), buffer.Buffer{}, e.ops.GetMark())
e.snd.writeList.PushBack(s)
e.updateConnDirectionState(connDirectionStateSndClosed)
switch e.EndpointState() {
case StateCloseWait:
e.setEndpointState(StateLastAck)
default:
e.setEndpointState(StateFinWait1)
}
// Mark endpoint as closed.
e.sndQueueInfo.SndClosed = true
e.sndQueueInfo.sndQueueMu.Unlock()
@ -2924,7 +2937,8 @@ func (e *Endpoint) onICMPError(err tcpip.Error, transErr stack.TransportError, p
if e.EndpointState().connecting() {
e.mu.Lock()
if lEP := e.h.listenEP; lEP != nil {
if e.h != nil && e.h.listenEP != nil {
lEP := e.h.listenEP
// Remove from listening endpoints pending list.
lEP.acceptMu.Lock()
delete(lEP.acceptQueue.pendingEndpoints, e)
@ -2956,7 +2970,7 @@ func (e *Endpoint) HandleError(transErr stack.TransportError, pkt *stack.PacketB
e.mu.Lock()
defer e.mu.Unlock()
if e.snd != nil {
e.snd.updateMaxPayloadSize(newMTU, 1 /* count */) // +checklocksforce:e.snd.ep.mu
e.snd.updateMaxPayloadSize(newMTU, 1 /* count */)
}
}
}
@ -2986,7 +3000,6 @@ func (e *Endpoint) HandleError(transErr stack.TransportError, pkt *stack.PacketB
// number of newly available bytes is v.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) updateSndBufferUsage(v int) {
sendBufferSize := e.getSendBufferSize()
e.sndQueueInfo.sndQueueMu.Lock()
@ -3168,7 +3181,6 @@ func (e *Endpoint) maxOptionSize() (size int) {
// used before invoking the probe.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) completeStateLocked(s *TCPEndpointState) {
s.TCPEndpointStateInner = e.TCPEndpointStateInner
s.ID = TCPEndpointID(e.TransportEndpointInfo.ID)
@ -3277,11 +3289,7 @@ func GetTCPSendBufferLimits(sh tcpip.StackHandler) tcpip.SendBufferSizeOption {
// This type assertion is safe because only the TCP stack calls this
// function.
ss := sh.(*stack.Stack).TCPSendBufferLimits()
return tcpip.SendBufferSizeOption{
Min: ss.Min,
Default: ss.Default,
Max: ss.Max,
}
return tcpip.SendBufferSizeOption(ss)
}
// allowOutOfWindowAck returns true if an out-of-window ACK can be sent now.
@ -3309,18 +3317,13 @@ func GetTCPReceiveBufferLimits(s tcpip.StackHandler) tcpip.ReceiveBufferSizeOpti
panic(fmt.Sprintf("s.TransportProtocolOption(%d, %#v) = %s", header.TCPProtocolNumber, ss, err))
}
return tcpip.ReceiveBufferSizeOption{
Min: ss.Min,
Default: ss.Default,
Max: ss.Max,
}
return tcpip.ReceiveBufferSizeOption(ss)
}
// computeTCPSendBufferSize implements auto tuning of send buffer size and
// returns the new send buffer size.
//
// +checklocks:e.mu
// +checklocksalias:e.snd.ep.mu=e.mu
func (e *Endpoint) computeTCPSendBufferSize() int64 {
curSndBufSz := int64(e.getSendBufferSize())

182
pkg/tcpip/transport/tcp/endpoint_state.go
View file

@ -23,7 +23,6 @@ import (
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/ports"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
)
@ -33,7 +32,7 @@ var logDisconnectOnce sync.Once
func logDisconnect() {
logDisconnectOnce.Do(func() {
log.Infof("One or more TCP connections terminated during save")
log.Infof("One or more TCP connections terminated during save restore")
})
}
@ -49,17 +48,24 @@ func (e *Endpoint) beforeSave() {
switch {
case epState == StateInitial || epState == StateBound:
case epState.connected() || epState.handshake():
if !e.route.HasSaveRestoreCapability() {
if !e.route.HasDisconnectOkCapability() {
panic(&tcpip.ErrSaveRejection{
Err: fmt.Errorf("endpoint cannot be saved in connected state: local %s:%d, remote %s:%d", e.TransportEndpointInfo.ID.LocalAddress, e.TransportEndpointInfo.ID.LocalPort, e.TransportEndpointInfo.ID.RemoteAddress, e.TransportEndpointInfo.ID.RemotePort),
})
// Terminate valid connections only for restore.
if !e.stack.GetAllowConnectedOnSave() && !e.route.HasSaveRestoreCapability() {
if e.stack.GetRemoveConf() {
// Terminate the endpoint when resume=false.
e.terminateAtRestore = false
if !e.stack.AllowLiveTCPMigration() {
logDisconnect()
e.resetConnectionLocked(&tcpip.ErrConnectionAborted{})
e.mu.Unlock()
e.Close()
e.mu.Lock()
}
} else {
// This is set only when resume=true, the termination
// of this endpoint will happen during restore of the
// saved snapshot.
e.terminateAtRestore = true
}
logDisconnect()
e.resetConnectionLocked(&tcpip.ErrConnectionAborted{})
e.mu.Unlock()
e.Close()
e.mu.Lock()
}
fallthrough
case epState == StateListen:
@ -133,10 +139,41 @@ func (e *Endpoint) afterLoad(ctx context.Context) {
// Restore the endpoint to InitialState as it will be moved to
// its origEndpointState during Restore.
e.state = atomicbitops.FromUint32(uint32(StateInitial))
if e.stack.IsSaveRestoreEnabled() {
e.stack.RegisterRestoredEndpoint(e)
e.stack.RegisterRestoredEndpoint(e)
}
// Close the endpoint during restore if terminateAtRestore was set for the endpoint.
func (e *Endpoint) closeEndpointAtRestore() {
e.mu.Lock()
defer e.mu.Unlock()
epState := EndpointState(e.origEndpointState)
if !epState.connected() && !epState.handshake() {
log.Debugf("endpoint was marked to terminate at restore in a wrong state, ID: %+v state: %v", e.ID, epState)
return
}
if epState.handshake() {
connectedLoading.Wait()
listenLoading.Wait()
}
// Put the endpoint in the error state and do cleanup. Do not
// attempt to send RST as route will be nil.
e.purgeReadQueue()
if epState.connected() {
e.purgeWriteQueue()
e.purgePendingRcvQueue()
e.cleanupLocked()
}
e.state.Store(uint32(StateError))
e.closeNoShutdownLocked()
tcpip.DeleteDanglingEndpoint(e)
if epState.connected() {
connectedLoading.Done()
} else {
stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e)
connectingLoading.Done()
}
}
@ -151,48 +188,52 @@ func (e *Endpoint) Restore(s *stack.Stack) {
snd.probeTimer.init(s.Clock(), timerHandler(e, e.snd.probeTimerExpired))
snd.corkTimer.init(s.Clock(), timerHandler(e, e.snd.corkTimerExpired))
}
saveRestoreEnabled := e.stack.IsSaveRestoreEnabled()
if !saveRestoreEnabled {
e.stack = s
e.protocol = protocolFromStack(s)
}
e.ops.InitHandler(e, e.stack, GetTCPSendBufferLimits, GetTCPReceiveBufferLimits)
e.segmentQueue.thaw()
e.mu.Lock()
id := e.ID
terminateAtRestore := e.terminateAtRestore
e.mu.Unlock()
bind := func() {
e.mu.Lock()
defer e.mu.Unlock()
if !saveRestoreEnabled {
addr, _, err := e.checkV4MappedLocked(tcpip.FullAddress{Addr: e.BindAddr, Port: e.TransportEndpointInfo.ID.LocalPort}, true /* bind */)
if err != nil {
panic("unable to parse BindAddr: " + err.String())
}
portRes := ports.Reservation{
Networks: e.effectiveNetProtos,
Transport: ProtocolNumber,
Addr: addr.Addr,
Port: addr.Port,
Flags: e.boundPortFlags,
BindToDevice: e.boundBindToDevice,
Dest: e.boundDest,
}
if ok := e.stack.ReserveTuple(portRes); !ok {
panic(fmt.Sprintf("unable to re-reserve tuple (%v, %q, %d, %+v, %d, %v)", e.effectiveNetProtos, addr.Addr, addr.Port, e.boundPortFlags, e.boundBindToDevice, e.boundDest))
}
}
e.isPortReserved = true
// Mark endpoint as bound.
e.setEndpointState(StateBound)
}
if terminateAtRestore && !e.stack.AllowLiveTCPMigration() {
e.closeEndpointAtRestore()
return
}
epState := EndpointState(e.origEndpointState)
switch {
case epState.connected():
if e.stack.AllowLiveTCPMigration() {
// Handle dual stack addresses.
netProto := e.NetProto
switch e.TransportEndpointInfo.ID.LocalAddress.BitLen() {
case header.IPv4AddressSizeBits:
netProto = header.IPv4ProtocolNumber
case header.IPv6AddressSizeBits:
netProto = header.IPv6ProtocolNumber
}
// Get the new local NIC for source IP and do a FindRoute here to
// identify if the network config is same. Then only attempt restore,
// else close the connection on our end.
r, err := e.stack.FindRoute(0, e.TransportEndpointInfo.ID.LocalAddress, e.TransportEndpointInfo.ID.RemoteAddress, netProto, false /* multicastLoop */)
if err != nil {
e.closeEndpointAtRestore()
log.Infof("Cannot find the route %+v", e.TransportEndpointInfo.ID)
return
}
e.boundNICID = r.NICID()
r.Release()
}
bind()
if e.connectingAddress.BitLen() == 0 {
e.connectingAddress = e.TransportEndpointInfo.ID.RemoteAddress
@ -210,10 +251,8 @@ func (e *Endpoint) Restore(s *stack.Stack) {
// Reset the scoreboard to reinitialize the sack information as
// we do not restore SACK information.
e.scoreboard.Reset()
if saveRestoreEnabled {
// Unregister the endpoint before registering again during Connect.
e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, header.TCPProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice)
}
// Unregister the endpoint before registering again during Connect.
e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, header.TCPProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice)
e.mu.Lock()
err := e.connect(tcpip.FullAddress{NIC: e.boundNICID, Addr: e.connectingAddress, Port: e.TransportEndpointInfo.ID.RemotePort}, false /* handshake */)
if _, ok := err.(*tcpip.ErrConnectStarted); !ok {
@ -224,6 +263,7 @@ func (e *Endpoint) Restore(s *stack.Stack) {
return
}
e.state.Store(e.origEndpointState)
log.Infof("connect success: %+v", e.TransportEndpointInfo.ID)
// For FIN-WAIT-2 and TIME-WAIT we need to start the appropriate timers so
// that the socket is closed correctly.
switch epState {
@ -239,42 +279,24 @@ func (e *Endpoint) Restore(s *stack.Stack) {
e.snd.corkTimer.enable(MinRTO)
}
e.mu.Unlock()
e.requeueOnRestore()
connectedLoading.Done()
case epState == StateListen:
tcpip.AsyncLoading.Add(1)
if !saveRestoreEnabled {
go func() {
connectedLoading.Wait()
bind()
e.acceptMu.Lock()
backlog := e.acceptQueue.capacity
e.acceptMu.Unlock()
if err := e.Listen(backlog); err != nil {
panic("endpoint listening failed: " + err.String())
}
e.LockUser()
if e.shutdownFlags != 0 {
e.shutdownLocked(e.shutdownFlags)
}
e.UnlockUser()
listenLoading.Done()
tcpip.AsyncLoading.Done()
}()
} else {
go func() {
connectedLoading.Wait()
e.LockUser()
// All endpoints will be moved to initial state after
// restore. Set endpoint to its originial listen state.
e.setEndpointState(StateListen)
// Initialize the listening context.
rcvWnd := seqnum.Size(e.receiveBufferAvailable())
e.listenCtx = newListenContext(e.stack, e.protocol, e, rcvWnd, e.ops.GetV6Only(), e.NetProto)
e.UnlockUser()
listenLoading.Done()
tcpip.AsyncLoading.Done()
}()
}
go func() {
connectedLoading.Wait()
e.LockUser()
// All endpoints will be moved to initial state after
// restore. Set endpoint to its originial listen state.
e.setEndpointState(StateListen)
// Initialize the listening context.
rcvWnd := seqnum.Size(e.receiveBufferAvailable())
e.listenCtx = newListenContext(e.stack, e.protocol, e, rcvWnd, e.ops.GetV6Only(), e.NetProto)
e.UnlockUser()
e.requeueOnRestore()
listenLoading.Done()
tcpip.AsyncLoading.Done()
}()
case epState == StateConnecting:
// Initial SYN hasn't been sent yet so initiate a connect.
tcpip.AsyncLoading.Add(1)
@ -319,6 +341,7 @@ func (e *Endpoint) Restore(s *stack.Stack) {
connectingLoading.Done()
tcpip.AsyncLoading.Done()
e.mu.Unlock()
e.requeueOnRestore()
}()
case epState == StateBound:
tcpip.AsyncLoading.Add(1)
@ -345,3 +368,12 @@ func (e *Endpoint) Restore(s *stack.Stack) {
func (e *Endpoint) Resume() {
e.segmentQueue.thaw()
}
// requeueOnRestore re-adds the endpoint to its processor's run-queue if it has
// queued segments. The run-queue is not saved across checkpoint/restore.
func (e *Endpoint) requeueOnRestore() {
if e.segmentQueue.empty() || e.isOwnedByUser() {
return
}
e.protocol.dispatcher.selectProcessor(e.TransportEndpointInfo.ID).queueEndpoint(e)
}

2
pkg/tcpip/transport/tcp/ep_queue_mutex.go
View file

@ -60,5 +60,5 @@ func epQueueinitLockNames() {}
func init() {
epQueueinitLockNames()
epQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(epQueueMutex{}), epQueuelockNames)
epQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[epQueueMutex](), epQueuelockNames)
}

53
pkg/tcpip/transport/tcp/forwarder.go
View file

@ -15,9 +15,6 @@
package tcp
import (
"fmt"
"github.com/sagernet/gvisor/pkg/buffer"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
@ -179,53 +176,5 @@ func (r *ForwarderRequest) CreateEndpoint(queue *waiter.Queue) (tcpip.Endpoint,
func (r *ForwarderRequest) ForwardedPacketExperimentOption() (uint16, bool) {
r.mu.Lock()
defer r.mu.Unlock()
switch r.segment.pkt.NetworkProtocolNumber {
case header.IPv4ProtocolNumber:
h := header.IPv4(r.segment.pkt.NetworkHeader().Slice())
opts := h.Options()
iter := opts.MakeIterator()
for {
opt, done, err := iter.Next()
if err != nil {
return 0, false
}
if done {
return 0, false
}
if opt.Type() == header.IPv4OptionExperimentType {
return opt.(*header.IPv4OptionExperiment).Value(), true
}
}
case header.IPv6ProtocolNumber:
h := header.IPv6(r.segment.pkt.NetworkHeader().Slice())
v := r.segment.pkt.NetworkHeader().View()
if v != nil {
v.TrimFront(header.IPv6MinimumSize)
}
buf := buffer.MakeWithView(v)
buf.Append(r.segment.pkt.TransportHeader().View())
dataBuf := r.segment.pkt.Data().ToBuffer()
buf.Merge(&dataBuf)
it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(h.NextHeader()), buf)
for {
hdr, done, err := it.Next()
if done || err != nil {
break
}
if h, ok := hdr.(header.IPv6ExperimentExtHdr); ok {
hdr.Release()
return h.Value, true
}
hdr.Release()
}
default:
panic(fmt.Sprintf("Unexpected network protocol number %d", r.segment.pkt.NetworkProtocolNumber))
}
return 0, false
}
func (r *ForwarderRequest) Packet() *stack.PacketBuffer {
return r.segment.pkt
return r.segment.pkt.ExperimentOptionValue()
}

2
pkg/tcpip/transport/tcp/forwarder_mutex.go
View file

@ -60,5 +60,5 @@ func forwarderinitLockNames() {}
func init() {
forwarderinitLockNames()
forwarderprefixIndex = locking.NewMutexClass(reflect.TypeOf(forwarderMutex{}), forwarderlockNames)
forwarderprefixIndex = locking.NewMutexClass(reflect.TypeFor[forwarderMutex](), forwarderlockNames)
}

2
pkg/tcpip/transport/tcp/forwarder_request_mutex.go
View file

@ -60,5 +60,5 @@ func forwarderRequestinitLockNames() {}
func init() {
forwarderRequestinitLockNames()
forwarderRequestprefixIndex = locking.NewMutexClass(reflect.TypeOf(forwarderRequestMutex{}), forwarderRequestlockNames)
forwarderRequestprefixIndex = locking.NewMutexClass(reflect.TypeFor[forwarderRequestMutex](), forwarderRequestlockNames)
}

2
pkg/tcpip/transport/tcp/hasher_mutex.go
View file

@ -60,5 +60,5 @@ func hasherinitLockNames() {}
func init() {
hasherinitLockNames()
hasherprefixIndex = locking.NewMutexClass(reflect.TypeOf(hasherMutex{}), hasherlockNames)
hasherprefixIndex = locking.NewMutexClass(reflect.TypeFor[hasherMutex](), hasherlockNames)
}

2
pkg/tcpip/transport/tcp/keepalive_mutex.go
View file

@ -60,5 +60,5 @@ func keepaliveinitLockNames() {}
func init() {
keepaliveinitLockNames()
keepaliveprefixIndex = locking.NewMutexClass(reflect.TypeOf(keepaliveMutex{}), keepalivelockNames)
keepaliveprefixIndex = locking.NewMutexClass(reflect.TypeFor[keepaliveMutex](), keepalivelockNames)
}

2
pkg/tcpip/transport/tcp/last_error_mutex.go
View file

@ -60,5 +60,5 @@ func lastErrorinitLockNames() {}
func init() {
lastErrorinitLockNames()
lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeOf(lastErrorMutex{}), lastErrorlockNames)
lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeFor[lastErrorMutex](), lastErrorlockNames)
}

2
pkg/tcpip/transport/tcp/pending_processing_mutex.go
View file

@ -60,5 +60,5 @@ func pendingProcessinginitLockNames() {}
func init() {
pendingProcessinginitLockNames()
pendingProcessingprefixIndex = locking.NewMutexClass(reflect.TypeOf(pendingProcessingMutex{}), pendingProcessinglockNames)
pendingProcessingprefixIndex = locking.NewMutexClass(reflect.TypeFor[pendingProcessingMutex](), pendingProcessinglockNames)
}

9
pkg/tcpip/transport/tcp/protocol.go
View file

@ -114,9 +114,11 @@ type protocol struct {
// This is immutable after creation.
probe TCPProbeFunc `state:"nosave"`
// The following secrets are initialized once and stay unchanged after.
seqnumSecret [16]byte
tsOffsetSecret [16]byte
// The following secrets are used for ISN and timestamp-offset
// generation. They are not serialized into checkpoint state and are
// freshly drawn from the secure RNG on restore.
seqnumSecret [16]byte `state:"nosave"`
tsOffsetSecret [16]byte `state:"nosave"`
}
// Number returns the tcp protocol number.
@ -202,6 +204,7 @@ func (p *protocol) tsOffset(src, dst tcpip.Address) tcp.TSOffset {
// then the route's default TTL will be used.
func replyWithReset(st *stack.Stack, s *segment, tos, ipv4TTL uint8, ipv6HopLimit int16) tcpip.Error {
net := s.pkt.Network()
// TODO: b/528377510 - Verify if passing the NICID is correct.
route, err := st.FindRoute(s.pkt.NICID, net.DestinationAddress(), net.SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */)
if err != nil {
return err

2
pkg/tcpip/transport/tcp/protocol_mutex.go
View file

@ -92,5 +92,5 @@ func protocolinitLockNames() {}
func init() {
protocolinitLockNames()
protocolprefixIndex = locking.NewMutexClass(reflect.TypeOf(protocolRWMutex{}), protocollockNames)
protocolprefixIndex = locking.NewMutexClass(reflect.TypeFor[protocolRWMutex](), protocollockNames)
}

31
pkg/tcpip/transport/tcp/protocol_state.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2026 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"context"
"fmt"
)
// afterLoad is invoked by stateify.
func (p *protocol) afterLoad(ctx context.Context) {
rng := p.stack.SecureRNG()
if n, err := rng.Reader.Read(p.seqnumSecret[:]); err != nil || n != len(p.seqnumSecret) {
panic(fmt.Sprintf("rng.Reader.Read(seqnumSecret) failed: n=%d err=%v", n, err))
}
if n, err := rng.Reader.Read(p.tsOffsetSecret[:]); err != nil || n != len(p.tsOffsetSecret) {
panic(fmt.Sprintf("rng.Reader.Read(tsOffsetSecret) failed: n=%d err=%v", n, err))
}
}

22
pkg/tcpip/transport/tcp/rack.go
View file

@ -78,7 +78,15 @@ func (rc *rackControl) init(snd *sender, iss seqnum.Value) {
// update will update the RACK related fields when an ACK has been received.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-09#section-6.2
func (rc *rackControl) update(seg *segment, ackSeg *segment) {
rtt := rc.snd.ep.stack.Clock().NowMonotonic().Sub(seg.xmitTime)
// Compute the RTT sample against the time the ACK was received at ingress
// (ackSeg.rcvdTime), not the current clock. The two differ when the ACK was
// delayed inside the stack before being processed (e.g. it sat in the
// endpoint's segment queue while the application held the endpoint lock
// during a Write that synchronously flushed a window). Using the processing
// time would inflate the RTT by that internal delay, corrupting RACK.RTT,
// RACK.minRTT and the reorder window, and causing spurious loss detection.
// detectLoss already uses ackSeg.rcvdTime; this keeps update consistent.
rtt := ackSeg.rcvdTime.Sub(seg.xmitTime)
// If the ACK is for a retransmitted packet, do not update if it is a
// spurious inference which is determined by below checks:
@ -400,7 +408,17 @@ func (rc *rackControl) reorderTimerExpired() tcpip.Error {
return nil
}
numLost := rc.detectLoss(rc.snd.ep.stack.Clock().NowMonotonic())
// Evaluate loss as of the time the reorder timer was scheduled to fire (its
// target), not the current clock. The timer is armed for the instant a
// segment's reorder window elapses, but the timer callback itself can run
// arbitrarily late under load (the processor goroutine is busy, or the
// endpoint lock is held while a Write flushes a window). Using the
// (possibly much later) processing time would make timeRemaining strongly
// negative and mark not-yet-lost segments as lost, triggering spurious
// retransmits and recovery on a path with little or no real loss
// (gvisor#9707/#9778). detectLoss arms the timer from rcvdTime-derived
// values, so evaluating it at the timer target keeps the two consistent.
numLost := rc.detectLoss(rc.snd.reorderTimer.target)
if numLost == 0 {
return nil
}

36
pkg/tcpip/transport/tcp/rcv.go
View file

@ -96,7 +96,6 @@ func (r *receiver) currentWindow() (curWnd seqnum.Size) {
// getSendParams returns the parameters needed by the sender when building
// segments to send.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) getSendParams() (RcvNxt seqnum.Value, rcvWnd seqnum.Size) {
newWnd := r.ep.selectWindow()
curWnd := r.currentWindow()
@ -187,7 +186,6 @@ func (r *receiver) getSendParams() (RcvNxt seqnum.Value, rcvWnd seqnum.Size) {
// in such cases we may need to send an ack to indicate to our peer that it can
// resume sending data.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) nonZeroWindow() {
// Immediately send an ack.
r.ep.snd.sendAck()
@ -200,7 +198,6 @@ func (r *receiver) nonZeroWindow() {
// Returns true if the segment was consumed, false if it cannot be consumed
// yet because of a missing segment.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum.Size) bool {
if segLen > 0 {
// If the segment doesn't include the seqnum we're expecting to
@ -265,7 +262,7 @@ func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum
case StateEstablished:
r.ep.setEndpointState(StateCloseWait)
case StateFinWait1:
if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt {
if s.flags.Contains(header.TCPFlagAck) && r.ep.snd.finSent && s.ackNumber == r.ep.snd.SndNxt {
// FIN-ACK, transition to TIME-WAIT.
r.ep.setEndpointState(StateTimeWait)
} else {
@ -299,8 +296,9 @@ func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum
}
// Handle ACK (not FIN-ACK, which we handled above) during one of the
// shutdown states.
if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt {
// shutdown states. These completions require that our FIN was sent;
// without finSent a data ACK would be mistaken for a FIN ACK.
if s.flags.Contains(header.TCPFlagAck) && r.ep.snd.finSent && s.ackNumber == r.ep.snd.SndNxt {
switch r.ep.EndpointState() {
case StateFinWait1:
r.ep.setEndpointState(StateFinWait2)
@ -320,9 +318,12 @@ func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum
return true
}
// updateRTT updates the receiver RTT measurement based on the sequence number
// of the received segment.
func (r *receiver) updateRTT() {
// updateRTT estimates a receiver-side RTT for receive-buffer autotuning, based
// on the sequence number of the received segment. rcvdTime is the ingress
// timestamp of the segment that triggered this measurement; it is used instead
// of the current clock so that an internal processing delay does not inflate the
// estimate (which would size the receive buffer too large).
func (r *receiver) updateRTT(rcvdTime tcpip.MonotonicTime) {
// From: https://public.lanl.gov/radiant/pubs/drs/sc2001-poster.pdf
//
// A system that is only transmitting acknowledgements can still
@ -332,7 +333,7 @@ func (r *receiver) updateRTT() {
r.ep.rcvQueueMu.Lock()
if r.ep.RcvAutoParams.RTTMeasureTime == (tcpip.MonotonicTime{}) {
// New measurement.
r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic()
r.ep.RcvAutoParams.RTTMeasureTime = rcvdTime
r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd)
r.ep.rcvQueueMu.Unlock()
return
@ -341,20 +342,19 @@ func (r *receiver) updateRTT() {
r.ep.rcvQueueMu.Unlock()
return
}
rtt := r.ep.stack.Clock().NowMonotonic().Sub(r.ep.RcvAutoParams.RTTMeasureTime)
rtt := rcvdTime.Sub(r.ep.RcvAutoParams.RTTMeasureTime)
// We only store the minimum observed RTT here as this is only used in
// absence of a SRTT available from either timestamps or a sender
// measurement of RTT.
if r.ep.RcvAutoParams.RTT == 0 || rtt < r.ep.RcvAutoParams.RTT {
r.ep.RcvAutoParams.RTT = rtt
}
r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic()
r.ep.RcvAutoParams.RTTMeasureTime = rcvdTime
r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd)
r.ep.rcvQueueMu.Unlock()
}
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) handleRcvdSegmentClosing(s *segment, state EndpointState, closed bool) (drop bool, err tcpip.Error) {
r.ep.rcvQueueMu.Lock()
rcvClosed := r.ep.RcvClosed || r.closed
@ -452,7 +452,6 @@ func (r *receiver) handleRcvdSegmentClosing(s *segment, state EndpointState, clo
// handleRcvdSegment handles TCP segments directed at the connection managed by
// r as they arrive. It is called by the protocol main loop.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) {
state := r.ep.EndpointState()
closed := r.ep.closed
@ -475,8 +474,10 @@ func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) {
}
}
// Store the time of the last ack.
r.lastRcvdAckTime = r.ep.stack.Clock().NowMonotonic()
// Store the time of the last ack. Use the segment's ingress time rather than
// the current clock so a segment delayed inside the stack before processing
// records when it actually arrived (consumed by the user-timeout check).
r.lastRcvdAckTime = s.rcvdTime
// Defer segment processing if it can't be consumed now.
if !r.consumeSegment(s, segSeq, segLen) {
@ -519,7 +520,7 @@ func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) {
// Since we consumed a segment update the receiver's RTT estimate
// if required.
if segLen > 0 {
r.updateRTT()
r.updateRTT(s.rcvdTime)
}
// By consuming the current segment, we may have filled a gap in the
@ -548,7 +549,6 @@ func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) {
// handleTimeWaitSegment handles inbound segments received when the endpoint
// has entered the TIME_WAIT state.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) handleTimeWaitSegment(s *segment) (resetTimeWait bool, newSyn bool) {
segSeq := s.sequenceNumber
segLen := seqnum.Size(s.payloadSize())

2
pkg/tcpip/transport/tcp/rcv_queue_mutex.go
View file

@ -60,5 +60,5 @@ func rcvQueueinitLockNames() {}
func init() {
rcvQueueinitLockNames()
rcvQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvQueueMutex{}), rcvQueuelockNames)
rcvQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[rcvQueueMutex](), rcvQueuelockNames)
}

4
pkg/tcpip/transport/tcp/reno.go
View file

@ -16,6 +16,8 @@ package tcp
import (
"time"
"github.com/sagernet/gvisor/pkg/tcpip"
)
// renoState stores the variables related to TCP New Reno congestion
@ -80,7 +82,7 @@ func (r *renoState) reduceSlowStartThreshold() {
// Update implements congestionControl.Update.
//
// +checklocks:r.s.ep.mu
func (r *renoState) Update(packetsAcked int, _ time.Duration) {
func (r *renoState) Update(packetsAcked int, _ time.Duration, _ tcpip.MonotonicTime) {
if r.s.SndCwnd < r.s.Ssthresh {
packetsAcked = r.updateSlowStart(packetsAcked)
if packetsAcked == 0 {

2
pkg/tcpip/transport/tcp/rtt_mutex.go
View file

@ -60,5 +60,5 @@ func rttinitLockNames() {}
func init() {
rttinitLockNames()
rttprefixIndex = locking.NewMutexClass(reflect.TypeOf(rttMutex{}), rttlockNames)
rttprefixIndex = locking.NewMutexClass(reflect.TypeFor[rttMutex](), rttlockNames)
}

69
pkg/tcpip/transport/tcp/sack_scoreboard.go
View file

@ -34,6 +34,12 @@ const (
defaultBtreeDegree = 2
)
// sackBlockLess is the comparison function for BTreeG, replacing the
// btree.Item interface method.
func sackBlockLess(a, b header.SACKBlock) bool {
return a.Start.LessThan(b.Start)
}
// SACKScoreboard stores a set of disjoint SACK ranges.
//
// +stateify savable
@ -47,22 +53,22 @@ type SACKScoreboard struct {
// the TCP/IP headers and options.
smss uint16
maxSACKED seqnum.Value
sacked seqnum.Size `state:"nosave"`
ranges *btree.BTree `state:"nosave"`
sacked seqnum.Size `state:"nosave"`
ranges *btree.BTreeG[header.SACKBlock] `state:"nosave"`
}
// NewSACKScoreboard returns a new SACK Scoreboard.
func NewSACKScoreboard(smss uint16, iss seqnum.Value) *SACKScoreboard {
return &SACKScoreboard{
smss: smss,
ranges: btree.New(defaultBtreeDegree),
ranges: btree.NewG[header.SACKBlock](defaultBtreeDegree, sackBlockLess),
maxSACKED: iss,
}
}
// Reset erases all known range information from the SACK scoreboard.
func (s *SACKScoreboard) Reset() {
s.ranges = btree.New(defaultBtreeDegree)
s.ranges = btree.NewG[header.SACKBlock](defaultBtreeDegree, sackBlockLess)
s.sacked = 0
}
@ -73,15 +79,14 @@ func (s *SACKScoreboard) Insert(r header.SACKBlock) {
}
// Check if we can merge the new range with a range before or after it.
var toDelete []btree.Item
var toDelete []header.SACKBlock
if s.maxSACKED.LessThan(r.End - 1) {
s.maxSACKED = r.End - 1
}
s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool {
if i == r {
s.ranges.AscendGreaterOrEqual(r, func(sacked header.SACKBlock) bool {
if sacked == r {
return true
}
sacked := i.(header.SACKBlock)
// There is a hole between these two SACK blocks, so we can't
// merge anymore.
if r.End.LessThan(sacked.Start) {
@ -96,21 +101,20 @@ func (s *SACKScoreboard) Insert(r header.SACKBlock) {
if sacked.End.LessThan(r.End) {
// sacked is contained in the newly inserted range.
// Delete this block.
toDelete = append(toDelete, i)
toDelete = append(toDelete, sacked)
return true
}
// sacked covers a range past end of the newly inserted
// block.
r.End = sacked.End
toDelete = append(toDelete, i)
toDelete = append(toDelete, sacked)
return true
})
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
if i == r {
s.ranges.DescendLessOrEqual(r, func(sacked header.SACKBlock) bool {
if sacked == r {
return true
}
sacked := i.(header.SACKBlock)
// sA------sE
// rA----rE
if sacked.End.LessThan(r.Start) {
@ -126,18 +130,17 @@ func (s *SACKScoreboard) Insert(r header.SACKBlock) {
if r.End.LessThan(sacked.End) {
r.End = sacked.End
}
toDelete = append(toDelete, i)
toDelete = append(toDelete, sacked)
return true
})
for _, i := range toDelete {
if sb := s.ranges.Delete(i); sb != nil {
sb := i.(header.SACKBlock)
for _, sb := range toDelete {
if _, ok := s.ranges.Delete(sb); ok {
s.sacked -= sb.Start.Size(sb.End)
}
}
replaced := s.ranges.ReplaceOrInsert(r)
if replaced == nil {
_, replaced := s.ranges.ReplaceOrInsert(r)
if !replaced {
s.sacked += r.Start.Size(r.End)
}
}
@ -150,8 +153,7 @@ func (s *SACKScoreboard) IsSACKED(r header.SACKBlock) bool {
}
found := false
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
sacked := i.(header.SACKBlock)
s.ranges.DescendLessOrEqual(r, func(sacked header.SACKBlock) bool {
if sacked.End.LessThan(r.Start) {
return false
}
@ -168,8 +170,8 @@ func (s *SACKScoreboard) IsSACKED(r header.SACKBlock) bool {
func (s *SACKScoreboard) String() string {
var str strings.Builder
str.WriteString("SACKScoreboard: {")
s.ranges.Ascend(func(i btree.Item) bool {
str.WriteString(fmt.Sprintf("%v,", i))
s.ranges.Ascend(func(sb header.SACKBlock) bool {
fmt.Fprintf(&str, "%v,", sb)
return true
})
str.WriteString("}\n")
@ -181,15 +183,14 @@ func (s *SACKScoreboard) Delete(seq seqnum.Value) {
if s.Empty() {
return
}
toDelete := []btree.Item{}
toInsert := []btree.Item{}
var toDelete []header.SACKBlock
var toInsert []header.SACKBlock
r := header.SACKBlock{seq, seq.Add(1)}
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
if i == r {
s.ranges.DescendLessOrEqual(r, func(sb header.SACKBlock) bool {
if sb == r {
return true
}
sb := i.(header.SACKBlock)
toDelete = append(toDelete, i)
toDelete = append(toDelete, sb)
if sb.End.LessThanEq(seq) {
s.sacked -= sb.Start.Size(sb.End)
} else {
@ -209,8 +210,8 @@ func (s *SACKScoreboard) Delete(seq seqnum.Value) {
// Copy provides a copy of the SACK scoreboard.
func (s *SACKScoreboard) Copy() (sackBlocks []header.SACKBlock, maxSACKED seqnum.Value) {
s.ranges.Ascend(func(i btree.Item) bool {
sackBlocks = append(sackBlocks, i.(header.SACKBlock))
s.ranges.Ascend(func(sb header.SACKBlock) bool {
sackBlocks = append(sackBlocks, sb)
return true
})
return sackBlocks, s.maxSACKED
@ -232,8 +233,7 @@ func (s *SACKScoreboard) IsRangeLost(r header.SACKBlock) bool {
// We need to check if the immediate lower (if any) sacked
// range contains or partially overlaps with r.
searchMore := true
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
sacked := i.(header.SACKBlock)
s.ranges.DescendLessOrEqual(r, func(sacked header.SACKBlock) bool {
if sacked.Contains(r) {
searchMore = false
return false
@ -256,8 +256,7 @@ func (s *SACKScoreboard) IsRangeLost(r header.SACKBlock) bool {
return isLost
}
s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool {
sacked := i.(header.SACKBlock)
s.ranges.AscendGreaterOrEqual(r, func(sacked header.SACKBlock) bool {
if sacked.Contains(r) {
return false
}

7
pkg/tcpip/transport/tcp/segment.go
View file

@ -139,11 +139,14 @@ func newIncomingSegment(id stack.TransportEndpointID, clock tcpip.Clock, pkt *st
return s, nil
}
func newOutgoingSegment(id stack.TransportEndpointID, clock tcpip.Clock, buf buffer.Buffer) *segment {
func newOutgoingSegment(id stack.TransportEndpointID, clock tcpip.Clock, buf buffer.Buffer, mark uint32) *segment {
s := newSegment()
s.id = id
s.rcvdTime = clock.NowMonotonic()
s.pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf})
s.pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{
Payload: buf,
Mark: mark,
})
s.dataMemSize = s.pkt.MemSize()
return s
}

2
pkg/tcpip/transport/tcp/segment_queue_mutex.go
View file

@ -60,5 +60,5 @@ func segmentQueueinitLockNames() {}
func init() {
segmentQueueinitLockNames()
segmentQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(segmentQueueMutex{}), segmentQueuelockNames)
segmentQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[segmentQueueMutex](), segmentQueuelockNames)
}

103
pkg/tcpip/transport/tcp/snd.go
View file

@ -74,8 +74,11 @@ type congestionControl interface {
// Update is invoked when processing inbound acks. It's passed the
// number of packet's that were acked by the most recent cumulative
// acknowledgement. rtt is the round-trip time, or is set to unknownRTT
// (above) to indicate the time is unknown.
Update(packetsAcked int, rtt time.Duration)
// (above) to indicate the time is unknown. ackTime is the time the
// processed ACK arrived at the stack (its ingress timestamp), used for
// arrival-anchored timing such as CUBIC HyStart's ACK-train detection so
// that an ACK delayed inside the stack does not distort it.
Update(packetsAcked int, rtt time.Duration, ackTime tcpip.MonotonicTime)
// PostRecovery is invoked when the sender is exiting a fast retransmit/
// recovery phase. This provides congestion control algorithms a way
@ -96,6 +99,7 @@ type lossRecovery interface {
// sender holds the state necessary to send TCP segments.
//
// +checklocksalias:rc.snd.ep.mu=ep.mu
// +stateify savable
type sender struct {
// +checklocks:ep.mu
@ -103,6 +107,11 @@ type sender struct {
ep *Endpoint
// finSent is set when the FIN segment is actually transmitted.
// The endpoint may be in FIN_WAIT1/LAST_ACK while the FIN is still
// queued behind blocked data; finSent guards closing ACK completions.
finSent bool
// lr is the loss recovery algorithm used by the sender.
lr lossRecovery
@ -242,13 +251,13 @@ type rtt struct {
}
// +checklocks:ep.mu
func newSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int) *sender {
func initSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int) {
// The sender MUST reduce the TCP data length to account for any IP or
// TCP options that it is including in the packets that it sends.
// See: https://tools.ietf.org/html/rfc6691#section-2
maxPayloadSize := int(mss) - ep.maxOptionSize()
s := &sender{
ep.snd = &sender{
ep: ep,
TCPSenderState: TCPSenderState{
SndWnd: sndWnd,
@ -271,59 +280,50 @@ func newSender(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint
set: make(map[*segment]struct{}),
},
}
return newSenderHelper(ep, iss, irs, sndWnd, mss, sndWndScale, maxPayloadSize, s)
}
// newSenderHelper exists to sate checklocks.
//
// +checklocks:ep.mu
// +checklocksalias:s.ep.mu=ep.mu
func newSenderHelper(ep *Endpoint, iss, irs seqnum.Value, sndWnd seqnum.Size, mss uint16, sndWndScale int, maxPayloadSize int, s *sender) *sender {
if s.gso {
s.ep.gso.MSS = uint16(maxPayloadSize)
if ep.snd.gso {
ep.snd.ep.gso.MSS = uint16(maxPayloadSize)
}
s.cc = s.initCongestionControl(ep.cc)
s.lr = s.initLossRecovery()
s.rc.init(s, iss)
ep.snd.cc = ep.snd.initCongestionControl(ep.cc)
ep.snd.lr = ep.snd.initLossRecovery()
ep.snd.rc.init(ep.snd, iss)
// A negative sndWndScale means that no scaling is in use, otherwise we
// store the scaling value.
if sndWndScale > 0 {
s.SndWndScale = uint8(sndWndScale)
ep.snd.SndWndScale = uint8(sndWndScale)
}
s.resendTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.retransmitTimerExpired))
s.reorderTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.rc.reorderTimerExpired))
s.probeTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.probeTimerExpired))
s.corkTimer.init(s.ep.stack.Clock(), timerHandler(s.ep, s.corkTimerExpired))
ep.snd.resendTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.retransmitTimerExpired))
ep.snd.reorderTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.rc.reorderTimerExpired))
ep.snd.probeTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.probeTimerExpired))
ep.snd.corkTimer.init(ep.snd.ep.stack.Clock(), timerHandler(ep.snd.ep, ep.snd.corkTimerExpired))
s.updateMaxPayloadSize(int(ep.route.MTU()), 0)
ep.snd.updateMaxPayloadSize(int(ep.snd.ep.route.MTU()), 0)
// Initialize SACK Scoreboard after updating max payload size as we use
// the maxPayloadSize as the smss when determining if a segment is lost
// etc.
s.ep.scoreboard = NewSACKScoreboard(uint16(s.MaxPayloadSize), iss)
ep.snd.ep.scoreboard = NewSACKScoreboard(uint16(ep.snd.MaxPayloadSize), iss)
// Get Stack wide config.
var minRTO tcpip.TCPMinRTOOption
if err := ep.stack.TransportProtocolOption(ProtocolNumber, &minRTO); err != nil {
if err := ep.snd.ep.stack.TransportProtocolOption(ProtocolNumber, &minRTO); err != nil {
panic(fmt.Sprintf("unable to get minRTO from stack: %s", err))
}
s.minRTO = time.Duration(minRTO)
ep.snd.minRTO = time.Duration(minRTO)
var maxRTO tcpip.TCPMaxRTOOption
if err := ep.stack.TransportProtocolOption(ProtocolNumber, &maxRTO); err != nil {
if err := ep.snd.ep.stack.TransportProtocolOption(ProtocolNumber, &maxRTO); err != nil {
panic(fmt.Sprintf("unable to get maxRTO from stack: %s", err))
}
s.maxRTO = time.Duration(maxRTO)
ep.snd.maxRTO = time.Duration(maxRTO)
var maxRetries tcpip.TCPMaxRetriesOption
if err := ep.stack.TransportProtocolOption(ProtocolNumber, &maxRetries); err != nil {
if err := ep.snd.ep.stack.TransportProtocolOption(ProtocolNumber, &maxRetries); err != nil {
panic(fmt.Sprintf("unable to get maxRetries from stack: %s", err))
}
s.maxRetries = uint32(maxRetries)
return s
ep.snd.maxRetries = uint32(maxRetries)
}
// initCongestionControl initializes the specified congestion control module and
@ -434,6 +434,13 @@ func (s *sender) sendAck() {
//
// +checklocks:s.ep.mu
func (s *sender) updateRTO(rtt time.Duration) {
// A negative RTT sample is nonsensical and would skew SRTT/RTTVar (and thus
// RTO). RTT samples are now anchored to a segment's ingress time, which is
// monotonic and never after the corresponding send time, so this should not
// occur; guard defensively rather than corrupt the estimator.
if rtt < 0 {
return
}
s.rtt.Lock()
if !s.rtt.TCPRTTState.SRTTInited {
s.rtt.TCPRTTState.RTTVar = rtt / 2
@ -902,15 +909,9 @@ func (s *sender) maybeSendSegment(seg *segment, limit int, end seqnum.Value) (se
}
seg.flags = header.TCPFlagAck | header.TCPFlagFin
segEnd = seg.sequenceNumber.Add(1)
// Update the state to reflect that we have now
// queued a FIN.
s.ep.updateConnDirectionState(connDirectionStateSndClosed)
switch s.ep.EndpointState() {
case StateCloseWait:
s.ep.setEndpointState(StateLastAck)
default:
s.ep.setEndpointState(StateFinWait1)
}
// FIN is now being transmitted; mark it so the receiver can tell
// a data-only ACK apart from one that acknowledges our FIN.
s.finSent = true
} else {
// We're sending a non-FIN segment.
if seg.flags&header.TCPFlagFin != 0 {
@ -1013,6 +1014,7 @@ func (s *sender) sendZeroWindowProbe() {
// we re-send an ACKed byte to goad the receiver into responding.
pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
Payload: buffer.MakeWithData(zeroProbeJunk),
Mark: s.ep.ops.GetMark(),
})
defer pkt.DecRef()
s.sendSegmentFromPacketBuffer(pkt, header.TCPFlagAck, s.SndUna-1)
@ -1518,13 +1520,16 @@ func (s *sender) inRecovery() bool {
// handleRcvdSegment is called when a segment is received; it is responsible for
// updating the send-related state.
// +checklocks:s.ep.mu
// +checklocksalias:s.rc.snd.ep.mu=s.ep.mu
func (s *sender) handleRcvdSegment(rcvdSeg *segment) {
bestRTT := unknownRTT
// Check if we can extract an RTT measurement from this ack.
// Check if we can extract an RTT measurement from this ack. Measure against
// the ACK's ingress time (rcvdSeg.rcvdTime), not the current clock: if the
// ACK was delayed inside the stack before being processed (e.g. queued while
// the application held the endpoint lock during a Write), using the
// processing time would inflate the RTT sample and thus SRTT/RTO.
if !rcvdSeg.parsedOptions.TS && s.RTTMeasureSeqNum.LessThan(rcvdSeg.ackNumber) {
bestRTT = s.ep.stack.Clock().NowMonotonic().Sub(s.RTTMeasureTime)
bestRTT = rcvdSeg.rcvdTime.Sub(s.RTTMeasureTime)
s.updateRTO(bestRTT)
s.RTTMeasureSeqNum = s.SndNxt
}
@ -1627,7 +1632,10 @@ func (s *sender) handleRcvdSegment(rcvdSeg *segment) {
// some new data, i.e., only if it advances the left edge of
// the send window.
if s.ep.SendTSOk && rcvdSeg.parsedOptions.TSEcr != 0 {
tsRTT := s.ep.elapsed(s.ep.stack.Clock().NowMonotonic(), rcvdSeg.parsedOptions.TSEcr)
// Compute elapsed time from the ACK's ingress time, not the current
// clock, so an ACK delayed inside the stack before processing does
// not inflate the timestamp-based RTT sample (and thus SRTT/RTO).
tsRTT := s.ep.elapsed(rcvdSeg.rcvdTime, rcvdSeg.parsedOptions.TSEcr)
s.updateRTO(tsRTT)
// Following Linux, prefer RTT computed from ACKs to TSEcr because,
// "broken middle-boxes or peers may corrupt TS-ECR fields"
@ -1707,7 +1715,7 @@ func (s *sender) handleRcvdSegment(rcvdSeg *segment) {
// If we are not in fast recovery then update the congestion
// window based on the number of acknowledged packets.
if !s.FastRecovery.Active {
s.cc.Update(originalOutstanding-s.Outstanding, bestRTT)
s.cc.Update(originalOutstanding-s.Outstanding, bestRTT, rcvdSeg.rcvdTime)
if s.FastRecovery.Last.LessThan(s.SndUna) {
s.state = tcpip.Open
// Update RACK when we are exiting fast or RTO
@ -1822,8 +1830,6 @@ func (s *sender) sendSegment(seg *segment) tcpip.Error {
// sendSegmentFromPacketBuffer sends a new segment containing the given payload,
// flags and sequence number.
// +checklocks:s.ep.mu
// +checklocksalias:s.ep.rcv.ep.mu=s.ep.mu
// +checklocksalias:s.ep.rcv.ep.snd.ep.mu=s.ep.mu
func (s *sender) sendSegmentFromPacketBuffer(pkt *stack.PacketBuffer, flags header.TCPFlags, seq seqnum.Value) tcpip.Error {
s.LastSendTime = s.ep.stack.Clock().NowMonotonic()
if seq == s.RTTMeasureSeqNum {
@ -1845,9 +1851,6 @@ func (s *sender) sendSegmentFromPacketBuffer(pkt *stack.PacketBuffer, flags head
// sendEmptySegment sends a new empty segment, flags and sequence number.
// +checklocks:s.ep.mu
// +checklocksalias:s.ep.rcv.ep.snd.ep.mu=s.ep.mu
// +checklocksalias:s.ep.rcv.ep.mu=s.ep.mu
// +checklocksalias:s.ep.snd.ep.mu=s.ep.mu
func (s *sender) sendEmptySegment(flags header.TCPFlags, seq seqnum.Value) tcpip.Error {
s.LastSendTime = s.ep.stack.Clock().NowMonotonic()
if seq == s.RTTMeasureSeqNum {

2
pkg/tcpip/transport/tcp/snd_queue_mutex.go
View file

@ -60,5 +60,5 @@ func sndQueueinitLockNames() {}
func init() {
sndQueueinitLockNames()
sndQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(sndQueueMutex{}), sndQueuelockNames)
sndQueueprefixIndex = locking.NewMutexClass(reflect.TypeFor[sndQueueMutex](), sndQueuelockNames)
}

93
pkg/tcpip/transport/tcp/tcp_state_autogen.go
View file

@ -25,8 +25,8 @@ func (a *acceptQueue) beforeSave() {}
// +checklocksignore
func (a *acceptQueue) StateSave(stateSinkObject state.Sink) {
a.beforeSave()
var endpointsValue []*Endpoint
endpointsValue = a.saveEndpoints()
endpointsValue := a.saveEndpoints()
_ = ([]*Endpoint)(endpointsValue)
stateSinkObject.SaveValue(0, endpointsValue)
stateSinkObject.Save(1, &a.pendingEndpoints)
stateSinkObject.Save(2, &a.capacity)
@ -150,9 +150,7 @@ func (q *epQueue) StateTypeName() string {
}
func (q *epQueue) StateFields() []string {
return []string{
"list",
}
return []string{}
}
func (q *epQueue) beforeSave() {}
@ -160,14 +158,12 @@ func (q *epQueue) beforeSave() {}
// +checklocksignore
func (q *epQueue) StateSave(stateSinkObject state.Sink) {
q.beforeSave()
stateSinkObject.Save(0, &q.list)
}
func (q *epQueue) afterLoad(context.Context) {}
// +checklocksignore
func (q *epQueue) StateLoad(ctx context.Context, stateSourceObject state.Source) {
stateSourceObject.Load(0, &q.list)
}
func (p *processor) StateTypeName() string {
@ -496,14 +492,15 @@ func (e *Endpoint) StateFields() []string {
"lastOutOfWindowAckTime",
"pmtud",
"alsoBindToV4",
"terminateAtRestore",
}
}
// +checklocksignore
func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
e.beforeSave()
var stateValue EndpointState
stateValue = e.saveState()
stateValue := e.saveState()
_ = (EndpointState)(stateValue)
stateSinkObject.SaveValue(12, stateValue)
stateSinkObject.Save(0, &e.TCPEndpointStateInner)
stateSinkObject.Save(1, &e.TransportEndpointInfo)
@ -561,6 +558,7 @@ func (e *Endpoint) StateSave(stateSinkObject state.Sink) {
stateSinkObject.Save(54, &e.lastOutOfWindowAckTime)
stateSinkObject.Save(55, &e.pmtud)
stateSinkObject.Save(56, &e.alsoBindToV4)
stateSinkObject.Save(57, &e.terminateAtRestore)
}
// +checklocksignore
@ -621,6 +619,7 @@ func (e *Endpoint) StateLoad(ctx context.Context, stateSourceObject state.Source
stateSourceObject.Load(54, &e.lastOutOfWindowAckTime)
stateSourceObject.Load(55, &e.pmtud)
stateSourceObject.Load(56, &e.alsoBindToV4)
stateSourceObject.Load(57, &e.terminateAtRestore)
stateSourceObject.LoadValue(12, new(EndpointState), func(y any) { e.loadState(ctx, y.(EndpointState)) })
stateSourceObject.AfterLoad(func() { e.afterLoad(ctx) })
}
@ -683,8 +682,6 @@ func (p *protocol) StateFields() []string {
"maxRetries",
"synRetries",
"dispatcher",
"seqnumSecret",
"tsOffsetSecret",
}
}
@ -711,12 +708,8 @@ func (p *protocol) StateSave(stateSinkObject state.Sink) {
stateSinkObject.Save(15, &p.maxRetries)
stateSinkObject.Save(16, &p.synRetries)
stateSinkObject.Save(17, &p.dispatcher)
stateSinkObject.Save(18, &p.seqnumSecret)
stateSinkObject.Save(19, &p.tsOffsetSecret)
}
func (p *protocol) afterLoad(context.Context) {}
// +checklocksignore
func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source) {
stateSourceObject.Load(0, &p.stack)
@ -737,8 +730,7 @@ func (p *protocol) StateLoad(ctx context.Context, stateSourceObject state.Source
stateSourceObject.Load(15, &p.maxRetries)
stateSourceObject.Load(16, &p.synRetries)
stateSourceObject.Load(17, &p.dispatcher)
stateSourceObject.Load(18, &p.seqnumSecret)
stateSourceObject.Load(19, &p.tsOffsetSecret)
stateSourceObject.AfterLoad(func() { p.afterLoad(ctx) })
}
func (rc *rackControl) StateTypeName() string {
@ -964,8 +956,8 @@ func (s *segment) beforeSave() {}
// +checklocksignore
func (s *segment) StateSave(stateSinkObject state.Sink) {
s.beforeSave()
var optionsValue []byte
optionsValue = s.saveOptions()
optionsValue := s.saveOptions()
_ = ([]byte)(optionsValue)
stateSinkObject.SaveValue(12, optionsValue)
stateSinkObject.Save(0, &s.segmentEntry)
stateSinkObject.Save(1, &s.segmentRefs)
@ -1053,6 +1045,7 @@ func (s *sender) StateFields() []string {
return []string{
"TCPSenderState",
"ep",
"finSent",
"lr",
"firstRetransmittedSegXmitTime",
"writeNext",
@ -1078,21 +1071,22 @@ func (s *sender) StateSave(stateSinkObject state.Sink) {
s.beforeSave()
stateSinkObject.Save(0, &s.TCPSenderState)
stateSinkObject.Save(1, &s.ep)
stateSinkObject.Save(2, &s.lr)
stateSinkObject.Save(3, &s.firstRetransmittedSegXmitTime)
stateSinkObject.Save(4, &s.writeNext)
stateSinkObject.Save(5, &s.writeList)
stateSinkObject.Save(6, &s.rtt)
stateSinkObject.Save(7, &s.minRTO)
stateSinkObject.Save(8, &s.maxRTO)
stateSinkObject.Save(9, &s.maxRetries)
stateSinkObject.Save(10, &s.gso)
stateSinkObject.Save(11, &s.state)
stateSinkObject.Save(12, &s.cc)
stateSinkObject.Save(13, &s.rc)
stateSinkObject.Save(14, &s.spuriousRecovery)
stateSinkObject.Save(15, &s.retransmitTS)
stateSinkObject.Save(16, &s.startCork)
stateSinkObject.Save(2, &s.finSent)
stateSinkObject.Save(3, &s.lr)
stateSinkObject.Save(4, &s.firstRetransmittedSegXmitTime)
stateSinkObject.Save(5, &s.writeNext)
stateSinkObject.Save(6, &s.writeList)
stateSinkObject.Save(7, &s.rtt)
stateSinkObject.Save(8, &s.minRTO)
stateSinkObject.Save(9, &s.maxRTO)
stateSinkObject.Save(10, &s.maxRetries)
stateSinkObject.Save(11, &s.gso)
stateSinkObject.Save(12, &s.state)
stateSinkObject.Save(13, &s.cc)
stateSinkObject.Save(14, &s.rc)
stateSinkObject.Save(15, &s.spuriousRecovery)
stateSinkObject.Save(16, &s.retransmitTS)
stateSinkObject.Save(17, &s.startCork)
}
func (s *sender) afterLoad(context.Context) {}
@ -1101,21 +1095,22 @@ func (s *sender) afterLoad(context.Context) {}
func (s *sender) StateLoad(ctx context.Context, stateSourceObject state.Source) {
stateSourceObject.Load(0, &s.TCPSenderState)
stateSourceObject.Load(1, &s.ep)
stateSourceObject.Load(2, &s.lr)
stateSourceObject.Load(3, &s.firstRetransmittedSegXmitTime)
stateSourceObject.Load(4, &s.writeNext)
stateSourceObject.Load(5, &s.writeList)
stateSourceObject.Load(6, &s.rtt)
stateSourceObject.Load(7, &s.minRTO)
stateSourceObject.Load(8, &s.maxRTO)
stateSourceObject.Load(9, &s.maxRetries)
stateSourceObject.Load(10, &s.gso)
stateSourceObject.Load(11, &s.state)
stateSourceObject.Load(12, &s.cc)
stateSourceObject.Load(13, &s.rc)
stateSourceObject.Load(14, &s.spuriousRecovery)
stateSourceObject.Load(15, &s.retransmitTS)
stateSourceObject.Load(16, &s.startCork)
stateSourceObject.Load(2, &s.finSent)
stateSourceObject.Load(3, &s.lr)
stateSourceObject.Load(4, &s.firstRetransmittedSegXmitTime)
stateSourceObject.Load(5, &s.writeNext)
stateSourceObject.Load(6, &s.writeList)
stateSourceObject.Load(7, &s.rtt)
stateSourceObject.Load(8, &s.minRTO)
stateSourceObject.Load(9, &s.maxRTO)
stateSourceObject.Load(10, &s.maxRetries)
stateSourceObject.Load(11, &s.gso)
stateSourceObject.Load(12, &s.state)
stateSourceObject.Load(13, &s.cc)
stateSourceObject.Load(14, &s.rc)
stateSourceObject.Load(15, &s.spuriousRecovery)
stateSourceObject.Load(16, &s.retransmitTS)
stateSourceObject.Load(17, &s.startCork)
}
func (wl *protectedWriteList) StateTypeName() string {

10
pkg/tcpip/transport/udp/endpoint.go
View file

@ -294,6 +294,10 @@ func (e *endpoint) Read(dst io.Writer, opts tcpip.ReadOptions) (tcpip.ReadResult
if opts.NeedRemoteAddr {
res.RemoteAddr = p.senderAddress
}
if opts.NeedReceivedExperimentOption {
expOptVal, _ := p.pkt.ExperimentOptionValue()
res.ReceivedExperimentOption = expOptVal
}
n, err := p.pkt.Data().ReadTo(dst, opts.Peek)
if n == 0 && err != nil {
@ -463,9 +467,9 @@ func (e *endpoint) write(p tcpip.Payloader, opts tcpip.WriteOptions) (int64, tcp
dataSz := p.Len()
pktInfo := udpInfo.ctx.PacketInfo()
pkt := udpInfo.ctx.TryNewPacketBufferFromPayloader(header.UDPMinimumSize+int(pktInfo.MaxHeaderLength), p)
if pkt == nil {
return 0, &tcpip.ErrWouldBlock{}
pkt, err := udpInfo.ctx.TryNewPacketBufferFromPayloader(header.UDPMinimumSize+int(pktInfo.MaxHeaderLength), p)
if err != nil {
return 0, err
}
defer pkt.DecRef()

32
pkg/tcpip/transport/udp/endpoint_state.go
View file

@ -21,7 +21,6 @@ import (
"github.com/sagernet/gvisor/pkg/log"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/tcpip/transport"
)
// saveReceivedAt is invoked by stateify.
@ -36,11 +35,7 @@ func (p *udpPacket) loadReceivedAt(_ context.Context, nsec int64) {
// afterLoad is invoked by stateify.
func (e *endpoint) afterLoad(ctx context.Context) {
if e.stack.IsSaveRestoreEnabled() {
e.stack.RegisterRestoredEndpoint(e)
} else {
stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e)
}
e.stack.RegisterRestoredEndpoint(e)
}
// beforeSave is invoked by stateify.
@ -62,32 +57,7 @@ func (e *endpoint) Restore(s *stack.Stack) {
// Unfreeze the endpoint to handle packets.
e.frozen = false
if e.stack.IsSaveRestoreEnabled() {
e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
return
}
e.stack = s
e.ops.InitHandler(e, e.stack, tcpip.GetStackSendBufferLimits, tcpip.GetStackReceiveBufferLimits)
switch state := e.net.State(); state {
case transport.DatagramEndpointStateInitial, transport.DatagramEndpointStateClosed:
case transport.DatagramEndpointStateBound, transport.DatagramEndpointStateConnected:
// Our saved state had a port, but we don't actually have a
// reservation. We need to remove the port from our state, but still
// pass it to the reservation machinery.
var err tcpip.Error
id := e.net.Info().ID
id.LocalPort = e.localPort
id.RemotePort = e.remotePort
id, e.boundBindToDevice, err = e.registerWithStack(e.effectiveNetProtos, id)
if err != nil {
panic("registering udp endpoint with the stack failed during restore")
}
e.localPort = id.LocalPort
e.remotePort = id.RemotePort
default:
panic("unhandled state")
}
}
// Resume implements tcpip.ResumableEndpoint.Resume.

13
pkg/tcpip/transport/udp/forwarder.go
View file

@ -65,6 +65,15 @@ type ForwarderRequest struct {
pkt *stack.PacketBuffer
}
// NewForwarderRequest creates a new ForwarderRequest.
func NewForwarderRequest(stack *stack.Stack, id stack.TransportEndpointID, pkt *stack.PacketBuffer) *ForwarderRequest {
return &ForwarderRequest{
stack: stack,
id: id,
pkt: pkt,
}
}
// ID returns the 4-tuple (src address, src port, dst address, dst port) that
// represents the session request.
func (r *ForwarderRequest) ID() stack.TransportEndpointID {
@ -106,7 +115,3 @@ func (r *ForwarderRequest) CreateEndpoint(queue *waiter.Queue) (tcpip.Endpoint,
return ep, nil
}
func (r *ForwarderRequest) Packet() *stack.PacketBuffer {
return r.pkt
}

4
pkg/tcpip/transport/udp/udp_state_autogen.go
View file

@ -31,8 +31,8 @@ func (p *udpPacket) beforeSave() {}
// +checklocksignore
func (p *udpPacket) StateSave(stateSinkObject state.Sink) {
p.beforeSave()
var receivedAtValue int64
receivedAtValue = p.saveReceivedAt()
receivedAtValue := p.saveReceivedAt()
_ = (int64)(receivedAtValue)
stateSinkObject.SaveValue(6, receivedAtValue)
stateSinkObject.Save(0, &p.udpPacketEntry)
stateSinkObject.Save(1, &p.netProto)