eburet-oss/gvisor-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

snapshot: sagernet/gvisor v0.0.0-20250811.0-sing-box-mod.1

Browse source 
Содержимое пина, зафиксированного в go.mod sing-box-lx, одним коммитом
без истории. Полная история SagerNet/gvisor — 1.45 ГБ и клонируется в
каждой CI-джобе; наша дельта — одна вставка в одну функцию, история для
неё не нужна.

Module path github.com/sagernet/gvisor сохранён намеренно: на него
опирается replace-директива суперпроекта.

Патч поверх — отдельным коммитом, чтобы дельта читалась одним git show
и переносилась на новый пин копированием.

SPECS/TASKS/048-GVISOR_HANDSHAKE_NIL_CRASH
This commit is contained in:
Leadaxe 2026-08-04 15:50:08 +03:00
commit 2c4ae3b0a4
712 changed files with 185689 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

724
pkg/tcpip/transport/tcp/accept.go Normal file
View file

@ -0,0 +1,724 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"container/list"
"crypto/sha1"
"encoding/binary"
"fmt"
"hash"
"io"
"time"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/ports"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/waiter"
)
const (
// tsLen is the length, in bits, of the timestamp in the SYN cookie.
tsLen = 8
// tsMask is a mask for timestamp values (i.e., tsLen bits).
tsMask = (1 << tsLen) - 1
// tsOffset is the offset, in bits, of the timestamp in the SYN cookie.
tsOffset = 24
// hashMask is the mask for hash values (i.e., tsOffset bits).
hashMask = (1 << tsOffset) - 1
// maxTSDiff is the maximum allowed difference between a received cookie
// timestamp and the current timestamp. If the difference is greater
// than maxTSDiff, the cookie is expired.
maxTSDiff = 2
)
// mssTable is a slice containing the possible MSS values that we
// encode in the SYN cookie with two bits.
var mssTable = []uint16{536, 1300, 1440, 1460}
func encodeMSS(mss uint16) uint32 {
for i := len(mssTable) - 1; i > 0; i-- {
if mss >= mssTable[i] {
return uint32(i)
}
}
return 0
}
// listenContext is used by a listening endpoint to store state used while
// listening for connections. This struct is allocated by the listen goroutine
// and must not be accessed or have its methods called concurrently as they
// may mutate the stored objects.
type listenContext struct {
stack *stack.Stack
protocol *protocol
// rcvWnd is the receive window that is sent by this listening context
// in the initial SYN-ACK.
rcvWnd seqnum.Size
// nonce are random bytes that are initialized once when the context
// is created and used to seed the hash function when generating
// the SYN cookie.
nonce [2][sha1.BlockSize]byte
// listenEP is a reference to the listening endpoint associated with
// this context. Can be nil if the context is created by the forwarder.
listenEP *Endpoint
// hasherMu protects hasher.
hasherMu hasherMutex
// hasher is the hash function used to generate a SYN cookie.
hasher hash.Hash
// v6Only is true if listenEP is a dual stack socket and has the
// IPV6_V6ONLY option set.
v6Only bool
// netProto indicates the network protocol(IPv4/v6) for the listening
// endpoint.
netProto tcpip.NetworkProtocolNumber
}
// timeStamp returns an 8-bit timestamp with a granularity of 64 seconds.
func timeStamp(clock tcpip.Clock) uint32 {
return uint32(clock.NowMonotonic().Sub(tcpip.MonotonicTime{}).Seconds()) >> 6 & tsMask
}
// newListenContext creates a new listen context.
func newListenContext(stk *stack.Stack, protocol *protocol, listenEP *Endpoint, rcvWnd seqnum.Size, v6Only bool, netProto tcpip.NetworkProtocolNumber) *listenContext {
l := &listenContext{
stack: stk,
protocol: protocol,
rcvWnd: rcvWnd,
hasher: sha1.New(),
v6Only: v6Only,
netProto: netProto,
listenEP: listenEP,
}
for i := range l.nonce {
if _, err := io.ReadFull(stk.SecureRNG().Reader, l.nonce[i][:]); err != nil {
panic(err)
}
}
return l
}
// cookieHash calculates the cookieHash for the given id, timestamp and nonce
// index. The hash is used to create and validate cookies.
func (l *listenContext) cookieHash(id stack.TransportEndpointID, ts uint32, nonceIndex int) uint32 {
// Initialize block with fixed-size data: local ports and v.
var payload [8]byte
binary.BigEndian.PutUint16(payload[0:], id.LocalPort)
binary.BigEndian.PutUint16(payload[2:], id.RemotePort)
binary.BigEndian.PutUint32(payload[4:], ts)
// Feed everything to the hasher.
l.hasherMu.Lock()
l.hasher.Reset()
// Per hash.Hash.Writer:
//
// It never returns an error.
l.hasher.Write(payload[:])
l.hasher.Write(l.nonce[nonceIndex][:])
l.hasher.Write(id.LocalAddress.AsSlice())
l.hasher.Write(id.RemoteAddress.AsSlice())
// Finalize the calculation of the hash and return the first 4 bytes.
h := l.hasher.Sum(nil)
l.hasherMu.Unlock()
return binary.BigEndian.Uint32(h[:])
}
// createCookie creates a SYN cookie for the given id and incoming sequence
// number.
func (l *listenContext) createCookie(id stack.TransportEndpointID, seq seqnum.Value, data uint32) seqnum.Value {
ts := timeStamp(l.stack.Clock())
v := l.cookieHash(id, 0, 0) + uint32(seq) + (ts << tsOffset)
v += (l.cookieHash(id, ts, 1) + data) & hashMask
return seqnum.Value(v)
}
// isCookieValid checks if the supplied cookie is valid for the given id and
// sequence number. If it is, it also returns the data originally encoded in the
// cookie when createCookie was called.
func (l *listenContext) isCookieValid(id stack.TransportEndpointID, cookie seqnum.Value, seq seqnum.Value) (uint32, bool) {
ts := timeStamp(l.stack.Clock())
v := uint32(cookie) - l.cookieHash(id, 0, 0) - uint32(seq)
cookieTS := v >> tsOffset
if ((ts - cookieTS) & tsMask) > maxTSDiff {
return 0, false
}
return (v - l.cookieHash(id, cookieTS, 1)) & hashMask, true
}
// createConnectingEndpoint creates a new endpoint in a connecting state, with
// the connection parameters given by the arguments. The newly created endpoint
// will be locked.
// +checklocksacquire:n.mu
func (l *listenContext) createConnectingEndpoint(s *segment, rcvdSynOpts header.TCPSynOptions, queue *waiter.Queue) (n *Endpoint, _ tcpip.Error) {
// Create a new endpoint.
netProto := l.netProto
if netProto == 0 {
netProto = s.pkt.NetworkProtocolNumber
}
route, err := l.stack.FindRoute(s.pkt.NICID, s.pkt.Network().DestinationAddress(), s.pkt.Network().SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */)
if err != nil {
return nil, err // +checklocksignore
}
n = newEndpoint(l.stack, l.protocol, netProto, queue)
n.mu.Lock()
n.ops.SetV6Only(l.v6Only)
n.TransportEndpointInfo.ID = s.id
n.boundNICID = s.pkt.NICID
n.route = route
n.effectiveNetProtos = []tcpip.NetworkProtocolNumber{s.pkt.NetworkProtocolNumber}
n.ops.SetReceiveBufferSize(int64(l.rcvWnd), false /* notify */)
n.amss = calculateAdvertisedMSS(n.userMSS, n.route)
n.setEndpointState(StateConnecting)
n.maybeEnableTimestamp(rcvdSynOpts)
n.maybeEnableSACKPermitted(rcvdSynOpts)
n.initGSO()
// Bootstrap the auto tuning algorithm. Starting at zero will result in
// a large step function on the first window adjustment causing the
// window to grow to a really large value.
initWnd := n.initialReceiveWindow()
n.rcvQueueMu.Lock()
n.RcvAutoParams.PrevCopiedBytes = initWnd
n.rcvQueueMu.Unlock()
return n, nil
}
// startHandshake creates a new endpoint in connecting state and then sends
// the SYN-ACK for the TCP 3-way handshake. It returns the state of the
// handshake in progress, which includes the new endpoint in the SYN-RCVD
// state.
//
// On success, a handshake h is returned.
//
// NOTE: h.ep.mu is not held and must be acquired if any state needs to be
// modified.
//
// Precondition: if l.listenEP != nil, l.listenEP.mu must be locked.
func (l *listenContext) startHandshake(s *segment, opts header.TCPSynOptions, queue *waiter.Queue, owner tcpip.PacketOwner) (h *handshake, _ tcpip.Error) {
// Create new endpoint.
irs := s.sequenceNumber
isn := generateSecureISN(s.id, l.stack.Clock(), l.protocol.seqnumSecret)
ep, err := l.createConnectingEndpoint(s, opts, queue)
if err != nil {
return nil, err // +checklocksignore
}
ep.owner = owner
// listenEP is nil when listenContext is used by tcp.Forwarder.
deferAccept := time.Duration(0)
if l.listenEP != nil {
if l.listenEP.EndpointState() != StateListen {
// Ensure we release any registrations done by the newly
// created endpoint.
ep.mu.Unlock()
ep.Close()
return nil, &tcpip.ErrConnectionAborted{} // +checklocksignore
}
// Propagate any inheritable options from the listening endpoint
// to the newly created endpoint.
l.listenEP.propagateInheritableOptionsLocked(ep) // +checklocksforce
if !ep.reserveTupleLocked() {
ep.mu.Unlock()
ep.Close()
return nil, &tcpip.ErrConnectionAborted{} // +checklocksignore
}
deferAccept = l.listenEP.deferAccept
}
// Register new endpoint so that packets are routed to it.
if err := ep.stack.RegisterTransportEndpoint(
ep.effectiveNetProtos,
ProtocolNumber,
ep.TransportEndpointInfo.ID,
ep,
ep.boundPortFlags,
ep.boundBindToDevice,
); err != nil {
ep.mu.Unlock()
ep.Close()
ep.drainClosingSegmentQueue()
return nil, err // +checklocksignore
}
ep.isRegistered = true
// Initialize and start the handshake.
h = ep.newPassiveHandshake(isn, irs, opts, deferAccept)
h.listenEP = l.listenEP
h.start()
h.ep.mu.Unlock()
return h, nil
}
// performHandshake performs a TCP 3-way handshake. On success, the new
// established endpoint is returned.
//
// Precondition: if l.listenEP != nil, l.listenEP.mu must be locked.
func (l *listenContext) performHandshake(s *segment, opts header.TCPSynOptions, queue *waiter.Queue, owner tcpip.PacketOwner) (*Endpoint, tcpip.Error) {
waitEntry, notifyCh := waiter.NewChannelEntry(waiter.WritableEvents)
queue.EventRegister(&waitEntry)
defer queue.EventUnregister(&waitEntry)
h, err := l.startHandshake(s, opts, queue, owner)
if err != nil {
return nil, err
}
// performHandshake is used by the Forwarder which will block till the
// handshake either succeeds or fails. We do this by registering for
// events above and block on the notification channel.
<-notifyCh
ep := h.ep
ep.mu.Lock()
if !ep.EndpointState().connected() {
ep.stack.Stats().TCP.FailedConnectionAttempts.Increment()
ep.stats.FailedConnectionAttempts.Increment()
ep.h = nil
ep.mu.Unlock()
ep.Close()
ep.notifyAborted()
ep.drainClosingSegmentQueue()
err := ep.LastError()
if err == nil {
// If err was nil then return the best error we can to indicate
// a connection failure.
err = &tcpip.ErrConnectionAborted{}
}
return nil, err
}
ep.isConnectNotified = true
// Transfer any state from the completed handshake to the endpoint.
//
// Update the receive window scaling. We can't do it before the
// handshake because it's possible that the peer doesn't support window
// scaling.
ep.rcv.RcvWndScale = ep.h.effectiveRcvWndScale()
// Clean up handshake state stored in the endpoint so that it can be
// GCed.
ep.h = nil
ep.mu.Unlock()
return ep, nil
}
// propagateInheritableOptionsLocked propagates any options set on the listening
// endpoint to the newly created endpoint.
//
// +checklocks:e.mu
// +checklocks:n.mu
func (e *Endpoint) propagateInheritableOptionsLocked(n *Endpoint) {
n.userTimeout = e.userTimeout
n.portFlags = e.portFlags
n.boundBindToDevice = e.boundBindToDevice
n.boundPortFlags = e.boundPortFlags
n.userMSS = e.userMSS
}
// reserveTupleLocked reserves an accepted endpoint's tuple.
//
// Precondition: e.propagateInheritableOptionsLocked has been called.
//
// +checklocks:e.mu
func (e *Endpoint) reserveTupleLocked() bool {
dest := tcpip.FullAddress{
Addr: e.TransportEndpointInfo.ID.RemoteAddress,
Port: e.TransportEndpointInfo.ID.RemotePort,
}
portRes := ports.Reservation{
Networks: e.effectiveNetProtos,
Transport: ProtocolNumber,
Addr: e.TransportEndpointInfo.ID.LocalAddress,
Port: e.TransportEndpointInfo.ID.LocalPort,
Flags: e.boundPortFlags,
BindToDevice: e.boundBindToDevice,
Dest: dest,
}
if !e.stack.ReserveTuple(portRes) {
e.stack.Stats().TCP.FailedPortReservations.Increment()
return false
}
e.isPortReserved = true
e.boundDest = dest
return true
}
// notifyAborted wakes up any waiters on registered, but not accepted
// endpoints.
//
// This is strictly not required normally as a socket that was never accepted
// can't really have any registered waiters except when stack.Wait() is called
// which waits for all registered endpoints to stop and expects an EventHUp.
func (e *Endpoint) notifyAborted() {
e.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents)
}
func (e *Endpoint) acceptQueueIsFull() bool {
e.acceptMu.Lock()
full := e.acceptQueue.isFull()
e.acceptMu.Unlock()
return full
}
// +stateify savable
type acceptQueue struct {
// NB: this could be an endpointList, but ilist only permits endpoints to
// belong to one list at a time, and endpoints are already stored in the
// dispatcher's list.
endpoints list.List `state:".([]*Endpoint)"`
// pendingEndpoints is a set of all endpoints for which a handshake is
// in progress.
pendingEndpoints map[*Endpoint]struct{}
// capacity is the maximum number of endpoints that can be in endpoints.
capacity int
}
func (a *acceptQueue) isFull() bool {
return a.endpoints.Len() >= a.capacity
}
// handleListenSegment is called when a listening endpoint receives a segment
// and needs to handle it.
//
// +checklocks:e.mu
func (e *Endpoint) handleListenSegment(ctx *listenContext, s *segment) tcpip.Error {
e.rcvQueueMu.Lock()
rcvClosed := e.RcvClosed
e.rcvQueueMu.Unlock()
if rcvClosed || s.flags.Contains(header.TCPFlagSyn|header.TCPFlagAck) {
// If the endpoint is shutdown, reply with reset.
//
// RFC 793 section 3.4 page 35 (figure 12) outlines that a RST
// must be sent in response to a SYN-ACK while in the listen
// state to prevent completing a handshake from an old SYN.
return replyWithReset(e.stack, s, e.sendTOS, e.ipv4TTL, e.ipv6HopLimit)
}
switch {
case s.flags.Contains(header.TCPFlagRst):
e.stack.Stats().DroppedPackets.Increment()
return nil
case s.flags.Contains(header.TCPFlagSyn):
if e.acceptQueueIsFull() {
e.stack.Stats().TCP.ListenOverflowSynDrop.Increment()
e.stats.ReceiveErrors.ListenOverflowSynDrop.Increment()
e.stack.Stats().DroppedPackets.Increment()
return nil
}
opts := parseSynSegmentOptions(s)
useSynCookies, err := func() (bool, tcpip.Error) {
var alwaysUseSynCookies tcpip.TCPAlwaysUseSynCookies
if err := e.stack.TransportProtocolOption(header.TCPProtocolNumber, &alwaysUseSynCookies); err != nil {
panic(fmt.Sprintf("TransportProtocolOption(%d, %T) = %s", header.TCPProtocolNumber, alwaysUseSynCookies, err))
}
if alwaysUseSynCookies {
return true, nil
}
e.acceptMu.Lock()
defer e.acceptMu.Unlock()
// The capacity of the accepted queue would always be one greater than the
// listen backlog. But, the SYNRCVD connections count is always checked
// against the listen backlog value for Linux parity reason.
// https://github.com/torvalds/linux/blob/7acac4b3196/include/net/inet_connection_sock.h#L280
if len(e.acceptQueue.pendingEndpoints) == e.acceptQueue.capacity-1 {
return true, nil
}
h, err := ctx.startHandshake(s, opts, &waiter.Queue{}, e.owner)
if err != nil {
e.stack.Stats().TCP.FailedConnectionAttempts.Increment()
e.stats.FailedConnectionAttempts.Increment()
return false, err
}
e.acceptQueue.pendingEndpoints[h.ep] = struct{}{}
return false, nil
}()
if err != nil {
return err
}
if !useSynCookies {
return nil
}
net := s.pkt.Network()
route, err := e.stack.FindRoute(s.pkt.NICID, net.DestinationAddress(), net.SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */)
if err != nil {
return err
}
defer route.Release()
// Send SYN without window scaling because we currently
// don't encode this information in the cookie.
//
// Enable Timestamp option if the original syn did have
// the timestamp option specified.
//
// Use the user supplied MSS on the listening socket for
// new connections, if available.
synOpts := header.TCPSynOptions{
WS: -1,
TS: opts.TS,
TSEcr: opts.TSVal,
MSS: calculateAdvertisedMSS(e.userMSS, route),
}
if opts.TS {
offset := e.protocol.tsOffset(net.DestinationAddress(), net.SourceAddress())
now := e.stack.Clock().NowMonotonic()
synOpts.TSVal = offset.TSVal(now)
}
cookie := ctx.createCookie(s.id, s.sequenceNumber, encodeMSS(opts.MSS))
fields := tcpFields{
id: s.id,
ttl: calculateTTL(route, e.ipv4TTL, e.ipv6HopLimit),
tos: e.sendTOS,
flags: header.TCPFlagSyn | header.TCPFlagAck,
seq: cookie,
ack: s.sequenceNumber + 1,
rcvWnd: ctx.rcvWnd,
expOptVal: e.getExperimentOptionValue(route),
}
if err := e.sendSynTCP(route, fields, synOpts); err != nil {
return err
}
e.stack.Stats().TCP.ListenOverflowSynCookieSent.Increment()
return nil
case s.flags.Contains(header.TCPFlagAck):
iss := s.ackNumber - 1
irs := s.sequenceNumber - 1
// As an edge case when SYN-COOKIES are in use and we receive a
// segment that has data and is valid we should check if it
// already matches a created endpoint and redirect the segment
// rather than try and create a new endpoint. This can happen
// where the final ACK for the handshake and other data packets
// arrive at the same time and are queued to the listening
// endpoint before the listening endpoint has had time to
// process the first ACK and create the endpoint that matches
// the incoming packet's full 5 tuple.
netProtos := []tcpip.NetworkProtocolNumber{s.pkt.NetworkProtocolNumber}
// If the local address is an IPv4 Address then also look for IPv6
// dual stack endpoints.
if s.id.LocalAddress.To4() != (tcpip.Address{}) {
netProtos = []tcpip.NetworkProtocolNumber{header.IPv4ProtocolNumber, header.IPv6ProtocolNumber}
}
for _, netProto := range netProtos {
if newEP := e.stack.FindTransportEndpoint(netProto, ProtocolNumber, s.id, s.pkt.NICID); newEP != nil && newEP != e {
tcpEP := newEP.(*Endpoint)
if !tcpEP.EndpointState().connected() {
continue
}
if !tcpEP.enqueueSegment(s) {
// Just silently drop the segment as we failed
// to queue, we don't want to generate a RST
// further below or try and create a new
// endpoint etc.
return nil
}
tcpEP.notifyProcessor()
return nil
}
}
// Since SYN cookies are in use this is potentially an ACK to a
// SYN-ACK we sent but don't have a half open connection state
// as cookies are being used to protect against a potential SYN
// flood. In such cases validate the cookie and if valid create
// a fully connected endpoint and deliver to the accept queue.
//
// If not, silently drop the ACK to avoid leaking information
// when under a potential syn flood attack.
//
// Validate the cookie.
data, ok := ctx.isCookieValid(s.id, iss, irs)
if !ok || int(data) >= len(mssTable) {
e.stack.Stats().TCP.ListenOverflowInvalidSynCookieRcvd.Increment()
e.stack.Stats().DroppedPackets.Increment()
// When not using SYN cookies, as per RFC 793, section 3.9, page 64:
// Any acknowledgment is bad if it arrives on a connection still in
// the LISTEN state. An acceptable reset segment should be formed
// for any arriving ACK-bearing segment. The RST should be
// formatted as follows:
//
// <SEQ=SEG.ACK><CTL=RST>
//
// Send a reset as this is an ACK for which there is no
// half open connections and we are not using cookies
// yet.
//
// The only time we should reach here when a connection
// was opened and closed really quickly and a delayed
// ACK was received from the sender.
return replyWithReset(e.stack, s, e.sendTOS, e.ipv4TTL, e.ipv6HopLimit)
}
// Keep hold of acceptMu until the new endpoint is in the accept queue (or
// if there is an error), to guarantee that we will keep our spot in the
// queue even if another handshake from the syn queue completes.
e.acceptMu.Lock()
if e.acceptQueue.isFull() {
// Silently drop the ack as the application can't accept
// the connection at this point. The ack will be
// retransmitted by the sender anyway and we can
// complete the connection at the time of retransmit if
// the backlog has space.
e.acceptMu.Unlock()
e.stack.Stats().TCP.ListenOverflowAckDrop.Increment()
e.stats.ReceiveErrors.ListenOverflowAckDrop.Increment()
e.stack.Stats().DroppedPackets.Increment()
return nil
}
e.stack.Stats().TCP.ListenOverflowSynCookieRcvd.Increment()
// Create newly accepted endpoint and deliver it.
rcvdSynOptions := header.TCPSynOptions{
MSS: mssTable[data],
// Disable Window scaling as original SYN is
// lost.
WS: -1,
}
// When syn cookies are in use we enable timestamp only
// if the ack specifies the timestamp option assuming
// that the other end did in fact negotiate the
// timestamp option in the original SYN.
if s.parsedOptions.TS {
rcvdSynOptions.TS = true
rcvdSynOptions.TSVal = s.parsedOptions.TSVal
rcvdSynOptions.TSEcr = s.parsedOptions.TSEcr
}
n, err := ctx.createConnectingEndpoint(s, rcvdSynOptions, &waiter.Queue{})
if err != nil {
e.acceptMu.Unlock()
return err
}
// Propagate any inheritable options from the listening endpoint
// to the newly created endpoint.
e.propagateInheritableOptionsLocked(n)
if !n.reserveTupleLocked() {
n.mu.Unlock()
e.acceptMu.Unlock()
n.Close()
e.stack.Stats().TCP.FailedConnectionAttempts.Increment()
e.stats.FailedConnectionAttempts.Increment()
return nil
}
// Register new endpoint so that packets are routed to it.
if err := n.stack.RegisterTransportEndpoint(
n.effectiveNetProtos,
ProtocolNumber,
n.TransportEndpointInfo.ID,
n,
n.boundPortFlags,
n.boundBindToDevice,
); err != nil {
n.mu.Unlock()
e.acceptMu.Unlock()
n.Close()
e.stack.Stats().TCP.FailedConnectionAttempts.Increment()
e.stats.FailedConnectionAttempts.Increment()
return err
}
n.isRegistered = true
net := s.pkt.Network()
n.TSOffset = n.protocol.tsOffset(net.DestinationAddress(), net.SourceAddress())
// Switch state to connected.
n.isConnectNotified = true
h := handshake{
ep: n,
iss: iss,
ackNum: irs + 1,
rcvWnd: seqnum.Size(n.initialReceiveWindow()),
sndWnd: s.window,
rcvWndScale: e.rcvWndScaleForHandshake(),
sndWndScale: rcvdSynOptions.WS,
mss: rcvdSynOptions.MSS,
sampleRTTWithTSOnly: true,
}
h.ep.AssertLockHeld(n)
h.transitionToStateEstablishedLocked(s)
n.mu.Unlock()
// Requeue the segment if the ACK completing the handshake has more info
// to be processed by the newly established endpoint.
if (s.flags.Contains(header.TCPFlagFin) || s.payloadSize() > 0) && n.enqueueSegment(s) {
n.notifyProcessor()
}
e.stack.Stats().TCP.PassiveConnectionOpenings.Increment()
// Deliver the endpoint to the accept queue.
e.acceptQueue.endpoints.PushBack(n)
e.acceptMu.Unlock()
e.waiterQueue.Notify(waiter.ReadableEvents)
return nil
default:
e.stack.Stats().DroppedPackets.Increment()
return nil
}
}

64
pkg/tcpip/transport/tcp/accept_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type acceptMutex struct {
mu sync.Mutex
}
var acceptprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var acceptlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type acceptlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *acceptMutex) Lock() {
locking.AddGLock(acceptprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *acceptMutex) NestedLock(i acceptlockNameIndex) {
locking.AddGLock(acceptprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *acceptMutex) Unlock() {
locking.DelGLock(acceptprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *acceptMutex) NestedUnlock(i acceptlockNameIndex) {
locking.DelGLock(acceptprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func acceptinitLockNames() {}
func init() {
acceptinitLockNames()
acceptprefixIndex = locking.NewMutexClass(reflect.TypeOf(acceptMutex{}), acceptlockNames)
}

1532
pkg/tcpip/transport/tcp/connect.go Normal file
View file

File diff suppressed because it is too large Load diff

30
pkg/tcpip/transport/tcp/connect_unsafe.go Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"reflect"
"unsafe"
)
// optionsToArray converts a slice of capacity >-= maxOptionSize to an array.
//
// optionsToArray panics if the capacity of options is smaller than
// maxOptionSize.
func optionsToArray(options []byte) *[maxOptionSize]byte {
// Reslice to full capacity.
options = options[0:maxOptionSize]
return (*[maxOptionSize]byte)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&options)).Data))
}

318
pkg/tcpip/transport/tcp/cubic.go Normal file
View file

@ -0,0 +1,318 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"math"
"time"
"github.com/sagernet/gvisor/pkg/tcpip"
)
// effectivelyInfinity is an initialization value used for round-trip times
// that are then set using min. It is equal to approximately 100 years: large
// enough that it will always be greater than a real TCP round-trip time, and
// small enough that it fits in time.Duration.
const effectivelyInfinity = time.Duration(math.MaxInt64)
const (
// RTT = round-trip time.
// The delay increase sensitivity is determined by minRTTThresh and
// maxRTTThresh. Smaller values of minRTTThresh may cause spurious exits
// from slow start. Larger values of maxRTTThresh may result in slow start
// not exiting until loss is encountered for connections on large RTT paths.
minRTTThresh = 4 * time.Millisecond
maxRTTThresh = 16 * time.Millisecond
// minRTTDivisor is a fraction of RTT to compute the delay threshold. A
// smaller value would mean a larger threshold and thus less sensitivity to
// delay increase, and vice versa.
minRTTDivisor = 8
// nRTTSample is the minimum number of RTT samples in the round before
// considering whether to exit the round due to increased RTT.
nRTTSample = 8
// ackDelta is the maximum time between ACKs for them to be considered part
// of the same ACK Train during HyStart
ackDelta = 2 * time.Millisecond
)
// cubicState stores the variables related to TCP CUBIC congestion
// control algorithm state.
//
// See: https://tools.ietf.org/html/rfc8312.
// +stateify savable
type cubicState struct {
TCPCubicState
// numCongestionEvents tracks the number of congestion events since last
// RTO.
numCongestionEvents int
s *sender
}
// newCubicCC returns a partially initialized cubic state with the constants
// beta and c set and t set to current time.
//
// +checklocks:s.ep.mu
func newCubicCC(s *sender) *cubicState {
now := s.ep.stack.Clock().NowMonotonic()
return &cubicState{
TCPCubicState: TCPCubicState{
T: now,
Beta: 0.7,
C: 0.4,
// By this point, the sender has initialized it's initial sequence
// number.
EndSeq: s.SndNxt,
LastRTT: effectivelyInfinity,
CurrRTT: effectivelyInfinity,
LastAck: now,
RoundStart: now,
},
s: s,
}
}
// enterCongestionAvoidance is used to initialize cubic in cases where we exit
// SlowStart without a real congestion event taking place. This can happen when
// a connection goes back to slow start due to a retransmit and we exceed the
// previously lowered ssThresh without experiencing packet loss.
//
// Refer: https://tools.ietf.org/html/rfc8312#section-4.8
//
// +checklocks:c.s.ep.mu
func (c *cubicState) enterCongestionAvoidance() {
// See: https://tools.ietf.org/html/rfc8312#section-4.7 &
// https://tools.ietf.org/html/rfc8312#section-4.8
if c.numCongestionEvents == 0 {
c.K = 0
c.T = c.s.ep.stack.Clock().NowMonotonic()
c.WLastMax = c.WMax
c.WMax = float64(c.s.SndCwnd)
}
}
// updateHyStart tracks packet round-trip time (rtt) to find a safe threshold
// to exit slow start without triggering packet loss. It updates the SSThresh
// when it does.
//
// Implementation of HyStart follows the algorithm from the Linux kernel, rather
// than RFC 9406 (https://www.rfc-editor.org/rfc/rfc9406.html). Briefly, the
// Linux kernel algorithm is based directly on the original HyStart paper
// (https://doi.org/10.1016/j.comnet.2011.01.014), and differs from the RFC in
// that two detection algorithms run in parallel ('ACK train' and 'Delay
// increase'). The RFC version includes only the latter algorithm and adds an
// intermediate phase called Conservative Slow Start, which is not implemented
// here.
//
// +checklocks:c.s.ep.mu
func (c *cubicState) updateHyStart(rtt time.Duration) {
if rtt < 0 {
// negative indicates unknown
return
}
now := c.s.ep.stack.Clock().NowMonotonic()
if c.EndSeq.LessThan(c.s.SndUna) {
c.beginHyStartRound(now)
}
// ACK train
if now.Sub(c.LastAck) < ackDelta && // ensures acks are part of the same "train"
c.LastRTT < effectivelyInfinity {
c.LastAck = now
if thresh := c.LastRTT / 2; now.Sub(c.RoundStart) > thresh {
c.s.Ssthresh = c.s.SndCwnd
}
}
// Delay increase
c.CurrRTT = min(c.CurrRTT, rtt)
c.SampleCount++
if c.SampleCount >= nRTTSample && c.LastRTT < effectivelyInfinity {
// i.e. LastRTT/minRTTDivisor, but clamped to minRTTThresh & maxRTTThresh
thresh := max(
minRTTThresh,
min(maxRTTThresh, c.LastRTT/minRTTDivisor),
)
if c.CurrRTT >= (c.LastRTT + thresh) {
// Triggered HyStart safe exit threshold
c.s.Ssthresh = c.s.SndCwnd
}
}
}
// +checklocks:c.s.ep.mu
func (c *cubicState) beginHyStartRound(now tcpip.MonotonicTime) {
c.EndSeq = c.s.SndNxt
c.SampleCount = 0
c.LastRTT = c.CurrRTT
c.CurrRTT = effectivelyInfinity
c.LastAck = now
c.RoundStart = now
}
// updateSlowStart will update the congestion window as per the slow-start
// algorithm used by NewReno. If after adjusting the congestion window we cross
// the ssThresh then it will return the number of packets that must be consumed
// in congestion avoidance mode.
//
// +checklocks:c.s.ep.mu
func (c *cubicState) updateSlowStart(packetsAcked int) int {
// Don't let the congestion window cross into the congestion
// avoidance range.
newcwnd := c.s.SndCwnd + packetsAcked
enterCA := false
if newcwnd >= c.s.Ssthresh {
newcwnd = c.s.Ssthresh
c.s.SndCAAckCount = 0
enterCA = true
}
packetsAcked -= newcwnd - c.s.SndCwnd
c.s.SndCwnd = newcwnd
if enterCA {
c.enterCongestionAvoidance()
}
return packetsAcked
}
// Update updates cubic's internal state variables. It must be called on every
// ACK received.
// Refer: https://tools.ietf.org/html/rfc8312#section-4
//
// +checklocks:c.s.ep.mu
func (c *cubicState) Update(packetsAcked int, rtt time.Duration) {
if c.s.Ssthresh == InitialSsthresh && c.s.SndCwnd < c.s.Ssthresh {
c.updateHyStart(rtt)
}
if c.s.SndCwnd < c.s.Ssthresh {
packetsAcked = c.updateSlowStart(packetsAcked)
if packetsAcked == 0 {
return
}
} else {
c.s.rtt.Lock()
srtt := c.s.rtt.TCPRTTState.SRTT
c.s.rtt.Unlock()
c.s.SndCwnd = c.getCwnd(packetsAcked, c.s.SndCwnd, srtt)
}
}
// cubicCwnd computes the CUBIC congestion window after t seconds from last
// congestion event.
func (c *cubicState) cubicCwnd(t float64) float64 {
return c.C*math.Pow(t, 3.0) + c.WMax
}
// getCwnd returns the current congestion window as computed by CUBIC.
// Refer: https://tools.ietf.org/html/rfc8312#section-4
func (c *cubicState) getCwnd(packetsAcked, sndCwnd int, srtt time.Duration) int {
elapsed := c.s.ep.stack.Clock().NowMonotonic().Sub(c.T)
elapsedSeconds := elapsed.Seconds()
// Compute the window as per Cubic after 'elapsed' time
// since last congestion event.
c.WC = c.cubicCwnd(elapsedSeconds - c.K)
// Compute the TCP friendly estimate of the congestion window.
c.WEst = c.WMax*c.Beta + (3.0*((1.0-c.Beta)/(1.0+c.Beta)))*(elapsedSeconds/srtt.Seconds())
// Make sure in the TCP friendly region CUBIC performs at least
// as well as Reno.
if c.WC < c.WEst && float64(sndCwnd) < c.WEst {
// TCP Friendly region of cubic.
return int(c.WEst)
}
// In Concave/Convex region of CUBIC, calculate what CUBIC window
// will be after 1 RTT and use that to grow congestion window
// for every ack.
tEst := (elapsed + srtt).Seconds()
wtRtt := c.cubicCwnd(tEst - c.K)
// As per 4.3 for each received ACK cwnd must be incremented
// by (w_cubic(t+RTT) - cwnd/cwnd.
cwnd := float64(sndCwnd)
for i := 0; i < packetsAcked; i++ {
// Concave/Convex regions of cubic have the same formulas.
// See: https://tools.ietf.org/html/rfc8312#section-4.3
cwnd += (wtRtt - cwnd) / cwnd
}
return int(cwnd)
}
// HandleLossDetected implements congestionControl.HandleLossDetected.
//
// +checklocks:c.s.ep.mu
func (c *cubicState) HandleLossDetected() {
// See: https://tools.ietf.org/html/rfc8312#section-4.5
c.numCongestionEvents++
c.T = c.s.ep.stack.Clock().NowMonotonic()
c.WLastMax = c.WMax
c.WMax = float64(c.s.SndCwnd)
c.fastConvergence()
c.reduceSlowStartThreshold()
}
// HandleRTOExpired implements congestionContrl.HandleRTOExpired.
//
// +checklocks:c.s.ep.mu
func (c *cubicState) HandleRTOExpired() {
// See: https://tools.ietf.org/html/rfc8312#section-4.6
c.T = c.s.ep.stack.Clock().NowMonotonic()
c.numCongestionEvents = 0
c.WLastMax = c.WMax
c.WMax = float64(c.s.SndCwnd)
c.fastConvergence()
// We lost a packet, so reduce ssthresh.
c.reduceSlowStartThreshold()
// Reduce the congestion window to 1, i.e., enter slow-start. Per
// RFC 5681, page 7, we must use 1 regardless of the value of the
// initial congestion window.
c.s.SndCwnd = 1
}
// fastConvergence implements the logic for Fast Convergence algorithm as
// described in https://tools.ietf.org/html/rfc8312#section-4.6.
func (c *cubicState) fastConvergence() {
if c.WMax < c.WLastMax {
c.WLastMax = c.WMax
c.WMax = c.WMax * (1.0 + c.Beta) / 2.0
} else {
c.WLastMax = c.WMax
}
// Recompute k as wMax may have changed.
c.K = math.Cbrt(c.WMax * (1 - c.Beta) / c.C)
}
// PostRecovery implements congestionControl.PostRecovery.
func (c *cubicState) PostRecovery() {
c.T = c.s.ep.stack.Clock().NowMonotonic()
}
// reduceSlowStartThreshold returns new SsThresh as described in
// https://tools.ietf.org/html/rfc8312#section-4.7.
//
// +checklocks:c.s.ep.mu
func (c *cubicState) reduceSlowStartThreshold() {
c.s.Ssthresh = int(math.Max(float64(c.s.SndCwnd)*c.Beta, 2.0))
}

534
pkg/tcpip/transport/tcp/dispatcher.go Normal file
View file

@ -0,0 +1,534 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"encoding/binary"
"fmt"
"math/rand"
"github.com/sagernet/gvisor/pkg/sleep"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/hash/jenkins"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/waiter"
)
// epQueue is a queue of endpoints.
//
// +stateify savable
type epQueue struct {
mu epQueueMutex `state:"nosave"`
list endpointList
}
// enqueue adds e to the queue if the endpoint is not already on the queue.
func (q *epQueue) enqueue(e *Endpoint) {
q.mu.Lock()
defer q.mu.Unlock()
e.pendingProcessingMu.Lock()
defer e.pendingProcessingMu.Unlock()
if e.pendingProcessing {
return
}
q.list.PushBack(e)
e.pendingProcessing = true
}
// dequeue removes and returns the first element from the queue if available,
// returns nil otherwise.
func (q *epQueue) dequeue() *Endpoint {
q.mu.Lock()
if e := q.list.Front(); e != nil {
q.list.Remove(e)
e.pendingProcessingMu.Lock()
e.pendingProcessing = false
e.pendingProcessingMu.Unlock()
q.mu.Unlock()
return e
}
q.mu.Unlock()
return nil
}
// empty returns true if the queue is empty, false otherwise.
func (q *epQueue) empty() bool {
q.mu.Lock()
v := q.list.Empty()
q.mu.Unlock()
return v
}
// processor is responsible for processing packets queued to a tcp endpoint.
//
// +stateify savable
type processor struct {
epQ epQueue
sleeper sleep.Sleeper `state:"nosave"`
newEndpointWaker sleep.Waker `state:"nosave"`
closeWaker sleep.Waker `state:"nosave"`
pauseWaker sleep.Waker `state:"nosave"`
pauseChan chan struct{} `state:"nosave"`
resumeChan chan struct{} `state:"nosave"`
}
func (p *processor) close() {
p.closeWaker.Assert()
}
func (p *processor) queueEndpoint(ep *Endpoint) {
// Queue an endpoint for processing by the processor goroutine.
p.epQ.enqueue(ep)
p.newEndpointWaker.Assert()
}
// deliverAccepted delivers a passively connected endpoint to the accept queue
// of its associated listening endpoint.
//
// +checklocks:ep.mu
func deliverAccepted(ep *Endpoint) bool {
lEP := ep.h.listenEP
lEP.acceptMu.Lock()
// Remove endpoint from list of pendingEndpoints as the handshake is now
// complete.
delete(lEP.acceptQueue.pendingEndpoints, ep)
// Deliver this endpoint to the listening socket's accept queue.
if lEP.acceptQueue.capacity == 0 {
lEP.acceptMu.Unlock()
return false
}
// NOTE: We always queue the endpoint and on purpose do not check if
// accept queue is full at this point. This is similar to linux because
// two racing incoming ACK's can both pass the acceptQueue.isFull check
// and proceed to ESTABLISHED state. In such a case its better to
// deliver both even if it temporarily exceeds the queue limit rather
// than drop a connection that is fully connected.
//
// For reference see:
// https://github.com/torvalds/linux/blob/169e77764adc041b1dacba84ea90516a895d43b2/net/ipv4/tcp_minisocks.c#L764
// https://github.com/torvalds/linux/blob/169e77764adc041b1dacba84ea90516a895d43b2/net/ipv4/tcp_ipv4.c#L1500
lEP.acceptQueue.endpoints.PushBack(ep)
lEP.acceptMu.Unlock()
ep.h.listenEP.waiterQueue.Notify(waiter.ReadableEvents)
return true
}
// handleConnecting is responsible for TCP processing for an endpoint in one of
// the connecting states.
func handleConnecting(ep *Endpoint) {
if !ep.TryLock() {
return
}
cleanup := func() {
ep.mu.Unlock()
ep.drainClosingSegmentQueue()
ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents)
}
if !ep.EndpointState().connecting() {
// If the endpoint has already transitioned out of a connecting
// stage then just return (only possible if it was closed or
// timed out by the time we got around to processing the wakeup.
ep.mu.Unlock()
return
}
if err := ep.h.processSegments(); err != nil { // +checklocksforce:ep.h.ep.mu
// handshake failed. clean up the tcp endpoint and handshake
// state.
if lEP := ep.h.listenEP; lEP != nil {
lEP.acceptMu.Lock()
delete(lEP.acceptQueue.pendingEndpoints, ep)
lEP.acceptMu.Unlock()
}
ep.handshakeFailed(err)
cleanup()
return
}
if ep.EndpointState() == StateEstablished && ep.h.listenEP != nil {
ep.isConnectNotified = true
ep.stack.Stats().TCP.PassiveConnectionOpenings.Increment()
if !deliverAccepted(ep) {
ep.resetConnectionLocked(&tcpip.ErrConnectionAborted{})
cleanup()
return
}
}
ep.mu.Unlock()
}
// handleConnected is responsible for TCP processing for an endpoint in one of
// the connected states(StateEstablished, StateFinWait1 etc.)
func handleConnected(ep *Endpoint) {
if !ep.TryLock() {
return
}
if !ep.EndpointState().connected() {
// If the endpoint has already transitioned out of a connected
// state then just return (only possible if it was closed or
// timed out by the time we got around to processing the wakeup.
ep.mu.Unlock()
return
}
// NOTE: We read this outside of e.mu lock which means that by the time
// we get to handleSegments the endpoint may not be in ESTABLISHED. But
// this should be fine as all normal shutdown states are handled by
// handleSegmentsLocked.
switch err := ep.handleSegmentsLocked(); {
case err != nil:
// Send any active resets if required.
ep.resetConnectionLocked(err)
fallthrough
case ep.EndpointState() == StateClose:
ep.mu.Unlock()
ep.drainClosingSegmentQueue()
ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents)
return
case ep.EndpointState() == StateTimeWait:
startTimeWait(ep)
}
ep.mu.Unlock()
}
// startTimeWait starts a new goroutine to handle TIME-WAIT.
//
// +checklocks:ep.mu
func startTimeWait(ep *Endpoint) {
// Disable close timer as we are now entering real TIME_WAIT.
if ep.finWait2Timer != nil {
ep.finWait2Timer.Stop()
}
// Wake up any waiters before we start TIME-WAIT.
ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents)
timeWaitDuration := ep.getTimeWaitDuration()
ep.timeWaitTimer = ep.stack.Clock().AfterFunc(timeWaitDuration, ep.timeWaitTimerExpired)
}
// handleTimeWait is responsible for TCP processing for an endpoint in TIME-WAIT
// state.
func handleTimeWait(ep *Endpoint) {
if !ep.TryLock() {
return
}
if ep.EndpointState() != StateTimeWait {
// If the endpoint has already transitioned out of a TIME-WAIT
// state then just return (only possible if it was closed or
// timed out by the time we got around to processing the wakeup.
ep.mu.Unlock()
return
}
extendTimeWait, reuseTW := ep.handleTimeWaitSegments()
if reuseTW != nil {
ep.transitionToStateCloseLocked()
ep.mu.Unlock()
ep.drainClosingSegmentQueue()
ep.waiterQueue.Notify(waiter.EventHUp | waiter.EventErr | waiter.ReadableEvents | waiter.WritableEvents)
reuseTW()
return
}
if extendTimeWait {
ep.timeWaitTimer.Reset(ep.getTimeWaitDuration())
}
ep.mu.Unlock()
}
// handleListen is responsible for TCP processing for an endpoint in LISTEN
// state.
func handleListen(ep *Endpoint) {
if !ep.TryLock() {
return
}
defer ep.mu.Unlock()
if ep.EndpointState() != StateListen {
// If the endpoint has already transitioned out of a LISTEN
// state then just return (only possible if it was closed or
// shutdown).
return
}
for i := 0; i < maxSegmentsPerWake; i++ {
s := ep.segmentQueue.dequeue()
if s == nil {
break
}
// TODO(gvisor.dev/issue/4690): Better handle errors instead of
// silently dropping.
_ = ep.handleListenSegment(ep.listenCtx, s)
s.DecRef()
}
}
// start runs the main loop for a processor which is responsible for all TCP
// processing for TCP endpoints.
func (p *processor) start(wg *sync.WaitGroup) {
defer wg.Done()
defer p.sleeper.Done()
for {
switch w := p.sleeper.Fetch(true); {
case w == &p.closeWaker:
return
case w == &p.pauseWaker:
if !p.epQ.empty() {
p.newEndpointWaker.Assert()
p.pauseWaker.Assert()
continue
} else {
p.pauseChan <- struct{}{}
<-p.resumeChan
}
case w == &p.newEndpointWaker:
for {
ep := p.epQ.dequeue()
if ep == nil {
break
}
if ep.segmentQueue.empty() {
continue
}
switch state := ep.EndpointState(); {
case state.connecting():
handleConnecting(ep)
case state.connected() && state != StateTimeWait:
handleConnected(ep)
case state == StateTimeWait:
handleTimeWait(ep)
case state == StateListen:
handleListen(ep)
case state == StateError || state == StateClose:
// Try to redeliver any still queued
// packets to another endpoint or send a
// RST if it can't be delivered.
ep.mu.Lock()
if st := ep.EndpointState(); st == StateError || st == StateClose {
ep.drainClosingSegmentQueue()
}
ep.mu.Unlock()
default:
panic(fmt.Sprintf("unexpected tcp state in processor: %v", state))
}
// If there are more segments to process and the
// endpoint lock is not held by user then
// requeue this endpoint for processing.
if !ep.segmentQueue.empty() && !ep.isOwnedByUser() {
p.epQ.enqueue(ep)
}
}
}
}
}
// pause pauses the processor loop.
func (p *processor) pause() chan struct{} {
p.pauseWaker.Assert()
return p.pauseChan
}
// resume resumes a previously paused loop.
//
// Precondition: Pause must have been called previously.
func (p *processor) resume() {
p.resumeChan <- struct{}{}
}
// dispatcher manages a pool of TCP endpoint processors which are responsible
// for the processing of inbound segments. This fixed pool of processor
// goroutines do full tcp processing. The processor is selected based on the
// hash of the endpoint id to ensure that delivery for the same endpoint happens
// in-order.
//
// +stateify savable
type dispatcher struct {
processors []processor
wg sync.WaitGroup `state:"nosave"`
hasher jenkinsHasher
mu dispatcherMutex `state:"nosave"`
// +checklocks:mu
paused bool
// +checklocks:mu
closed bool
}
// init initializes a dispatcher and starts the main loop for all the processors
// owned by this dispatcher.
func (d *dispatcher) init(rng *rand.Rand, nProcessors int) {
d.close()
d.wait()
d.mu.Lock()
defer d.mu.Unlock()
d.closed = false
d.processors = make([]processor, nProcessors)
d.hasher = jenkinsHasher{seed: rng.Uint32()}
d.startLocked()
}
// +checklocks:d.mu
func (d *dispatcher) startLocked() {
if d.closed {
return
}
for i := range d.processors {
p := &d.processors[i]
p.sleeper.AddWaker(&p.newEndpointWaker)
p.sleeper.AddWaker(&p.closeWaker)
p.sleeper.AddWaker(&p.pauseWaker)
p.pauseChan = make(chan struct{})
p.resumeChan = make(chan struct{})
d.wg.Add(1)
// NB: sleeper-waker registration must happen synchronously to avoid races
// with `close`. It's possible to pull all this logic into `start`, but
// that results in a heap-allocated function literal.
go p.start(&d.wg)
}
}
func (d *dispatcher) start() {
d.mu.Lock()
defer d.mu.Unlock()
d.startLocked()
}
// close closes a dispatcher and its processors.
func (d *dispatcher) close() {
d.mu.Lock()
d.closed = true
d.mu.Unlock()
for i := range d.processors {
d.processors[i].close()
}
}
// wait waits for all processor goroutines to end.
func (d *dispatcher) wait() {
d.wg.Wait()
}
// queuePacket queues an incoming packet to the matching tcp endpoint and
// also queues the endpoint to a processor queue for processing.
func (d *dispatcher) queuePacket(stackEP stack.TransportEndpoint, id stack.TransportEndpointID, clock tcpip.Clock, pkt *stack.PacketBuffer) {
d.mu.Lock()
closed := d.closed
d.mu.Unlock()
if closed {
return
}
ep := stackEP.(*Endpoint)
s, err := newIncomingSegment(id, clock, pkt)
if err != nil {
ep.stack.Stats().TCP.InvalidSegmentsReceived.Increment()
ep.stats.ReceiveErrors.MalformedPacketsReceived.Increment()
return
}
defer s.DecRef()
if !s.csumValid {
ep.stack.Stats().TCP.ChecksumErrors.Increment()
ep.stats.ReceiveErrors.ChecksumErrors.Increment()
return
}
ep.stack.Stats().TCP.ValidSegmentsReceived.Increment()
ep.stats.SegmentsReceived.Increment()
if (s.flags & header.TCPFlagRst) != 0 {
ep.stack.Stats().TCP.ResetsReceived.Increment()
}
if !ep.enqueueSegment(s) {
return
}
// Only wakeup the processor if endpoint lock is not held by a user
// goroutine as endpoint.UnlockUser will wake up the processor if the
// segment queue is not empty.
if !ep.isOwnedByUser() {
d.selectProcessor(id).queueEndpoint(ep)
}
}
// selectProcessor uses a hash of the transport endpoint ID to queue the
// endpoint to a specific processor. This is required to main TCP ordering as
// queueing the same endpoint to multiple processors can *potentially* result in
// out of order processing of incoming segments. It also ensures that a dispatcher
// evenly loads the processor goroutines.
func (d *dispatcher) selectProcessor(id stack.TransportEndpointID) *processor {
return &d.processors[d.hasher.hash(id)%uint32(len(d.processors))]
}
// pause pauses a dispatcher and all its processor goroutines.
func (d *dispatcher) pause() {
d.mu.Lock()
d.paused = true
d.mu.Unlock()
for i := range d.processors {
<-d.processors[i].pause()
}
}
// resume resumes a previously paused dispatcher and its processor goroutines.
// Calling resume on a dispatcher that was never paused is a no-op.
func (d *dispatcher) resume() {
d.mu.Lock()
if !d.paused {
// If this was a restore run the stack is a new instance and
// it was never paused, so just return as there is nothing to
// resume.
d.mu.Unlock()
return
}
d.paused = false
d.mu.Unlock()
for i := range d.processors {
d.processors[i].resume()
}
}
// jenkinsHasher contains state needed to for a jenkins hash.
//
// +stateify savable
type jenkinsHasher struct {
seed uint32
}
// hash hashes the provided TransportEndpointID using the jenkins hash
// algorithm.
func (j jenkinsHasher) hash(id stack.TransportEndpointID) uint32 {
var payload [4]byte
binary.LittleEndian.PutUint16(payload[0:], id.LocalPort)
binary.LittleEndian.PutUint16(payload[2:], id.RemotePort)
h := jenkins.Sum32(j.seed)
h.Write(payload[:])
h.Write(id.LocalAddress.AsSlice())
h.Write(id.RemoteAddress.AsSlice())
return h.Sum32()
}

64
pkg/tcpip/transport/tcp/dispatcher_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type dispatcherMutex struct {
mu sync.Mutex
}
var dispatcherprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var dispatcherlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type dispatcherlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *dispatcherMutex) Lock() {
locking.AddGLock(dispatcherprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *dispatcherMutex) NestedLock(i dispatcherlockNameIndex) {
locking.AddGLock(dispatcherprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *dispatcherMutex) Unlock() {
locking.DelGLock(dispatcherprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *dispatcherMutex) NestedUnlock(i dispatcherlockNameIndex) {
locking.DelGLock(dispatcherprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func dispatcherinitLockNames() {}
func init() {
dispatcherinitLockNames()
dispatcherprefixIndex = locking.NewMutexClass(reflect.TypeOf(dispatcherMutex{}), dispatcherlockNames)
}

3367
pkg/tcpip/transport/tcp/endpoint.go Normal file
View file

File diff suppressed because it is too large Load diff

347
pkg/tcpip/transport/tcp/endpoint_state.go Normal file
View file

@ -0,0 +1,347 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"context"
"fmt"
"github.com/sagernet/gvisor/pkg/atomicbitops"
"github.com/sagernet/gvisor/pkg/log"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/ports"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
)
// logDisconnectOnce ensures we don't spam logs when many connections are terminated.
var logDisconnectOnce sync.Once
func logDisconnect() {
logDisconnectOnce.Do(func() {
log.Infof("One or more TCP connections terminated during save")
})
}
// beforeSave is invoked by stateify.
func (e *Endpoint) beforeSave() {
// Stop incoming packets.
e.segmentQueue.freeze()
e.mu.Lock()
defer e.mu.Unlock()
epState := e.EndpointState()
switch {
case epState == StateInitial || epState == StateBound:
case epState.connected() || epState.handshake():
if !e.route.HasSaveRestoreCapability() {
if !e.route.HasDisconnectOkCapability() {
panic(&tcpip.ErrSaveRejection{
Err: fmt.Errorf("endpoint cannot be saved in connected state: local %s:%d, remote %s:%d", e.TransportEndpointInfo.ID.LocalAddress, e.TransportEndpointInfo.ID.LocalPort, e.TransportEndpointInfo.ID.RemoteAddress, e.TransportEndpointInfo.ID.RemotePort),
})
}
logDisconnect()
e.resetConnectionLocked(&tcpip.ErrConnectionAborted{})
e.mu.Unlock()
e.Close()
e.mu.Lock()
}
fallthrough
case epState == StateListen:
// Nothing to do.
case epState.closed():
// Nothing to do.
default:
panic(fmt.Sprintf("endpoint in unknown state %v", e.EndpointState()))
}
e.stack.RegisterResumableEndpoint(e)
}
// saveEndpoints is invoked by stateify.
func (a *acceptQueue) saveEndpoints() []*Endpoint {
acceptedEndpoints := make([]*Endpoint, a.endpoints.Len())
for i, e := 0, a.endpoints.Front(); e != nil; i, e = i+1, e.Next() {
acceptedEndpoints[i] = e.Value.(*Endpoint)
}
return acceptedEndpoints
}
// loadEndpoints is invoked by stateify.
func (a *acceptQueue) loadEndpoints(_ context.Context, acceptedEndpoints []*Endpoint) {
for _, ep := range acceptedEndpoints {
a.endpoints.PushBack(ep)
}
}
// saveState is invoked by stateify.
func (e *Endpoint) saveState() EndpointState {
return e.EndpointState()
}
// Endpoint loading must be done in the following ordering by their state, to
// avoid dangling connecting w/o listening peer, and to avoid conflicts in port
// reservation.
var (
connectedLoading sync.WaitGroup
listenLoading sync.WaitGroup
connectingLoading sync.WaitGroup
)
// Bound endpoint loading happens last.
// loadState is invoked by stateify.
func (e *Endpoint) loadState(_ context.Context, epState EndpointState) {
// This is to ensure that the loading wait groups include all applicable
// endpoints before any asynchronous calls to the Wait() methods.
// For restore purposes we treat all endpoints with state after
// StateEstablished and before StateClosed like connected endpoint.
if epState.connected() {
connectedLoading.Add(1)
}
switch {
case epState == StateListen:
listenLoading.Add(1)
case epState.connecting():
connectingLoading.Add(1)
}
// Directly update the state here rather than using e.setEndpointState
// as the endpoint is still being loaded and the stack reference is not
// yet initialized.
e.state.Store(uint32(epState))
}
// afterLoad is invoked by stateify.
func (e *Endpoint) afterLoad(ctx context.Context) {
// RacyLoad() can be used because we are initializing e.
e.origEndpointState = e.state.RacyLoad()
// Restore the endpoint to InitialState as it will be moved to
// its origEndpointState during Restore.
e.state = atomicbitops.FromUint32(uint32(StateInitial))
if e.stack.IsSaveRestoreEnabled() {
e.stack.RegisterRestoredEndpoint(e)
} else {
stack.RestoreStackFromContext(ctx).RegisterRestoredEndpoint(e)
}
}
// Restore implements tcpip.RestoredEndpoint.Restore.
func (e *Endpoint) Restore(s *stack.Stack) {
if !e.EndpointState().closed() {
e.keepalive.timer.init(s.Clock(), timerHandler(e, e.keepaliveTimerExpired))
}
if snd := e.snd; snd != nil {
snd.resendTimer.init(s.Clock(), timerHandler(e, e.snd.retransmitTimerExpired))
snd.reorderTimer.init(s.Clock(), timerHandler(e, e.snd.rc.reorderTimerExpired))
snd.probeTimer.init(s.Clock(), timerHandler(e, e.snd.probeTimerExpired))
snd.corkTimer.init(s.Clock(), timerHandler(e, e.snd.corkTimerExpired))
}
saveRestoreEnabled := e.stack.IsSaveRestoreEnabled()
if !saveRestoreEnabled {
e.stack = s
e.protocol = protocolFromStack(s)
}
e.ops.InitHandler(e, e.stack, GetTCPSendBufferLimits, GetTCPReceiveBufferLimits)
e.segmentQueue.thaw()
e.mu.Lock()
id := e.ID
e.mu.Unlock()
bind := func() {
e.mu.Lock()
defer e.mu.Unlock()
if !saveRestoreEnabled {
addr, _, err := e.checkV4MappedLocked(tcpip.FullAddress{Addr: e.BindAddr, Port: e.TransportEndpointInfo.ID.LocalPort}, true /* bind */)
if err != nil {
panic("unable to parse BindAddr: " + err.String())
}
portRes := ports.Reservation{
Networks: e.effectiveNetProtos,
Transport: ProtocolNumber,
Addr: addr.Addr,
Port: addr.Port,
Flags: e.boundPortFlags,
BindToDevice: e.boundBindToDevice,
Dest: e.boundDest,
}
if ok := e.stack.ReserveTuple(portRes); !ok {
panic(fmt.Sprintf("unable to re-reserve tuple (%v, %q, %d, %+v, %d, %v)", e.effectiveNetProtos, addr.Addr, addr.Port, e.boundPortFlags, e.boundBindToDevice, e.boundDest))
}
}
e.isPortReserved = true
// Mark endpoint as bound.
e.setEndpointState(StateBound)
}
epState := EndpointState(e.origEndpointState)
switch {
case epState.connected():
bind()
if e.connectingAddress.BitLen() == 0 {
e.connectingAddress = e.TransportEndpointInfo.ID.RemoteAddress
// This endpoint is accepted by netstack but not yet by
// the app. If the endpoint is IPv6 but the remote
// address is IPv4, we need to connect as IPv6 so that
// dual-stack mode can be properly activated.
if e.NetProto == header.IPv6ProtocolNumber && e.TransportEndpointInfo.ID.RemoteAddress.BitLen() != header.IPv6AddressSizeBits {
e.connectingAddress = tcpip.AddrFrom16Slice(append(
[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff},
e.TransportEndpointInfo.ID.RemoteAddress.AsSlice()...,
))
}
}
// Reset the scoreboard to reinitialize the sack information as
// we do not restore SACK information.
e.scoreboard.Reset()
if saveRestoreEnabled {
// Unregister the endpoint before registering again during Connect.
e.stack.UnregisterTransportEndpoint(e.effectiveNetProtos, header.TCPProtocolNumber, e.TransportEndpointInfo.ID, e, e.boundPortFlags, e.boundBindToDevice)
}
e.mu.Lock()
err := e.connect(tcpip.FullAddress{NIC: e.boundNICID, Addr: e.connectingAddress, Port: e.TransportEndpointInfo.ID.RemotePort}, false /* handshake */)
if _, ok := err.(*tcpip.ErrConnectStarted); !ok {
log.Warningf("TCP endpoint connect failed for connected endpoint with ID: %+v err: %v", id, err)
e.mu.Unlock()
e.Close()
connectedLoading.Done()
return
}
e.state.Store(e.origEndpointState)
// For FIN-WAIT-2 and TIME-WAIT we need to start the appropriate timers so
// that the socket is closed correctly.
switch epState {
case StateFinWait2:
e.finWait2Timer = e.stack.Clock().AfterFunc(e.tcpLingerTimeout, e.finWait2TimerExpired)
case StateTimeWait:
e.timeWaitTimer = e.stack.Clock().AfterFunc(e.getTimeWaitDuration(), e.timeWaitTimerExpired)
}
if e.ops.GetCorkOption() {
// Rearm the timer if TCP_CORK is enabled which will
// drain all the segments in the queue after restore.
e.snd.corkTimer.enable(MinRTO)
}
e.mu.Unlock()
connectedLoading.Done()
case epState == StateListen:
tcpip.AsyncLoading.Add(1)
if !saveRestoreEnabled {
go func() {
connectedLoading.Wait()
bind()
e.acceptMu.Lock()
backlog := e.acceptQueue.capacity
e.acceptMu.Unlock()
if err := e.Listen(backlog); err != nil {
panic("endpoint listening failed: " + err.String())
}
e.LockUser()
if e.shutdownFlags != 0 {
e.shutdownLocked(e.shutdownFlags)
}
e.UnlockUser()
listenLoading.Done()
tcpip.AsyncLoading.Done()
}()
} else {
go func() {
connectedLoading.Wait()
e.LockUser()
// All endpoints will be moved to initial state after
// restore. Set endpoint to its originial listen state.
e.setEndpointState(StateListen)
// Initialize the listening context.
rcvWnd := seqnum.Size(e.receiveBufferAvailable())
e.listenCtx = newListenContext(e.stack, e.protocol, e, rcvWnd, e.ops.GetV6Only(), e.NetProto)
e.UnlockUser()
listenLoading.Done()
tcpip.AsyncLoading.Done()
}()
}
case epState == StateConnecting:
// Initial SYN hasn't been sent yet so initiate a connect.
tcpip.AsyncLoading.Add(1)
go func() {
connectedLoading.Wait()
listenLoading.Wait()
bind()
err := e.Connect(tcpip.FullAddress{NIC: e.boundNICID, Addr: e.connectingAddress, Port: e.TransportEndpointInfo.ID.RemotePort})
if _, ok := err.(*tcpip.ErrConnectStarted); !ok {
log.Warningf("TCP endpoint connect failed for connecting endpoint with ID: %+v err: %v", id, err)
e.Close()
}
connectingLoading.Done()
tcpip.AsyncLoading.Done()
}()
case epState == StateSynSent || epState == StateSynRecv:
tcpip.AsyncLoading.Add(1)
go func() {
connectedLoading.Wait()
listenLoading.Wait()
// Initial SYN has been sent/received so we should bind the
// ports start the retransmit timer for the SYNs and let it
// naturally complete the connection.
bind()
e.mu.Lock()
e.setEndpointState(epState)
r, err := e.stack.FindRoute(e.boundNICID, e.TransportEndpointInfo.ID.LocalAddress, e.TransportEndpointInfo.ID.RemoteAddress, e.effectiveNetProtos[0], false /* multicastLoop */)
if err != nil {
e.mu.Unlock()
log.Warningf("FindRoute failed when restoring endpoint w/ ID: %+v err: %v", id, err)
e.Close()
connectingLoading.Done()
tcpip.AsyncLoading.Done()
return
}
e.route = r
timer, err := newBackoffTimer(e.stack.Clock(), InitialRTO, MaxRTO, timerHandler(e, e.h.retransmitHandlerLocked))
if err != nil {
panic(fmt.Sprintf("newBackOffTimer(_, %s, %s, _) failed: %s", InitialRTO, MaxRTO, err))
}
e.h.retransmitTimer = timer
connectingLoading.Done()
tcpip.AsyncLoading.Done()
e.mu.Unlock()
}()
case epState == StateBound:
tcpip.AsyncLoading.Add(1)
go func() {
connectedLoading.Wait()
listenLoading.Wait()
connectingLoading.Wait()
bind()
tcpip.AsyncLoading.Done()
}()
case epState == StateClose:
e.isPortReserved = false
e.state.Store(uint32(StateClose))
e.stack.CompleteTransportEndpointCleanup(e)
tcpip.DeleteDanglingEndpoint(e)
case epState == StateError:
e.state.Store(uint32(StateError))
e.stack.CompleteTransportEndpointCleanup(e)
tcpip.DeleteDanglingEndpoint(e)
}
}
// Resume implements tcpip.ResumableEndpoint.Resume.
func (e *Endpoint) Resume() {
e.segmentQueue.thaw()
}

64
pkg/tcpip/transport/tcp/ep_queue_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type epQueueMutex struct {
mu sync.Mutex
}
var epQueueprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var epQueuelockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type epQueuelockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *epQueueMutex) Lock() {
locking.AddGLock(epQueueprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *epQueueMutex) NestedLock(i epQueuelockNameIndex) {
locking.AddGLock(epQueueprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *epQueueMutex) Unlock() {
locking.DelGLock(epQueueprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *epQueueMutex) NestedUnlock(i epQueuelockNameIndex) {
locking.DelGLock(epQueueprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func epQueueinitLockNames() {}
func init() {
epQueueinitLockNames()
epQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(epQueueMutex{}), epQueuelockNames)
}

231
pkg/tcpip/transport/tcp/forwarder.go Normal file
View file

@ -0,0 +1,231 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"fmt"
"github.com/sagernet/gvisor/pkg/buffer"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/waiter"
)
// Forwarder is a connection request forwarder, which allows clients to decide
// what to do with a connection request, for example: ignore it, send a RST, or
// attempt to complete the 3-way handshake.
//
// The canonical way of using it is to pass the Forwarder.HandlePacket function
// to stack.SetTransportProtocolHandler.
type Forwarder struct {
stack *stack.Stack
maxInFlight int
handler func(*ForwarderRequest)
mu forwarderMutex
inFlight map[stack.TransportEndpointID]struct{}
listen *listenContext
}
// NewForwarder allocates and initializes a new forwarder with the given
// maximum number of in-flight connection attempts. Once the maximum is reached
// new incoming connection requests will be ignored.
//
// If rcvWnd is set to zero, the default buffer size is used instead.
func NewForwarder(s *stack.Stack, rcvWnd, maxInFlight int, handler func(*ForwarderRequest)) *Forwarder {
if rcvWnd == 0 {
rcvWnd = DefaultReceiveBufferSize
}
return &Forwarder{
stack: s,
maxInFlight: maxInFlight,
handler: handler,
inFlight: make(map[stack.TransportEndpointID]struct{}),
listen: newListenContext(s, protocolFromStack(s), nil /* listenEP */, seqnum.Size(rcvWnd), true, 0),
}
}
// HandlePacket handles a packet if it is of interest to the forwarder (i.e., if
// it's a SYN packet), returning true if it's the case. Otherwise the packet
// is not handled and false is returned.
//
// This function is expected to be passed as an argument to the
// stack.SetTransportProtocolHandler function.
func (f *Forwarder) HandlePacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) bool {
s, err := newIncomingSegment(id, f.stack.Clock(), pkt)
if err != nil {
return false
}
defer s.DecRef()
// We only care about well-formed SYN packets (not SYN-ACK) packets.
if !s.csumValid || !s.flags.Contains(header.TCPFlagSyn) || s.flags.Contains(header.TCPFlagAck) {
return false
}
opts := parseSynSegmentOptions(s)
f.mu.Lock()
defer f.mu.Unlock()
// We have an inflight request for this id, ignore this one for now.
if _, ok := f.inFlight[id]; ok {
return true
}
// Ignore the segment if we're beyond the limit.
if len(f.inFlight) >= f.maxInFlight {
f.stack.Stats().TCP.ForwardMaxInFlightDrop.Increment()
return true
}
// Launch a new goroutine to handle the request.
f.inFlight[id] = struct{}{}
s.IncRef()
go f.handler(&ForwarderRequest{ // S/R-SAFE: not used by Sentry.
forwarder: f,
segment: s,
synOptions: opts,
})
return true
}
// ForwarderRequest represents a connection request received by the forwarder
// and passed to the client. Clients must eventually call Complete() on it, and
// may optionally create an endpoint to represent it via CreateEndpoint.
type ForwarderRequest struct {
mu forwarderRequestMutex
forwarder *Forwarder
segment *segment
synOptions header.TCPSynOptions
}
// ID returns the 4-tuple (src address, src port, dst address, dst port) that
// represents the connection request.
func (r *ForwarderRequest) ID() stack.TransportEndpointID {
return r.segment.id
}
// Complete completes the request, and optionally sends a RST segment back to the
// sender.
func (r *ForwarderRequest) Complete(sendReset bool) {
r.mu.Lock()
defer r.mu.Unlock()
if r.segment == nil {
panic("Completing already completed forwarder request")
}
// Remove request from the forwarder.
r.forwarder.mu.Lock()
delete(r.forwarder.inFlight, r.segment.id)
r.forwarder.mu.Unlock()
if sendReset {
replyWithReset(r.forwarder.stack, r.segment, stack.DefaultTOS, tcpip.UseDefaultIPv4TTL, tcpip.UseDefaultIPv6HopLimit)
}
// Release all resources.
r.segment.DecRef()
r.segment = nil
r.forwarder = nil
}
// CreateEndpoint creates a TCP endpoint for the connection request, performing
// the 3-way handshake in the process.
func (r *ForwarderRequest) CreateEndpoint(queue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.segment == nil {
return nil, &tcpip.ErrInvalidEndpointState{}
}
f := r.forwarder
ep, err := f.listen.performHandshake(r.segment, header.TCPSynOptions{
MSS: r.synOptions.MSS,
WS: r.synOptions.WS,
TS: r.synOptions.TS,
TSVal: r.synOptions.TSVal,
TSEcr: r.synOptions.TSEcr,
SACKPermitted: r.synOptions.SACKPermitted,
}, queue, nil)
if err != nil {
return nil, err
}
return ep, nil
}
// ForwardedPacketExperimentOption returns the experiment option value from the
// forwarded packet and a bool indicating whether an experiment option value was
// found.
func (r *ForwarderRequest) ForwardedPacketExperimentOption() (uint16, bool) {
r.mu.Lock()
defer r.mu.Unlock()
switch r.segment.pkt.NetworkProtocolNumber {
case header.IPv4ProtocolNumber:
h := header.IPv4(r.segment.pkt.NetworkHeader().Slice())
opts := h.Options()
iter := opts.MakeIterator()
for {
opt, done, err := iter.Next()
if err != nil {
return 0, false
}
if done {
return 0, false
}
if opt.Type() == header.IPv4OptionExperimentType {
return opt.(*header.IPv4OptionExperiment).Value(), true
}
}
case header.IPv6ProtocolNumber:
h := header.IPv6(r.segment.pkt.NetworkHeader().Slice())
v := r.segment.pkt.NetworkHeader().View()
if v != nil {
v.TrimFront(header.IPv6MinimumSize)
}
buf := buffer.MakeWithView(v)
buf.Append(r.segment.pkt.TransportHeader().View())
dataBuf := r.segment.pkt.Data().ToBuffer()
buf.Merge(&dataBuf)
it := header.MakeIPv6PayloadIterator(header.IPv6ExtensionHeaderIdentifier(h.NextHeader()), buf)
for {
hdr, done, err := it.Next()
if done || err != nil {
break
}
if h, ok := hdr.(header.IPv6ExperimentExtHdr); ok {
hdr.Release()
return h.Value, true
}
hdr.Release()
}
default:
panic(fmt.Sprintf("Unexpected network protocol number %d", r.segment.pkt.NetworkProtocolNumber))
}
return 0, false
}
func (r *ForwarderRequest) Packet() *stack.PacketBuffer {
return r.segment.pkt
}

64
pkg/tcpip/transport/tcp/forwarder_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type forwarderMutex struct {
mu sync.Mutex
}
var forwarderprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var forwarderlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type forwarderlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *forwarderMutex) Lock() {
locking.AddGLock(forwarderprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *forwarderMutex) NestedLock(i forwarderlockNameIndex) {
locking.AddGLock(forwarderprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *forwarderMutex) Unlock() {
locking.DelGLock(forwarderprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *forwarderMutex) NestedUnlock(i forwarderlockNameIndex) {
locking.DelGLock(forwarderprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func forwarderinitLockNames() {}
func init() {
forwarderinitLockNames()
forwarderprefixIndex = locking.NewMutexClass(reflect.TypeOf(forwarderMutex{}), forwarderlockNames)
}

64
pkg/tcpip/transport/tcp/forwarder_request_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type forwarderRequestMutex struct {
mu sync.Mutex
}
var forwarderRequestprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var forwarderRequestlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type forwarderRequestlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *forwarderRequestMutex) Lock() {
locking.AddGLock(forwarderRequestprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *forwarderRequestMutex) NestedLock(i forwarderRequestlockNameIndex) {
locking.AddGLock(forwarderRequestprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *forwarderRequestMutex) Unlock() {
locking.DelGLock(forwarderRequestprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *forwarderRequestMutex) NestedUnlock(i forwarderRequestlockNameIndex) {
locking.DelGLock(forwarderRequestprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func forwarderRequestinitLockNames() {}
func init() {
forwarderRequestinitLockNames()
forwarderRequestprefixIndex = locking.NewMutexClass(reflect.TypeOf(forwarderRequestMutex{}), forwarderRequestlockNames)
}

64
pkg/tcpip/transport/tcp/hasher_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type hasherMutex struct {
mu sync.Mutex
}
var hasherprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var hasherlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type hasherlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *hasherMutex) Lock() {
locking.AddGLock(hasherprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *hasherMutex) NestedLock(i hasherlockNameIndex) {
locking.AddGLock(hasherprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *hasherMutex) Unlock() {
locking.DelGLock(hasherprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *hasherMutex) NestedUnlock(i hasherlockNameIndex) {
locking.DelGLock(hasherprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func hasherinitLockNames() {}
func init() {
hasherinitLockNames()
hasherprefixIndex = locking.NewMutexClass(reflect.TypeOf(hasherMutex{}), hasherlockNames)
}

64
pkg/tcpip/transport/tcp/keepalive_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type keepaliveMutex struct {
mu sync.Mutex
}
var keepaliveprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var keepalivelockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type keepalivelockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *keepaliveMutex) Lock() {
locking.AddGLock(keepaliveprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *keepaliveMutex) NestedLock(i keepalivelockNameIndex) {
locking.AddGLock(keepaliveprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *keepaliveMutex) Unlock() {
locking.DelGLock(keepaliveprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *keepaliveMutex) NestedUnlock(i keepalivelockNameIndex) {
locking.DelGLock(keepaliveprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func keepaliveinitLockNames() {}
func init() {
keepaliveinitLockNames()
keepaliveprefixIndex = locking.NewMutexClass(reflect.TypeOf(keepaliveMutex{}), keepalivelockNames)
}

64
pkg/tcpip/transport/tcp/last_error_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type lastErrorMutex struct {
mu sync.Mutex
}
var lastErrorprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var lastErrorlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type lastErrorlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *lastErrorMutex) Lock() {
locking.AddGLock(lastErrorprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *lastErrorMutex) NestedLock(i lastErrorlockNameIndex) {
locking.AddGLock(lastErrorprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *lastErrorMutex) Unlock() {
locking.DelGLock(lastErrorprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *lastErrorMutex) NestedUnlock(i lastErrorlockNameIndex) {
locking.DelGLock(lastErrorprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func lastErrorinitLockNames() {}
func init() {
lastErrorinitLockNames()
lastErrorprefixIndex = locking.NewMutexClass(reflect.TypeOf(lastErrorMutex{}), lastErrorlockNames)
}

64
pkg/tcpip/transport/tcp/pending_processing_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type pendingProcessingMutex struct {
mu sync.Mutex
}
var pendingProcessingprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var pendingProcessinglockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type pendingProcessinglockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *pendingProcessingMutex) Lock() {
locking.AddGLock(pendingProcessingprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *pendingProcessingMutex) NestedLock(i pendingProcessinglockNameIndex) {
locking.AddGLock(pendingProcessingprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *pendingProcessingMutex) Unlock() {
locking.DelGLock(pendingProcessingprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *pendingProcessingMutex) NestedUnlock(i pendingProcessinglockNameIndex) {
locking.DelGLock(pendingProcessingprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func pendingProcessinginitLockNames() {}
func init() {
pendingProcessinginitLockNames()
pendingProcessingprefixIndex = locking.NewMutexClass(reflect.TypeOf(pendingProcessingMutex{}), pendingProcessinglockNames)
}

606
pkg/tcpip/transport/tcp/protocol.go Normal file
View file

@ -0,0 +1,606 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package tcp contains the implementation of the TCP transport protocol.
package tcp
import (
"crypto/sha256"
"encoding/binary"
"fmt"
"runtime"
"strings"
"time"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/header/parse"
"github.com/sagernet/gvisor/pkg/tcpip/internal/tcp"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
"github.com/sagernet/gvisor/pkg/tcpip/transport/raw"
"github.com/sagernet/gvisor/pkg/waiter"
)
const (
// ProtocolNumber is the tcp protocol number.
ProtocolNumber = header.TCPProtocolNumber
// MinBufferSize is the smallest size of a receive or send buffer.
MinBufferSize = 4 << 10 // 4096 bytes.
// DefaultSendBufferSize is the default size of the send buffer for
// an endpoint.
DefaultSendBufferSize = 1 << 20 // 1MB
// DefaultReceiveBufferSize is the default size of the receive buffer
// for an endpoint.
DefaultReceiveBufferSize = 1 << 20 // 1MB
// MaxBufferSize is the largest size a receive/send buffer can grow to.
MaxBufferSize = 4 << 20 // 4MB
// DefaultTCPLingerTimeout is the amount of time that sockets linger in
// FIN_WAIT_2 state before being marked closed.
DefaultTCPLingerTimeout = 60 * time.Second
// MaxTCPLingerTimeout is the maximum amount of time that sockets
// linger in FIN_WAIT_2 state before being marked closed.
MaxTCPLingerTimeout = 120 * time.Second
// DefaultTCPTimeWaitTimeout is the amount of time that sockets linger
// in TIME_WAIT state before being marked closed.
DefaultTCPTimeWaitTimeout = 60 * time.Second
// DefaultSynRetries is the default value for the number of SYN retransmits
// before a connect is aborted.
DefaultSynRetries = 6
// DefaultKeepaliveIdle is the idle time for a connection before keep-alive
// probes are sent.
DefaultKeepaliveIdle = 2 * time.Hour
// DefaultKeepaliveInterval is the time between two successive keep-alive
// probes.
DefaultKeepaliveInterval = 75 * time.Second
// DefaultKeepaliveCount is the number of keep-alive probes that are sent
// before declaring the connection dead.
DefaultKeepaliveCount = 9
)
const (
ccReno = "reno"
ccCubic = "cubic"
)
// +stateify savable
type protocol struct {
stack *stack.Stack
mu protocolRWMutex `state:"nosave"`
sackEnabled bool
recovery tcpip.TCPRecovery
delayEnabled bool
alwaysUseSynCookies bool
sendBufferSize tcpip.TCPSendBufferSizeRangeOption
recvBufferSize tcpip.TCPReceiveBufferSizeRangeOption
congestionControl string
availableCongestionControl []string
moderateReceiveBuffer bool
lingerTimeout time.Duration
timeWaitTimeout time.Duration
timeWaitReuse tcpip.TCPTimeWaitReuseOption
minRTO time.Duration
maxRTO time.Duration
maxRetries uint32
synRetries uint8
dispatcher dispatcher
// probe, if not nil, will be invoked any time an endpoint receives a
// TCP segment.
//
// This is immutable after creation.
probe TCPProbeFunc `state:"nosave"`
// The following secrets are initialized once and stay unchanged after.
seqnumSecret [16]byte
tsOffsetSecret [16]byte
}
// Number returns the tcp protocol number.
func (*protocol) Number() tcpip.TransportProtocolNumber {
return ProtocolNumber
}
// NewEndpoint creates a new tcp endpoint.
func (p *protocol) NewEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) {
return newEndpoint(p.stack, p, netProto, waiterQueue), nil
}
// NewRawEndpoint creates a new raw TCP endpoint. Raw TCP sockets are currently
// unsupported. It implements stack.TransportProtocol.NewRawEndpoint.
func (p *protocol) NewRawEndpoint(netProto tcpip.NetworkProtocolNumber, waiterQueue *waiter.Queue) (tcpip.Endpoint, tcpip.Error) {
return raw.NewEndpoint(p.stack, netProto, header.TCPProtocolNumber, waiterQueue)
}
// MinimumPacketSize returns the minimum valid tcp packet size.
func (*protocol) MinimumPacketSize() int {
return header.TCPMinimumSize
}
// ParsePorts returns the source and destination ports stored in the given tcp
// packet.
func (*protocol) ParsePorts(v []byte) (src, dst uint16, err tcpip.Error) {
h := header.TCP(v)
return h.SourcePort(), h.DestinationPort(), nil
}
// QueuePacket queues packets targeted at an endpoint after hashing the packet
// to a specific processing queue. Each queue is serviced by its own processor
// goroutine which is responsible for dequeuing and doing full TCP dispatch of
// the packet.
func (p *protocol) QueuePacket(ep stack.TransportEndpoint, id stack.TransportEndpointID, pkt *stack.PacketBuffer) {
p.dispatcher.queuePacket(ep, id, p.stack.Clock(), pkt)
}
// HandleUnknownDestinationPacket handles packets targeted at this protocol but
// that don't match any existing endpoint.
//
// RFC 793, page 36, states that "If the connection does not exist (CLOSED) then
// a reset is sent in response to any incoming segment except another reset. In
// particular, SYNs addressed to a non-existent connection are rejected by this
// means."
func (p *protocol) HandleUnknownDestinationPacket(id stack.TransportEndpointID, pkt *stack.PacketBuffer) stack.UnknownDestinationPacketDisposition {
s, err := newIncomingSegment(id, p.stack.Clock(), pkt)
if err != nil {
return stack.UnknownDestinationPacketMalformed
}
defer s.DecRef()
if !s.csumValid {
return stack.UnknownDestinationPacketMalformed
}
if !s.flags.Contains(header.TCPFlagRst) {
replyWithReset(p.stack, s, stack.DefaultTOS, tcpip.UseDefaultIPv4TTL, tcpip.UseDefaultIPv6HopLimit)
}
return stack.UnknownDestinationPacketHandled
}
func (p *protocol) tsOffset(src, dst tcpip.Address) tcp.TSOffset {
// Initialize a random tsOffset that will be added to the recentTS
// everytime the timestamp is sent when the Timestamp option is enabled.
//
// See https://tools.ietf.org/html/rfc7323#section-5.4 for details on
// why this is required.
h := sha256.New()
// Per hash.Hash.Writer:
//
// It never returns an error.
_, _ = h.Write(p.tsOffsetSecret[:])
_, _ = h.Write(src.AsSlice())
_, _ = h.Write(dst.AsSlice())
return tcp.NewTSOffset(binary.LittleEndian.Uint32(h.Sum(nil)[:4]))
}
// replyWithReset replies to the given segment with a reset segment.
//
// If the relevant TTL has its reset value (0 for ipv4TTL, -1 for ipv6HopLimit),
// then the route's default TTL will be used.
func replyWithReset(st *stack.Stack, s *segment, tos, ipv4TTL uint8, ipv6HopLimit int16) tcpip.Error {
net := s.pkt.Network()
route, err := st.FindRoute(s.pkt.NICID, net.DestinationAddress(), net.SourceAddress(), s.pkt.NetworkProtocolNumber, false /* multicastLoop */)
if err != nil {
return err
}
defer route.Release()
ttl := calculateTTL(route, ipv4TTL, ipv6HopLimit)
// Get the seqnum from the packet if the ack flag is set.
seq := seqnum.Value(0)
ack := seqnum.Value(0)
flags := header.TCPFlagRst
// As per RFC 793 page 35 (Reset Generation)
// 1. If the connection does not exist (CLOSED) then a reset is sent
// in response to any incoming segment except another reset. In
// particular, SYNs addressed to a non-existent connection are rejected
// by this means.
// If the incoming segment has an ACK field, the reset takes its
// sequence number from the ACK field of the segment, otherwise the
// reset has sequence number zero and the ACK field is set to the sum
// of the sequence number and segment length of the incoming segment.
// The connection remains in the CLOSED state.
if s.flags.Contains(header.TCPFlagAck) {
seq = s.ackNumber
} else {
flags |= header.TCPFlagAck
ack = s.sequenceNumber.Add(s.logicalLen())
}
var expOptVal uint16
if s.ep != nil {
expOptVal = s.ep.getExperimentOptionValue(route)
}
hdrSize := header.TCPMinimumSize + int(route.MaxHeaderLength())
if route.NetProto() == header.IPv6ProtocolNumber && expOptVal != 0 {
hdrSize += header.IPv6ExperimentHdrLength
}
p := stack.NewPacketBuffer(stack.PacketBufferOptions{ReserveHeaderBytes: hdrSize})
defer p.DecRef()
return sendTCP(route, tcpFields{
id: s.id,
ttl: ttl,
tos: tos,
flags: flags,
seq: seq,
ack: ack,
rcvWnd: 0,
expOptVal: expOptVal,
}, p, stack.GSO{}, nil /* PacketOwner */)
}
// SetOption implements stack.TransportProtocol.SetOption.
func (p *protocol) SetOption(option tcpip.SettableTransportProtocolOption) tcpip.Error {
switch v := option.(type) {
case *tcpip.TCPSACKEnabled:
p.mu.Lock()
p.sackEnabled = bool(*v)
p.mu.Unlock()
return nil
case *tcpip.TCPRecovery:
p.mu.Lock()
p.recovery = *v
p.mu.Unlock()
return nil
case *tcpip.TCPDelayEnabled:
p.mu.Lock()
p.delayEnabled = bool(*v)
p.mu.Unlock()
return nil
case *tcpip.TCPSendBufferSizeRangeOption:
if v.Min <= 0 || v.Default < v.Min || v.Default > v.Max {
return &tcpip.ErrInvalidOptionValue{}
}
p.mu.Lock()
p.sendBufferSize = *v
p.mu.Unlock()
return nil
case *tcpip.TCPReceiveBufferSizeRangeOption:
if v.Min <= 0 || v.Default < v.Min || v.Default > v.Max {
return &tcpip.ErrInvalidOptionValue{}
}
p.mu.Lock()
p.recvBufferSize = *v
p.mu.Unlock()
return nil
case *tcpip.CongestionControlOption:
for _, c := range p.availableCongestionControl {
if string(*v) == c {
p.mu.Lock()
p.congestionControl = string(*v)
p.mu.Unlock()
return nil
}
}
// linux returns ENOENT when an invalid congestion control
// is specified.
return &tcpip.ErrNoSuchFile{}
case *tcpip.TCPModerateReceiveBufferOption:
p.mu.Lock()
p.moderateReceiveBuffer = bool(*v)
p.mu.Unlock()
return nil
case *tcpip.TCPLingerTimeoutOption:
p.mu.Lock()
if *v < 0 {
p.lingerTimeout = 0
} else {
p.lingerTimeout = time.Duration(*v)
}
p.mu.Unlock()
return nil
case *tcpip.TCPTimeWaitTimeoutOption:
p.mu.Lock()
if *v < 0 {
p.timeWaitTimeout = 0
} else {
p.timeWaitTimeout = time.Duration(*v)
}
p.mu.Unlock()
return nil
case *tcpip.TCPTimeWaitReuseOption:
if *v < tcpip.TCPTimeWaitReuseDisabled || *v > tcpip.TCPTimeWaitReuseLoopbackOnly {
return &tcpip.ErrInvalidOptionValue{}
}
p.mu.Lock()
p.timeWaitReuse = *v
p.mu.Unlock()
return nil
case *tcpip.TCPMinRTOOption:
p.mu.Lock()
defer p.mu.Unlock()
if *v < 0 {
p.minRTO = MinRTO
} else if minRTO := time.Duration(*v); minRTO <= p.maxRTO {
p.minRTO = minRTO
} else {
return &tcpip.ErrInvalidOptionValue{}
}
return nil
case *tcpip.TCPMaxRTOOption:
p.mu.Lock()
defer p.mu.Unlock()
if *v < 0 {
p.maxRTO = MaxRTO
} else if maxRTO := time.Duration(*v); maxRTO >= p.minRTO {
p.maxRTO = maxRTO
} else {
return &tcpip.ErrInvalidOptionValue{}
}
return nil
case *tcpip.TCPMaxRetriesOption:
p.mu.Lock()
p.maxRetries = uint32(*v)
p.mu.Unlock()
return nil
case *tcpip.TCPAlwaysUseSynCookies:
p.mu.Lock()
p.alwaysUseSynCookies = bool(*v)
p.mu.Unlock()
return nil
case *tcpip.TCPSynRetriesOption:
if *v < 1 {
return &tcpip.ErrInvalidOptionValue{}
}
p.mu.Lock()
p.synRetries = uint8(*v)
p.mu.Unlock()
return nil
default:
return &tcpip.ErrUnknownProtocolOption{}
}
}
// Option implements stack.TransportProtocol.Option.
func (p *protocol) Option(option tcpip.GettableTransportProtocolOption) tcpip.Error {
switch v := option.(type) {
case *tcpip.TCPSACKEnabled:
p.mu.RLock()
*v = tcpip.TCPSACKEnabled(p.sackEnabled)
p.mu.RUnlock()
return nil
case *tcpip.TCPRecovery:
p.mu.RLock()
*v = p.recovery
p.mu.RUnlock()
return nil
case *tcpip.TCPDelayEnabled:
p.mu.RLock()
*v = tcpip.TCPDelayEnabled(p.delayEnabled)
p.mu.RUnlock()
return nil
case *tcpip.TCPSendBufferSizeRangeOption:
p.mu.RLock()
*v = p.sendBufferSize
p.mu.RUnlock()
return nil
case *tcpip.TCPReceiveBufferSizeRangeOption:
p.mu.RLock()
*v = p.recvBufferSize
p.mu.RUnlock()
return nil
case *tcpip.CongestionControlOption:
p.mu.RLock()
*v = tcpip.CongestionControlOption(p.congestionControl)
p.mu.RUnlock()
return nil
case *tcpip.TCPAvailableCongestionControlOption:
p.mu.RLock()
*v = tcpip.TCPAvailableCongestionControlOption(strings.Join(p.availableCongestionControl, " "))
p.mu.RUnlock()
return nil
case *tcpip.TCPModerateReceiveBufferOption:
p.mu.RLock()
*v = tcpip.TCPModerateReceiveBufferOption(p.moderateReceiveBuffer)
p.mu.RUnlock()
return nil
case *tcpip.TCPLingerTimeoutOption:
p.mu.RLock()
*v = tcpip.TCPLingerTimeoutOption(p.lingerTimeout)
p.mu.RUnlock()
return nil
case *tcpip.TCPTimeWaitTimeoutOption:
p.mu.RLock()
*v = tcpip.TCPTimeWaitTimeoutOption(p.timeWaitTimeout)
p.mu.RUnlock()
return nil
case *tcpip.TCPTimeWaitReuseOption:
p.mu.RLock()
*v = p.timeWaitReuse
p.mu.RUnlock()
return nil
case *tcpip.TCPMinRTOOption:
p.mu.RLock()
*v = tcpip.TCPMinRTOOption(p.minRTO)
p.mu.RUnlock()
return nil
case *tcpip.TCPMaxRTOOption:
p.mu.RLock()
*v = tcpip.TCPMaxRTOOption(p.maxRTO)
p.mu.RUnlock()
return nil
case *tcpip.TCPMaxRetriesOption:
p.mu.RLock()
*v = tcpip.TCPMaxRetriesOption(p.maxRetries)
p.mu.RUnlock()
return nil
case *tcpip.TCPAlwaysUseSynCookies:
p.mu.RLock()
*v = tcpip.TCPAlwaysUseSynCookies(p.alwaysUseSynCookies)
p.mu.RUnlock()
return nil
case *tcpip.TCPSynRetriesOption:
p.mu.RLock()
*v = tcpip.TCPSynRetriesOption(p.synRetries)
p.mu.RUnlock()
return nil
default:
return &tcpip.ErrUnknownProtocolOption{}
}
}
// SendBufferSize implements stack.SendBufSizeProto.
func (p *protocol) SendBufferSize() tcpip.TCPSendBufferSizeRangeOption {
p.mu.RLock()
defer p.mu.RUnlock()
return p.sendBufferSize
}
// Close implements stack.TransportProtocol.Close.
func (p *protocol) Close() {
p.dispatcher.close()
}
// Wait implements stack.TransportProtocol.Wait.
func (p *protocol) Wait() {
p.dispatcher.wait()
}
// Pause implements stack.TransportProtocol.Pause.
func (p *protocol) Pause() {
p.dispatcher.pause()
}
// Resume implements stack.TransportProtocol.Resume.
func (p *protocol) Resume() {
p.dispatcher.resume()
}
// Restore implements stack.TransportProtocol.Restore.
func (p *protocol) Restore() {
p.dispatcher.start()
}
// Parse implements stack.TransportProtocol.Parse.
func (*protocol) Parse(pkt *stack.PacketBuffer) bool {
return parse.TCP(pkt)
}
// NewProtocol returns a TCP transport protocol with Reno congestion control.
func NewProtocol(s *stack.Stack) stack.TransportProtocol {
return newProtocol(s, ccReno, nil)
}
// NewProtocolProbe returns a TCP transport protocol with Reno congestion
// control and the given probe.
//
// The probe will be invoked on every segment received by TCP endpoints. The
// probe function is passed a copy of the TCP endpoint state before and after
// processing of the segment.
func NewProtocolProbe(probe TCPProbeFunc) func(*stack.Stack) stack.TransportProtocol {
return func(s *stack.Stack) stack.TransportProtocol {
return newProtocol(s, ccReno, probe)
}
}
// NewProtocolCUBIC returns a TCP transport protocol with CUBIC congestion
// control.
//
// TODO(b/345835636): Remove this and make CUBIC the default across the board.
func NewProtocolCUBIC(s *stack.Stack) stack.TransportProtocol {
return newProtocol(s, ccCubic, nil)
}
func newProtocol(s *stack.Stack, cc string, probe TCPProbeFunc) stack.TransportProtocol {
rng := s.SecureRNG()
var seqnumSecret [16]byte
var tsOffsetSecret [16]byte
if n, err := rng.Reader.Read(seqnumSecret[:]); err != nil || n != len(seqnumSecret) {
panic(fmt.Sprintf("Read() failed: %v", err))
}
if n, err := rng.Reader.Read(tsOffsetSecret[:]); err != nil || n != len(tsOffsetSecret) {
panic(fmt.Sprintf("Read() failed: %v", err))
}
p := protocol{
stack: s,
sendBufferSize: tcpip.TCPSendBufferSizeRangeOption{
Min: MinBufferSize,
Default: DefaultSendBufferSize,
Max: MaxBufferSize,
},
recvBufferSize: tcpip.TCPReceiveBufferSizeRangeOption{
Min: MinBufferSize,
Default: DefaultReceiveBufferSize,
Max: MaxBufferSize,
},
sackEnabled: true,
congestionControl: cc,
availableCongestionControl: []string{ccReno, ccCubic},
moderateReceiveBuffer: true,
lingerTimeout: DefaultTCPLingerTimeout,
timeWaitTimeout: DefaultTCPTimeWaitTimeout,
timeWaitReuse: tcpip.TCPTimeWaitReuseLoopbackOnly,
synRetries: DefaultSynRetries,
minRTO: MinRTO,
maxRTO: MaxRTO,
maxRetries: MaxRetries,
recovery: tcpip.TCPRACKLossDetection,
seqnumSecret: seqnumSecret,
tsOffsetSecret: tsOffsetSecret,
probe: probe,
}
p.dispatcher.init(s.InsecureRNG(), runtime.GOMAXPROCS(0))
return &p
}
// protocolFromStack retrieves the tcp.protocol instance from stack s.
func protocolFromStack(s *stack.Stack) *protocol {
return s.TransportProtocolInstance(ProtocolNumber).(*protocol)
}

96
pkg/tcpip/transport/tcp/protocol_mutex.go Normal file
View file

@ -0,0 +1,96 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// RWMutex is sync.RWMutex with the correctness validator.
type protocolRWMutex struct {
mu sync.RWMutex
}
// lockNames is a list of user-friendly lock names.
// Populated in init.
var protocollockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type protocollockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *protocolRWMutex) Lock() {
locking.AddGLock(protocolprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *protocolRWMutex) NestedLock(i protocollockNameIndex) {
locking.AddGLock(protocolprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *protocolRWMutex) Unlock() {
m.mu.Unlock()
locking.DelGLock(protocolprefixIndex, -1)
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *protocolRWMutex) NestedUnlock(i protocollockNameIndex) {
m.mu.Unlock()
locking.DelGLock(protocolprefixIndex, int(i))
}
// RLock locks m for reading.
// +checklocksignore
func (m *protocolRWMutex) RLock() {
locking.AddGLock(protocolprefixIndex, -1)
m.mu.RLock()
}
// RUnlock undoes a single RLock call.
// +checklocksignore
func (m *protocolRWMutex) RUnlock() {
m.mu.RUnlock()
locking.DelGLock(protocolprefixIndex, -1)
}
// RLockBypass locks m for reading without executing the validator.
// +checklocksignore
func (m *protocolRWMutex) RLockBypass() {
m.mu.RLock()
}
// RUnlockBypass undoes a single RLockBypass call.
// +checklocksignore
func (m *protocolRWMutex) RUnlockBypass() {
m.mu.RUnlock()
}
// DowngradeLock atomically unlocks rw for writing and locks it for reading.
// +checklocksignore
func (m *protocolRWMutex) DowngradeLock() {
m.mu.DowngradeLock()
}
var protocolprefixIndex *locking.MutexClass
// DO NOT REMOVE: The following function is automatically replaced.
func protocolinitLockNames() {}
func init() {
protocolinitLockNames()
protocolprefixIndex = locking.NewMutexClass(reflect.TypeOf(protocolRWMutex{}), protocollockNames)
}

459
pkg/tcpip/transport/tcp/rack.go Normal file
View file

@ -0,0 +1,459 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"time"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
)
const (
// wcDelayedACKTimeout is the recommended maximum delayed ACK timer
// value as defined in the RFC. It stands for worst case delayed ACK
// timer (WCDelAckT). When FlightSize is 1, PTO is inflated by
// WCDelAckT time to compensate for a potential long delayed ACK timer
// at the receiver.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.
wcDelayedACKTimeout = 200 * time.Millisecond
// tcpRACKRecoveryThreshold is the number of loss recoveries for which
// the reorder window is inflated and after that the reorder window is
// reset to its initial value of minRTT/4.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2.
tcpRACKRecoveryThreshold = 16
)
// RACK is a loss detection algorithm used in TCP to detect packet loss and
// reordering using transmission timestamp of the packets instead of packet or
// sequence counts. To use RACK, SACK should be enabled on the connection.
// rackControl stores the rack related fields.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-6.1
//
// +stateify savable
type rackControl struct {
TCPRACKState
// exitedRecovery indicates if the connection is exiting loss recovery.
// This flag is set if the sender is leaving the recovery after
// receiving an ACK and is reset during updating of reorder window.
exitedRecovery bool
// minRTT is the estimated minimum RTT of the connection.
minRTT time.Duration
// tlpRxtOut indicates whether there is an unacknowledged
// TLP retransmission.
tlpRxtOut bool
// tlpHighRxt the value of sender.sndNxt at the time of sending
// a TLP retransmission.
tlpHighRxt seqnum.Value
// snd is a reference to the sender.
snd *sender
}
// init initializes RACK specific fields.
func (rc *rackControl) init(snd *sender, iss seqnum.Value) {
rc.FACK = iss
rc.ReoWndIncr = 1
rc.snd = snd
}
// update will update the RACK related fields when an ACK has been received.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-09#section-6.2
func (rc *rackControl) update(seg *segment, ackSeg *segment) {
rtt := rc.snd.ep.stack.Clock().NowMonotonic().Sub(seg.xmitTime)
// If the ACK is for a retransmitted packet, do not update if it is a
// spurious inference which is determined by below checks:
// 1. When Timestamping option is available, if the TSVal is less than
// the transmit time of the most recent retransmitted packet.
// 2. When RTT calculated for the packet is less than the smoothed RTT
// for the connection.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2
// step 2
if seg.xmitCount > 1 {
if ackSeg.parsedOptions.TS && ackSeg.parsedOptions.TSEcr != 0 {
if ackSeg.parsedOptions.TSEcr < rc.snd.ep.tsVal(seg.xmitTime) {
return
}
}
if rtt < rc.minRTT {
return
}
}
rc.RTT = rtt
// The sender can either track a simple global minimum of all RTT
// measurements from the connection, or a windowed min-filtered value
// of recent RTT measurements. This implementation keeps track of the
// simple global minimum of all RTTs for the connection.
if rtt < rc.minRTT || rc.minRTT == 0 {
rc.minRTT = rtt
}
// Update rc.xmitTime and rc.endSequence to the transmit time and
// ending sequence number of the packet which has been acknowledged
// most recently.
endSeq := seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize()))
if rc.XmitTime.Before(seg.xmitTime) || (seg.xmitTime == rc.XmitTime && rc.EndSequence.LessThan(endSeq)) {
rc.XmitTime = seg.xmitTime
rc.EndSequence = endSeq
}
}
// detectReorder detects if packet reordering has been observed.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2
// - Step 3: Detect data segment reordering.
// To detect reordering, the sender looks for original data segments being
// delivered out of order. To detect such cases, the sender tracks the
// highest sequence selectively or cumulatively acknowledged in the RACK.fack
// variable. The name "fack" stands for the most "Forward ACK" (this term is
// adopted from [FACK]). If a never retransmitted segment that's below
// RACK.fack is (selectively or cumulatively) acknowledged, it has been
// delivered out of order. The sender sets RACK.reord to TRUE if such segment
// is identified.
func (rc *rackControl) detectReorder(seg *segment) {
endSeq := seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize()))
if rc.FACK.LessThan(endSeq) {
rc.FACK = endSeq
return
}
if endSeq.LessThan(rc.FACK) && seg.xmitCount == 1 {
rc.Reord = true
}
}
func (rc *rackControl) setDSACKSeen(dsackSeen bool) {
rc.DSACKSeen = dsackSeen
}
// shouldSchedulePTO dictates whether we should schedule a PTO or not.
// See https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.1.
func (s *sender) shouldSchedulePTO() bool {
// Schedule PTO only if RACK loss detection is enabled.
return s.ep.tcpRecovery&tcpip.TCPRACKLossDetection != 0 &&
// The connection supports SACK.
s.ep.SACKPermitted &&
// The connection is not in loss recovery.
(s.state != tcpip.RTORecovery && s.state != tcpip.SACKRecovery) &&
// The connection has no SACKed sequences in the SACK scoreboard.
s.ep.scoreboard.Sacked() == 0
}
// schedulePTO schedules the probe timeout as defined in
// https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.1.
//
// +checklocks:s.ep.mu
func (s *sender) schedulePTO() {
pto := time.Second
s.rtt.Lock()
if s.rtt.TCPRTTState.SRTTInited && s.rtt.TCPRTTState.SRTT > 0 {
pto = s.rtt.TCPRTTState.SRTT * 2
if s.Outstanding == 1 {
pto += wcDelayedACKTimeout
}
}
s.rtt.Unlock()
now := s.ep.stack.Clock().NowMonotonic()
if s.resendTimer.enabled() {
if now.Add(pto).After(s.resendTimer.target) {
pto = s.resendTimer.target.Sub(now)
}
s.resendTimer.disable()
}
s.probeTimer.enable(pto)
}
// probeTimerExpired is the same as TLP_send_probe() as defined in
// https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.5.2.
//
// +checklocks:s.ep.mu
func (s *sender) probeTimerExpired() tcpip.Error {
if s.probeTimer.isUninitialized() || !s.probeTimer.checkExpiration() {
return nil
}
var dataSent bool
if s.writeNext != nil && s.writeNext.xmitCount == 0 && s.Outstanding < s.SndCwnd {
dataSent = s.maybeSendSegment(s.writeNext, int(s.ep.scoreboard.SMSS()), s.SndUna.Add(s.SndWnd))
if dataSent {
s.Outstanding += s.pCount(s.writeNext, s.MaxPayloadSize)
s.updateWriteNext(s.writeNext.Next())
}
}
if !dataSent && !s.rc.tlpRxtOut {
var highestSeqXmit *segment
for highestSeqXmit = s.writeList.Front(); highestSeqXmit != nil; highestSeqXmit = highestSeqXmit.Next() {
if highestSeqXmit.xmitCount == 0 {
// Nothing in writeList is transmitted, no need to send a probe.
highestSeqXmit = nil
break
}
if highestSeqXmit.Next() == nil || highestSeqXmit.Next().xmitCount == 0 {
// Either everything in writeList has been transmitted or the next
// sequence has not been transmitted. Either way this is the highest
// sequence segment that was transmitted.
break
}
}
if highestSeqXmit != nil {
dataSent = s.maybeSendSegment(highestSeqXmit, int(s.ep.scoreboard.SMSS()), s.SndUna.Add(s.SndWnd))
if dataSent {
s.rc.tlpRxtOut = true
s.rc.tlpHighRxt = s.SndNxt
}
}
}
// Whether or not the probe was sent, the sender must arm the resend timer,
// not the probe timer. This ensures that the sender does not send repeated,
// back-to-back tail loss probes.
s.postXmit(dataSent, false /* shouldScheduleProbe */)
return nil
}
// detectTLPRecovery detects if recovery was accomplished by the loss probes
// and updates TLP state accordingly.
// See https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.6.3.
//
// +checklocks:s.ep.mu
func (s *sender) detectTLPRecovery(ack seqnum.Value, rcvdSeg *segment) {
if !(s.ep.SACKPermitted && s.rc.tlpRxtOut) {
return
}
// Step 1.
if s.isDupAck(rcvdSeg) && ack == s.rc.tlpHighRxt {
var sbAboveTLPHighRxt bool
for _, sb := range rcvdSeg.parsedOptions.SACKBlocks {
if s.rc.tlpHighRxt.LessThan(sb.End) {
sbAboveTLPHighRxt = true
break
}
}
if !sbAboveTLPHighRxt {
// TLP episode is complete.
s.rc.tlpRxtOut = false
}
}
if s.rc.tlpRxtOut && s.rc.tlpHighRxt.LessThanEq(ack) {
// TLP episode is complete.
s.rc.tlpRxtOut = false
if !checkDSACK(rcvdSeg) {
// Step 2. Either the original packet or the retransmission (in the
// form of a probe) was lost. Invoke a congestion control response
// equivalent to fast recovery.
s.cc.HandleLossDetected()
s.enterRecovery()
s.leaveRecovery()
}
}
}
// updateRACKReorderWindow updates the reorder window.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2
// - Step 4: Update RACK reordering window
// To handle the prevalent small degree of reordering, RACK.reo_wnd serves as
// an allowance for settling time before marking a packet lost. RACK starts
// initially with a conservative window of min_RTT/4. If no reordering has
// been observed RACK uses reo_wnd of zero during loss recovery, in order to
// retransmit quickly, or when the number of DUPACKs exceeds the classic
// DUPACKthreshold.
//
// +checklocks:rc.snd.ep.mu
func (rc *rackControl) updateRACKReorderWindow() {
dsackSeen := rc.DSACKSeen
snd := rc.snd
// React to DSACK once per round trip.
// If SND.UNA < RACK.rtt_seq:
// RACK.dsack = false
if snd.SndUna.LessThan(rc.RTTSeq) {
dsackSeen = false
}
// If RACK.dsack:
// RACK.reo_wnd_incr += 1
// RACK.dsack = false
// RACK.rtt_seq = SND.NXT
// RACK.reo_wnd_persist = 16
if dsackSeen {
rc.ReoWndIncr++
dsackSeen = false
rc.RTTSeq = snd.SndNxt
rc.ReoWndPersist = tcpRACKRecoveryThreshold
} else if rc.exitedRecovery {
// Else if exiting loss recovery:
// RACK.reo_wnd_persist -= 1
// If RACK.reo_wnd_persist <= 0:
// RACK.reo_wnd_incr = 1
rc.ReoWndPersist--
if rc.ReoWndPersist <= 0 {
rc.ReoWndIncr = 1
}
rc.exitedRecovery = false
}
// Reorder window is zero during loss recovery, or when the number of
// DUPACKs exceeds the classic DUPACKthreshold.
// If RACK.reord is FALSE:
// If in loss recovery: (If in fast or timeout recovery)
// RACK.reo_wnd = 0
// Return
// Else if RACK.pkts_sacked >= RACK.dupthresh:
// RACK.reo_wnd = 0
// return
if !rc.Reord {
if snd.state == tcpip.RTORecovery || snd.state == tcpip.SACKRecovery {
rc.ReoWnd = 0
return
}
if snd.SackedOut >= nDupAckThreshold {
rc.ReoWnd = 0
return
}
}
// Calculate reorder window.
// RACK.reo_wnd = RACK.min_RTT / 4 * RACK.reo_wnd_incr
// RACK.reo_wnd = min(RACK.reo_wnd, SRTT)
snd.rtt.Lock()
srtt := snd.rtt.TCPRTTState.SRTT
snd.rtt.Unlock()
rc.ReoWnd = time.Duration((int64(rc.minRTT) / 4) * int64(rc.ReoWndIncr))
if srtt < rc.ReoWnd {
rc.ReoWnd = srtt
}
}
func (rc *rackControl) exitRecovery() {
rc.exitedRecovery = true
}
// detectLoss marks the segment as lost if the reordering window has elapsed
// and the ACK is not received. It will also arm the reorder timer.
// See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.2 Step 5.
//
// +checklocks:rc.snd.ep.mu
func (rc *rackControl) detectLoss(rcvTime tcpip.MonotonicTime) int {
var timeout time.Duration
numLost := 0
for seg := rc.snd.writeList.Front(); seg != nil && seg.xmitCount != 0; seg = seg.Next() {
if rc.snd.ep.scoreboard.IsSACKED(seg.sackBlock()) {
continue
}
if seg.lost && seg.xmitCount == 1 {
numLost++
continue
}
endSeq := seg.sequenceNumber.Add(seqnum.Size(seg.payloadSize()))
if seg.xmitTime.Before(rc.XmitTime) || (seg.xmitTime == rc.XmitTime && rc.EndSequence.LessThan(endSeq)) {
timeRemaining := seg.xmitTime.Sub(rcvTime) + rc.RTT + rc.ReoWnd
if timeRemaining <= 0 {
seg.lost = true
numLost++
} else if timeRemaining > timeout {
timeout = timeRemaining
}
}
}
if timeout != 0 && !rc.snd.reorderTimer.enabled() {
rc.snd.reorderTimer.enable(timeout)
}
return numLost
}
// reorderTimerExpired will retransmit the segments which have not been acked
// before the reorder timer expired.
//
// +checklocks:rc.snd.ep.mu
func (rc *rackControl) reorderTimerExpired() tcpip.Error {
if rc.snd.reorderTimer.isUninitialized() || !rc.snd.reorderTimer.checkExpiration() {
return nil
}
numLost := rc.detectLoss(rc.snd.ep.stack.Clock().NowMonotonic())
if numLost == 0 {
return nil
}
fastRetransmit := false
if !rc.snd.FastRecovery.Active {
rc.snd.cc.HandleLossDetected()
rc.snd.enterRecovery()
fastRetransmit = true
}
rc.DoRecovery(nil, fastRetransmit)
return nil
}
// DoRecovery implements lossRecovery.DoRecovery.
//
// +checklocks:rc.snd.ep.mu
func (rc *rackControl) DoRecovery(_ *segment, fastRetransmit bool) {
snd := rc.snd
if fastRetransmit {
snd.resendSegment()
}
var dataSent bool
// Iterate the writeList and retransmit the segments which are marked
// as lost by RACK.
for seg := snd.writeList.Front(); seg != nil && seg.xmitCount > 0; seg = seg.Next() {
if seg == snd.writeNext {
break
}
if !seg.lost {
continue
}
// Reset seg.lost as it is already SACKed.
if snd.ep.scoreboard.IsSACKED(seg.sackBlock()) {
seg.lost = false
continue
}
// Check the congestion window after entering recovery.
if snd.Outstanding >= snd.SndCwnd {
break
}
if sent := snd.maybeSendSegment(seg, int(snd.ep.scoreboard.SMSS()), snd.SndUna.Add(snd.SndWnd)); !sent {
break
}
dataSent = true
snd.Outstanding += snd.pCount(seg, snd.MaxPayloadSize)
}
snd.postXmit(dataSent, true /* shouldScheduleProbe */)
}

616
pkg/tcpip/transport/tcp/rcv.go Normal file
View file

@ -0,0 +1,616 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"container/heap"
"math"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
)
// receiver holds the state necessary to receive TCP segments and turn them
// into a stream of bytes.
//
// +stateify savable
type receiver struct {
TCPReceiverState
ep *Endpoint
// rcvWnd is the non-scaled receive window last advertised to the peer.
rcvWnd seqnum.Size
// rcvWUP is the RcvNxt value at the last window update sent.
rcvWUP seqnum.Value
// prevBufused is the snapshot of endpoint rcvBufUsed taken when we
// advertise a receive window.
prevBufUsed int
closed bool
// pendingRcvdSegments is bounded by the receive buffer size of the
// endpoint.
pendingRcvdSegments segmentHeap
// Time when the last ack was received.
lastRcvdAckTime tcpip.MonotonicTime
}
func newReceiver(ep *Endpoint, irs seqnum.Value, rcvWnd seqnum.Size, rcvWndScale uint8) *receiver {
return &receiver{
ep: ep,
TCPReceiverState: TCPReceiverState{
RcvNxt: irs + 1,
RcvAcc: irs.Add(rcvWnd + 1),
RcvWndScale: rcvWndScale,
},
rcvWnd: rcvWnd,
rcvWUP: irs + 1,
lastRcvdAckTime: ep.stack.Clock().NowMonotonic(),
}
}
// acceptable checks if the segment sequence number range is acceptable
// according to the table on page 26 of RFC 793.
func (r *receiver) acceptable(segSeq seqnum.Value, segLen seqnum.Size) bool {
// r.rcvWnd could be much larger than the window size we advertised in our
// outgoing packets, we should use what we have advertised for acceptability
// test.
scaledWindowSize := r.rcvWnd >> r.RcvWndScale
if scaledWindowSize > math.MaxUint16 {
// This is what we actually put in the Window field.
scaledWindowSize = math.MaxUint16
}
advertisedWindowSize := scaledWindowSize << r.RcvWndScale
return header.Acceptable(segSeq, segLen, r.RcvNxt, r.RcvNxt.Add(advertisedWindowSize))
}
// currentWindow returns the available space in the window that was advertised
// last to our peer.
func (r *receiver) currentWindow() (curWnd seqnum.Size) {
endOfWnd := r.rcvWUP.Add(r.rcvWnd)
if endOfWnd.LessThan(r.RcvNxt) {
// return 0 if r.RcvNxt is past the end of the previously advertised window.
// This can happen because we accept a large segment completely even if
// accepting it causes it to partially exceed the advertised window.
return 0
}
return r.RcvNxt.Size(endOfWnd)
}
// getSendParams returns the parameters needed by the sender when building
// segments to send.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) getSendParams() (RcvNxt seqnum.Value, rcvWnd seqnum.Size) {
newWnd := r.ep.selectWindow()
curWnd := r.currentWindow()
unackLen := int(r.ep.snd.MaxSentAck.Size(r.RcvNxt))
bufUsed := r.ep.receiveBufferUsed()
// Grow the right edge of the window only for payloads larger than the
// the segment overhead OR if the application is actively consuming data.
//
// Avoiding growing the right edge otherwise, addresses a situation below:
// An application has been slow in reading data and we have burst of
// incoming segments lengths < segment overhead. Here, our available free
// memory would reduce drastically when compared to the advertised receive
// window.
//
// For example: With incoming 512 bytes segments, segment overhead of
// 552 bytes (at the time of writing this comment), with receive window
// starting from 1MB and with rcvAdvWndScale being 1, buffer would reach 0
// when the curWnd is still 19436 bytes, because for every incoming segment
// newWnd would reduce by (552+512) >> rcvAdvWndScale (current value 1),
// while curWnd would reduce by 512 bytes.
// Such a situation causes us to keep tail dropping the incoming segments
// and never advertise zero receive window to the peer.
//
// Linux does a similar check for minimal sk_buff size (128):
// https://github.com/torvalds/linux/blob/d5beb3140f91b1c8a3d41b14d729aefa4dcc58bc/net/ipv4/tcp_input.c#L783
//
// Also, if the application is reading the data, we keep growing the right
// edge, as we are still advertising a window that we think can be serviced.
toGrow := unackLen >= SegOverheadSize || bufUsed <= r.prevBufUsed
// Update RcvAcc only if new window is > previously advertised window. We
// should never shrink the acceptable sequence space once it has been
// advertised the peer. If we shrink the acceptable sequence space then we
// would end up dropping bytes that might already be in flight.
// ==================================================== sequence space.
// ^ ^ ^ ^
// rcvWUP RcvNxt RcvAcc new RcvAcc
// <=====curWnd ===>
// <========= newWnd > curWnd ========= >
if r.RcvNxt.Add(curWnd).LessThan(r.RcvNxt.Add(newWnd)) && toGrow {
// If the new window moves the right edge, then update RcvAcc.
r.RcvAcc = r.RcvNxt.Add(newWnd)
} else {
if newWnd == 0 {
// newWnd is zero but we can't advertise a zero as it would cause window
// to shrink so just increment a metric to record this event.
r.ep.stats.ReceiveErrors.WantZeroRcvWindow.Increment()
}
newWnd = curWnd
}
// Apply silly-window avoidance when recovering from zero-window situation.
// Keep advertising zero receive window up until the new window reaches a
// threshold.
if r.rcvWnd == 0 && newWnd != 0 {
r.ep.rcvQueueMu.Lock()
if crossed, above := r.ep.windowCrossedACKThresholdLocked(int(newWnd), int(r.ep.ops.GetReceiveBufferSize())); !crossed && !above {
newWnd = 0
}
r.ep.rcvQueueMu.Unlock()
}
// Stash away the non-scaled receive window as we use it for measuring
// receiver's estimated RTT.
r.rcvWnd = newWnd
r.rcvWUP = r.RcvNxt
r.prevBufUsed = bufUsed
scaledWnd := r.rcvWnd >> r.RcvWndScale
if scaledWnd == 0 {
// Increment a metric if we are advertising an actual zero window.
r.ep.stats.ReceiveErrors.ZeroRcvWindowState.Increment()
}
// If we started off with a window larger than what can he held in
// the 16bit window field, we ceil the value to the max value.
if scaledWnd > math.MaxUint16 {
scaledWnd = seqnum.Size(math.MaxUint16)
// Ensure that the stashed receive window always reflects what
// is being advertised.
r.rcvWnd = scaledWnd << r.RcvWndScale
}
return r.RcvNxt, scaledWnd
}
// nonZeroWindow is called when the receive window grows from zero to nonzero;
// in such cases we may need to send an ack to indicate to our peer that it can
// resume sending data.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) nonZeroWindow() {
// Immediately send an ack.
r.ep.snd.sendAck()
}
// consumeSegment attempts to consume a segment that was received by r. The
// segment may have just been received or may have been received earlier but
// wasn't ready to be consumed then.
//
// Returns true if the segment was consumed, false if it cannot be consumed
// yet because of a missing segment.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) consumeSegment(s *segment, segSeq seqnum.Value, segLen seqnum.Size) bool {
if segLen > 0 {
// If the segment doesn't include the seqnum we're expecting to
// consume now, we're missing a segment. We cannot proceed until
// we receive that segment though.
if !r.RcvNxt.InWindow(segSeq, segLen) {
return false
}
// Trim segment to eliminate already acknowledged data.
if segSeq.LessThan(r.RcvNxt) {
diff := segSeq.Size(r.RcvNxt)
segLen -= diff
segSeq.UpdateForward(diff)
s.sequenceNumber.UpdateForward(diff)
s.TrimFront(diff)
}
// Move segment to ready-to-deliver list. Wakeup any waiters.
r.ep.readyToRead(s)
} else if segSeq != r.RcvNxt {
return false
}
// Update the segment that we're expecting to consume.
r.RcvNxt = segSeq.Add(segLen)
// In cases of a misbehaving sender which could send more than the
// advertised window, we could end up in a situation where we get a
// segment that exceeds the window advertised. Instead of partially
// accepting the segment and discarding bytes beyond the advertised
// window, we accept the whole segment and make sure r.RcvAcc is moved
// forward to match r.RcvNxt to indicate that the window is now closed.
//
// In absence of this check the r.acceptable() check fails and accepts
// segments that should be dropped because rcvWnd is calculated as
// the size of the interval (RcvNxt, RcvAcc] which becomes extremely
// large if RcvAcc is ever less than RcvNxt.
if r.RcvAcc.LessThan(r.RcvNxt) {
r.RcvAcc = r.RcvNxt
}
// Trim SACK Blocks to remove any SACK information that covers
// sequence numbers that have been consumed.
TrimSACKBlockList(&r.ep.sack, r.RcvNxt)
// Handle FIN or FIN-ACK.
if s.flags.Contains(header.TCPFlagFin) {
r.RcvNxt++
// Send ACK immediately.
r.ep.snd.sendAck()
// Tell any readers that no more data will come.
r.closed = true
r.ep.readyToRead(nil)
// We just received a FIN, our next state depends on whether we sent a
// FIN already or not.
switch r.ep.EndpointState() {
case StateEstablished:
r.ep.setEndpointState(StateCloseWait)
case StateFinWait1:
if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt {
// FIN-ACK, transition to TIME-WAIT.
r.ep.setEndpointState(StateTimeWait)
} else {
// Simultaneous close, expecting a final ACK.
r.ep.setEndpointState(StateClosing)
}
case StateFinWait2:
r.ep.setEndpointState(StateTimeWait)
}
// Flush out any pending segments, except the very first one if
// it happens to be the one we're handling now because the
// caller is using it.
first := 0
if len(r.pendingRcvdSegments) != 0 && r.pendingRcvdSegments[0] == s {
first = 1
}
for i := first; i < len(r.pendingRcvdSegments); i++ {
r.PendingBufUsed -= r.pendingRcvdSegments[i].segMemSize()
r.pendingRcvdSegments[i].DecRef()
// Note that slice truncation does not allow garbage
// collection of truncated items, thus truncated items
// must be set to nil to avoid memory leaks.
r.pendingRcvdSegments[i] = nil
}
r.pendingRcvdSegments = r.pendingRcvdSegments[:first]
r.ep.updateConnDirectionState(connDirectionStateRcvClosed)
return true
}
// Handle ACK (not FIN-ACK, which we handled above) during one of the
// shutdown states.
if s.flags.Contains(header.TCPFlagAck) && s.ackNumber == r.ep.snd.SndNxt {
switch r.ep.EndpointState() {
case StateFinWait1:
r.ep.setEndpointState(StateFinWait2)
if e := r.ep; e.closed {
// The socket has been closed and we are in
// FIN-WAIT-2 so start the FIN-WAIT-2 timer.
e.finWait2Timer = e.stack.Clock().AfterFunc(e.tcpLingerTimeout, e.finWait2TimerExpired)
}
case StateClosing:
r.ep.setEndpointState(StateTimeWait)
case StateLastAck:
r.ep.transitionToStateCloseLocked()
}
}
return true
}
// updateRTT updates the receiver RTT measurement based on the sequence number
// of the received segment.
func (r *receiver) updateRTT() {
// From: https://public.lanl.gov/radiant/pubs/drs/sc2001-poster.pdf
//
// A system that is only transmitting acknowledgements can still
// estimate the round-trip time by observing the time between when a byte
// is first acknowledged and the receipt of data that is at least one
// window beyond the sequence number that was acknowledged.
r.ep.rcvQueueMu.Lock()
if r.ep.RcvAutoParams.RTTMeasureTime == (tcpip.MonotonicTime{}) {
// New measurement.
r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic()
r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd)
r.ep.rcvQueueMu.Unlock()
return
}
if r.RcvNxt.LessThan(r.ep.RcvAutoParams.RTTMeasureSeqNumber) {
r.ep.rcvQueueMu.Unlock()
return
}
rtt := r.ep.stack.Clock().NowMonotonic().Sub(r.ep.RcvAutoParams.RTTMeasureTime)
// We only store the minimum observed RTT here as this is only used in
// absence of a SRTT available from either timestamps or a sender
// measurement of RTT.
if r.ep.RcvAutoParams.RTT == 0 || rtt < r.ep.RcvAutoParams.RTT {
r.ep.RcvAutoParams.RTT = rtt
}
r.ep.RcvAutoParams.RTTMeasureTime = r.ep.stack.Clock().NowMonotonic()
r.ep.RcvAutoParams.RTTMeasureSeqNumber = r.RcvNxt.Add(r.rcvWnd)
r.ep.rcvQueueMu.Unlock()
}
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) handleRcvdSegmentClosing(s *segment, state EndpointState, closed bool) (drop bool, err tcpip.Error) {
r.ep.rcvQueueMu.Lock()
rcvClosed := r.ep.RcvClosed || r.closed
r.ep.rcvQueueMu.Unlock()
// If we are in one of the shutdown states then we need to do
// additional checks before we try and process the segment.
switch state {
case StateCloseWait, StateClosing, StateLastAck:
if !s.sequenceNumber.LessThanEq(r.RcvNxt) {
// Just drop the segment as we have
// already received a FIN and this
// segment is after the sequence number
// for the FIN.
return true, nil
}
fallthrough
case StateFinWait1, StateFinWait2:
// If the ACK acks something not yet sent then we send an ACK.
//
// RFC793, page 37: If the connection is in a synchronized state,
// (ESTABLISHED, FIN-WAIT-1, FIN-WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK,
// TIME-WAIT), any unacceptable segment (out of window sequence number
// or unacceptable acknowledgment number) must elicit only an empty
// acknowledgment segment containing the current send-sequence number
// and an acknowledgment indicating the next sequence number expected
// to be received, and the connection remains in the same state.
//
// Just as on Linux, we do not apply this behavior when state is
// ESTABLISHED.
// Linux receive processing for all states except ESTABLISHED and
// TIME_WAIT is here where if the ACK check fails, we attempt to
// reply back with an ACK with correct seq/ack numbers.
// https://github.com/torvalds/linux/blob/v5.8/net/ipv4/tcp_input.c#L6186
// The ESTABLISHED state processing is here where if the ACK check
// fails, we ignore the packet:
// https://github.com/torvalds/linux/blob/v5.8/net/ipv4/tcp_input.c#L5591
if r.ep.snd.SndNxt.LessThan(s.ackNumber) {
r.ep.snd.maybeSendOutOfWindowAck(s)
return true, nil
}
// If we are closed for reads (either due to an
// incoming FIN or the user calling shutdown(..,
// SHUT_RD) then any data past the RcvNxt should
// trigger a RST.
endDataSeq := s.sequenceNumber.Add(seqnum.Size(s.payloadSize()))
if state != StateCloseWait && rcvClosed && r.RcvNxt.LessThan(endDataSeq) {
return true, &tcpip.ErrConnectionAborted{}
}
if state == StateFinWait1 {
break
}
// If it's a retransmission of an old data segment
// or a pure ACK then allow it.
if s.sequenceNumber.Add(s.logicalLen()).LessThanEq(r.RcvNxt) ||
s.logicalLen() == 0 {
break
}
// In FIN-WAIT2 if the socket is fully
// closed(not owned by application on our end
// then the only acceptable segment is a
// FIN. Since FIN can technically also carry
// data we verify that the segment carrying a
// FIN ends at exactly e.RcvNxt+1.
//
// From RFC793 page 25.
//
// For sequence number purposes, the SYN is
// considered to occur before the first actual
// data octet of the segment in which it occurs,
// while the FIN is considered to occur after
// the last actual data octet in a segment in
// which it occurs.
if closed && (!s.flags.Contains(header.TCPFlagFin) || s.sequenceNumber.Add(s.logicalLen()) != r.RcvNxt+1) {
return true, &tcpip.ErrConnectionAborted{}
}
}
// We don't care about receive processing anymore if the receive side
// is closed.
//
// NOTE: We still want to permit a FIN as it's possible only our
// end has closed and the peer is yet to send a FIN. Hence we
// compare only the payload.
segEnd := s.sequenceNumber.Add(seqnum.Size(s.payloadSize()))
if rcvClosed && !segEnd.LessThanEq(r.RcvNxt) {
return true, nil
}
return false, nil
}
// handleRcvdSegment handles TCP segments directed at the connection managed by
// r as they arrive. It is called by the protocol main loop.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) handleRcvdSegment(s *segment) (drop bool, err tcpip.Error) {
state := r.ep.EndpointState()
closed := r.ep.closed
segLen := seqnum.Size(s.payloadSize())
segSeq := s.sequenceNumber
// If the sequence number range is outside the acceptable range, just
// send an ACK and stop further processing of the segment.
// This is according to RFC 793, page 68.
if !r.acceptable(segSeq, segLen) {
r.ep.snd.maybeSendOutOfWindowAck(s)
return true, nil
}
if state != StateEstablished {
drop, err := r.handleRcvdSegmentClosing(s, state, closed)
if drop || err != nil {
return drop, err
}
}
// Store the time of the last ack.
r.lastRcvdAckTime = r.ep.stack.Clock().NowMonotonic()
// Defer segment processing if it can't be consumed now.
if !r.consumeSegment(s, segSeq, segLen) {
if segLen > 0 || s.flags.Contains(header.TCPFlagFin) {
// We only store the segment if it's within our buffer
// size limit.
//
// Only use 75% of the receive buffer queue for
// out-of-order segments. This ensures that we always
// leave some space for the inorder segments to arrive
// allowing pending segments to be processed and
// delivered to the user.
//
// The ratio must be at least 50% (the size of rwnd) to
// leave space for retransmitted dropped packets. 51%
// would make recovery slow when there are multiple
// drops by necessitating multiple round trips. 100%
// would enable the buffer to be totally full of
// out-of-order data and stall the connection.
//
// An ideal solution is to ensure that there are at
// least N bytes free when N bytes are missing, but we
// don't have that computed at this point in the stack.
if rcvBufSize := r.ep.ops.GetReceiveBufferSize(); rcvBufSize > 0 && (r.PendingBufUsed+int(segLen)) < int(rcvBufSize-rcvBufSize/4) {
r.ep.rcvQueueMu.Lock()
r.PendingBufUsed += s.segMemSize()
r.ep.rcvQueueMu.Unlock()
s.IncRef()
heap.Push(&r.pendingRcvdSegments, s)
UpdateSACKBlocks(&r.ep.sack, segSeq, segSeq.Add(segLen), r.RcvNxt)
}
// Immediately send an ack so that the peer knows it may
// have to retransmit.
r.ep.snd.sendAck()
}
return false, nil
}
// Since we consumed a segment update the receiver's RTT estimate
// if required.
if segLen > 0 {
r.updateRTT()
}
// By consuming the current segment, we may have filled a gap in the
// sequence number domain that allows pending segments to be consumed
// now. So try to do it.
for !r.closed && r.pendingRcvdSegments.Len() > 0 {
s := r.pendingRcvdSegments[0]
segLen := seqnum.Size(s.payloadSize())
segSeq := s.sequenceNumber
// Skip segment altogether if it has already been acknowledged.
if !segSeq.Add(segLen-1).LessThan(r.RcvNxt) &&
!r.consumeSegment(s, segSeq, segLen) {
break
}
heap.Pop(&r.pendingRcvdSegments)
r.ep.rcvQueueMu.Lock()
r.PendingBufUsed -= s.segMemSize()
r.ep.rcvQueueMu.Unlock()
s.DecRef()
}
return false, nil
}
// handleTimeWaitSegment handles inbound segments received when the endpoint
// has entered the TIME_WAIT state.
// +checklocks:r.ep.mu
// +checklocksalias:r.ep.snd.ep.mu=r.ep.mu
func (r *receiver) handleTimeWaitSegment(s *segment) (resetTimeWait bool, newSyn bool) {
segSeq := s.sequenceNumber
segLen := seqnum.Size(s.payloadSize())
// Just silently drop any RST packets in TIME_WAIT. We do not support
// TIME_WAIT assassination as a result we confirm w/ fix 1 as described
// in https://tools.ietf.org/html/rfc1337#section-3.
//
// This behavior overrides RFC793 page 70 where we transition to CLOSED
// on receiving RST, which is also default Linux behavior.
// On Linux the RST can be ignored by setting sysctl net.ipv4.tcp_rfc1337.
//
// As we do not yet support PAWS, we are being conservative in ignoring
// RSTs by default.
if s.flags.Contains(header.TCPFlagRst) {
return false, false
}
// If it's a SYN and the sequence number is higher than any seen before
// for this connection then try and redirect it to a listening endpoint
// if available.
//
// RFC 1122:
// "When a connection is [...] on TIME-WAIT state [...]
// [a TCP] MAY accept a new SYN from the remote TCP to
// reopen the connection directly, if it:
// (1) assigns its initial sequence number for the new
// connection to be larger than the largest sequence
// number it used on the previous connection incarnation,
// and
// (2) returns to TIME-WAIT state if the SYN turns out
// to be an old duplicate".
if s.flags.Contains(header.TCPFlagSyn) && r.RcvNxt.LessThan(segSeq) {
return false, true
}
// Drop the segment if it does not contain an ACK.
if !s.flags.Contains(header.TCPFlagAck) {
return false, false
}
// Update Timestamp if required. See RFC7323, section-4.3.
if r.ep.SendTSOk && s.parsedOptions.TS {
r.ep.updateRecentTimestamp(s.parsedOptions.TSVal, r.ep.snd.MaxSentAck, segSeq)
}
if segSeq.Add(1) == r.RcvNxt && s.flags.Contains(header.TCPFlagFin) {
// If it's a FIN-ACK then resetTimeWait and send an ACK, as it
// indicates our final ACK could have been lost.
r.ep.snd.sendAck()
return true, false
}
// If the sequence number range is outside the acceptable range or
// carries data then just send an ACK. This is according to RFC 793,
// page 37.
//
// NOTE: In TIME_WAIT the only acceptable sequence number is RcvNxt.
if segSeq != r.RcvNxt || segLen != 0 {
r.ep.snd.sendAck()
}
return false, false
}

64
pkg/tcpip/transport/tcp/rcv_queue_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type rcvQueueMutex struct {
mu sync.Mutex
}
var rcvQueueprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var rcvQueuelockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type rcvQueuelockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *rcvQueueMutex) Lock() {
locking.AddGLock(rcvQueueprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *rcvQueueMutex) NestedLock(i rcvQueuelockNameIndex) {
locking.AddGLock(rcvQueueprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *rcvQueueMutex) Unlock() {
locking.DelGLock(rcvQueueprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *rcvQueueMutex) NestedUnlock(i rcvQueuelockNameIndex) {
locking.DelGLock(rcvQueueprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func rcvQueueinitLockNames() {}
func init() {
rcvQueueinitLockNames()
rcvQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(rcvQueueMutex{}), rcvQueuelockNames)
}

118
pkg/tcpip/transport/tcp/reno.go Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"time"
)
// renoState stores the variables related to TCP New Reno congestion
// control algorithm.
//
// +stateify savable
type renoState struct {
s *sender
}
// newRenoCC initializes the state for the NewReno congestion control algorithm.
func newRenoCC(s *sender) *renoState {
return &renoState{s: s}
}
// updateSlowStart will update the congestion window as per the slow-start
// algorithm used by NewReno. If after adjusting the congestion window
// we cross the SSthreshold then it will return the number of packets that
// must be consumed in congestion avoidance mode.
//
// +checklocks:r.s.ep.mu
func (r *renoState) updateSlowStart(packetsAcked int) int {
// Don't let the congestion window cross into the congestion
// avoidance range.
newcwnd := r.s.SndCwnd + packetsAcked
if newcwnd >= r.s.Ssthresh {
newcwnd = r.s.Ssthresh
r.s.SndCAAckCount = 0
}
packetsAcked -= newcwnd - r.s.SndCwnd
r.s.SndCwnd = newcwnd
return packetsAcked
}
// updateCongestionAvoidance will update congestion window in congestion
// avoidance mode as described in RFC5681 section 3.1
//
// +checklocks:r.s.ep.mu
func (r *renoState) updateCongestionAvoidance(packetsAcked int) {
// Consume the packets in congestion avoidance mode.
r.s.SndCAAckCount += packetsAcked
if r.s.SndCAAckCount >= r.s.SndCwnd {
r.s.SndCwnd += r.s.SndCAAckCount / r.s.SndCwnd
r.s.SndCAAckCount = r.s.SndCAAckCount % r.s.SndCwnd
}
}
// reduceSlowStartThreshold reduces the slow-start threshold per RFC 5681,
// page 6, eq. 4. It is called when we detect congestion in the network.
//
// +checklocks:r.s.ep.mu
func (r *renoState) reduceSlowStartThreshold() {
r.s.Ssthresh = r.s.Outstanding / 2
if r.s.Ssthresh < 2 {
r.s.Ssthresh = 2
}
}
// Update updates the congestion state based on the number of packets that
// were acknowledged.
// Update implements congestionControl.Update.
//
// +checklocks:r.s.ep.mu
func (r *renoState) Update(packetsAcked int, _ time.Duration) {
if r.s.SndCwnd < r.s.Ssthresh {
packetsAcked = r.updateSlowStart(packetsAcked)
if packetsAcked == 0 {
return
}
}
r.updateCongestionAvoidance(packetsAcked)
}
// HandleLossDetected implements congestionControl.HandleLossDetected.
//
// +checklocks:r.s.ep.mu
func (r *renoState) HandleLossDetected() {
// A retransmit was triggered due to nDupAckThreshold or when RACK
// detected loss. Reduce our slow start threshold.
r.reduceSlowStartThreshold()
}
// HandleRTOExpired implements congestionControl.HandleRTOExpired.
//
// +checklocks:r.s.ep.mu
func (r *renoState) HandleRTOExpired() {
// We lost a packet, so reduce ssthresh.
r.reduceSlowStartThreshold()
// Reduce the congestion window to 1, i.e., enter slow-start. Per
// RFC 5681, page 7, we must use 1 regardless of the value of the
// initial congestion window.
r.s.SndCwnd = 1
}
// PostRecovery implements congestionControl.PostRecovery.
func (r *renoState) PostRecovery() {
// noop.
}

68
pkg/tcpip/transport/tcp/reno_recovery.go Normal file
View file

@ -0,0 +1,68 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
// renoRecovery stores the variables related to TCP Reno loss recovery
// algorithm.
//
// +stateify savable
type renoRecovery struct {
s *sender
}
func newRenoRecovery(s *sender) *renoRecovery {
return &renoRecovery{s: s}
}
// +checklocks:rr.s.ep.mu
func (rr *renoRecovery) DoRecovery(rcvdSeg *segment, fastRetransmit bool) {
ack := rcvdSeg.ackNumber
snd := rr.s
// We are in fast recovery mode. Ignore the ack if it's out of range.
if !ack.InRange(snd.SndUna, snd.SndNxt+1) {
return
}
// Don't count this as a duplicate if it is carrying data or
// updating the window.
if rcvdSeg.logicalLen() != 0 || snd.SndWnd != rcvdSeg.window {
return
}
// Inflate the congestion window if we're getting duplicate acks
// for the packet we retransmitted.
if !fastRetransmit && ack == snd.FastRecovery.First {
// We received a dup, inflate the congestion window by 1 packet
// if we're not at the max yet. Only inflate the window if
// regular FastRecovery is in use, RFC6675 does not require
// inflating cwnd on duplicate ACKs.
if snd.SndCwnd < snd.FastRecovery.MaxCwnd {
snd.SndCwnd++
}
return
}
// A partial ack was received. Retransmit this packet and remember it
// so that we don't retransmit it again.
//
// We don't inflate the window because we're putting the same packet
// back onto the wire.
//
// N.B. The retransmit timer will be reset by the caller.
snd.FastRecovery.First = ack
snd.DupAckCount = 0
snd.resendSegment()
}

64
pkg/tcpip/transport/tcp/rtt_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type rttMutex struct {
mu sync.Mutex
}
var rttprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var rttlockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type rttlockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *rttMutex) Lock() {
locking.AddGLock(rttprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *rttMutex) NestedLock(i rttlockNameIndex) {
locking.AddGLock(rttprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *rttMutex) Unlock() {
locking.DelGLock(rttprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *rttMutex) NestedUnlock(i rttlockNameIndex) {
locking.DelGLock(rttprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func rttinitLockNames() {}
func init() {
rttinitLockNames()
rttprefixIndex = locking.NewMutexClass(reflect.TypeOf(rttMutex{}), rttlockNames)
}

105
pkg/tcpip/transport/tcp/sack.go Normal file
View file

@ -0,0 +1,105 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
)
const (
// MaxSACKBlocks is the maximum number of SACK blocks stored
// at receiver side.
MaxSACKBlocks = 6
)
// UpdateSACKBlocks updates the list of SACK blocks to include the segment
// specified by segStart->segEnd. If the segment happens to be an out of order
// delivery then the first block in the sack.blocks always includes the
// segment identified by segStart->segEnd.
func UpdateSACKBlocks(sack *SACKInfo, segStart seqnum.Value, segEnd seqnum.Value, rcvNxt seqnum.Value) {
newSB := header.SACKBlock{Start: segStart, End: segEnd}
// Ignore any invalid SACK blocks or blocks that are before rcvNxt as
// those bytes have already been acked.
if newSB.End.LessThanEq(newSB.Start) || newSB.End.LessThan(rcvNxt) {
return
}
if sack.NumBlocks == 0 {
sack.Blocks[0] = newSB
sack.NumBlocks = 1
return
}
n := 0
for i := 0; i < sack.NumBlocks; i++ {
start, end := sack.Blocks[i].Start, sack.Blocks[i].End
if end.LessThanEq(rcvNxt) {
// Discard any sack blocks that are before rcvNxt as
// those have already been acked.
continue
}
if newSB.Start.LessThanEq(end) && start.LessThanEq(newSB.End) {
// Merge this SACK block into newSB and discard this SACK
// block.
if start.LessThan(newSB.Start) {
newSB.Start = start
}
if newSB.End.LessThan(end) {
newSB.End = end
}
} else {
// Save this block.
sack.Blocks[n] = sack.Blocks[i]
n++
}
}
if rcvNxt.LessThan(newSB.Start) {
// If this was an out of order segment then make sure that the
// first SACK block is the one that includes the segment.
//
// See the first bullet point in
// https://tools.ietf.org/html/rfc2018#section-4
if n == MaxSACKBlocks {
// If the number of SACK blocks is equal to
// MaxSACKBlocks then discard the last SACK block.
n--
}
for i := n - 1; i >= 0; i-- {
sack.Blocks[i+1] = sack.Blocks[i]
}
sack.Blocks[0] = newSB
n++
}
sack.NumBlocks = n
}
// TrimSACKBlockList updates the sack block list by removing/modifying any block
// where start is < rcvNxt.
func TrimSACKBlockList(sack *SACKInfo, rcvNxt seqnum.Value) {
n := 0
for i := 0; i < sack.NumBlocks; i++ {
if sack.Blocks[i].End.LessThanEq(rcvNxt) {
continue
}
if sack.Blocks[i].Start.LessThan(rcvNxt) {
// Shrink this SACK block.
sack.Blocks[i].Start = rcvNxt
}
sack.Blocks[n] = sack.Blocks[i]
n++
}
sack.NumBlocks = n
}

122
pkg/tcpip/transport/tcp/sack_recovery.go Normal file
View file

@ -0,0 +1,122 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import "github.com/sagernet/gvisor/pkg/tcpip/seqnum"
// sackRecovery stores the variables related to TCP SACK loss recovery
// algorithm.
//
// +stateify savable
type sackRecovery struct {
s *sender
}
func newSACKRecovery(s *sender) *sackRecovery {
return &sackRecovery{s: s}
}
// handleSACKRecovery implements the loss recovery phase as described in RFC6675
// section 5, step C.
// +checklocks:sr.s.ep.mu
func (sr *sackRecovery) handleSACKRecovery(limit int, end seqnum.Value) (dataSent bool) {
snd := sr.s
snd.SetPipe()
if smss := int(snd.ep.scoreboard.SMSS()); limit > smss {
// Cap segment size limit to s.smss as SACK recovery requires
// that all retransmissions or new segments send during recovery
// be of <= SMSS.
limit = smss
}
nextSegHint := snd.writeList.Front()
for snd.Outstanding < snd.SndCwnd {
var nextSeg *segment
var rescueRtx bool
nextSeg, nextSegHint, rescueRtx = snd.NextSeg(nextSegHint)
if nextSeg == nil {
return dataSent
}
if !snd.isAssignedSequenceNumber(nextSeg) || snd.SndNxt.LessThanEq(nextSeg.sequenceNumber) {
// New data being sent.
// Step C.3 described below is handled by
// maybeSendSegment which increments sndNxt when
// a segment is transmitted.
//
// Step C.3 "If any of the data octets sent in
// (C.1) are above HighData, HighData must be
// updated to reflect the transmission of
// previously unsent data."
//
// We pass s.smss as the limit as the Step 2) requires that
// new data sent should be of size s.smss or less.
if sent := snd.maybeSendSegment(nextSeg, limit, end); !sent {
return dataSent
}
dataSent = true
snd.Outstanding++
snd.updateWriteNext(nextSeg.Next())
continue
}
// Now handle the retransmission case where we matched either step 1,3 or 4
// of the NextSeg algorithm.
// RFC 6675, Step C.4.
//
// "The estimate of the amount of data outstanding in the network
// must be updated by incrementing pipe by the number of octets
// transmitted in (C.1)."
snd.Outstanding++
dataSent = true
snd.sendSegment(nextSeg)
segEnd := nextSeg.sequenceNumber.Add(nextSeg.logicalLen())
if rescueRtx {
// We do the last part of rule (4) of NextSeg here to update
// RescueRxt as until this point we don't know if we are going
// to use the rescue transmission.
snd.FastRecovery.RescueRxt = snd.FastRecovery.Last
} else {
// RFC 6675, Step C.2
//
// "If any of the data octets sent in (C.1) are below
// HighData, HighRxt MUST be set to the highest sequence
// number of the retransmitted segment unless NextSeg ()
// rule (4) was invoked for this retransmission."
snd.FastRecovery.HighRxt = segEnd - 1
}
}
return dataSent
}
// +checklocks:sr.s.ep.mu
func (sr *sackRecovery) DoRecovery(rcvdSeg *segment, fastRetransmit bool) {
snd := sr.s
if fastRetransmit {
snd.resendSegment()
}
// We are in fast recovery mode. Ignore the ack if it's out of range.
if ack := rcvdSeg.ackNumber; !ack.InRange(snd.SndUna, snd.SndNxt+1) {
return
}
// RFC 6675 recovery algorithm step C 1-5.
end := snd.SndUna.Add(snd.SndWnd)
dataSent := sr.handleSACKRecovery(snd.MaxPayloadSize, end)
snd.postXmit(dataSent, true /* shouldScheduleProbe */)
}

306
pkg/tcpip/transport/tcp/sack_scoreboard.go Normal file
View file

@ -0,0 +1,306 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"fmt"
"strings"
"github.com/google/btree"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
)
const (
// maxSACKBlocks is the maximum number of distinct SACKBlocks the
// scoreboard will track. Once there are 100 distinct blocks, new
// insertions will fail.
maxSACKBlocks = 100
// defaultBtreeDegree is set to 2 as btree.New(2) results in a 2-3-4
// tree.
defaultBtreeDegree = 2
)
// SACKScoreboard stores a set of disjoint SACK ranges.
//
// +stateify savable
type SACKScoreboard struct {
// smss is defined in RFC5681 as following:
//
// The SMSS is the size of the largest segment that the sender can
// transmit. This value can be based on the maximum transmission unit
// of the network, the path MTU discovery [RFC1191, RFC4821] algorithm,
// RMSS (see next item), or other factors. The size does not include
// the TCP/IP headers and options.
smss uint16
maxSACKED seqnum.Value
sacked seqnum.Size `state:"nosave"`
ranges *btree.BTree `state:"nosave"`
}
// NewSACKScoreboard returns a new SACK Scoreboard.
func NewSACKScoreboard(smss uint16, iss seqnum.Value) *SACKScoreboard {
return &SACKScoreboard{
smss: smss,
ranges: btree.New(defaultBtreeDegree),
maxSACKED: iss,
}
}
// Reset erases all known range information from the SACK scoreboard.
func (s *SACKScoreboard) Reset() {
s.ranges = btree.New(defaultBtreeDegree)
s.sacked = 0
}
// Insert inserts/merges the provided SACKBlock into the scoreboard.
func (s *SACKScoreboard) Insert(r header.SACKBlock) {
if s.ranges.Len() >= maxSACKBlocks {
return
}
// Check if we can merge the new range with a range before or after it.
var toDelete []btree.Item
if s.maxSACKED.LessThan(r.End - 1) {
s.maxSACKED = r.End - 1
}
s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool {
if i == r {
return true
}
sacked := i.(header.SACKBlock)
// There is a hole between these two SACK blocks, so we can't
// merge anymore.
if r.End.LessThan(sacked.Start) {
return false
}
// There is some overlap at this point, merge the blocks and
// delete the other one.
//
// ----sS--------sE
// r.S---------------rE
// -------sE
if sacked.End.LessThan(r.End) {
// sacked is contained in the newly inserted range.
// Delete this block.
toDelete = append(toDelete, i)
return true
}
// sacked covers a range past end of the newly inserted
// block.
r.End = sacked.End
toDelete = append(toDelete, i)
return true
})
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
if i == r {
return true
}
sacked := i.(header.SACKBlock)
// sA------sE
// rA----rE
if sacked.End.LessThan(r.Start) {
return false
}
// The previous range extends into the current block. Merge it
// into the newly inserted range and delete the other one.
//
// <-rA---rE----<---rE--->
// sA--------------sE
r.Start = sacked.Start
// Extend r to cover sacked if sacked extends past r.
if r.End.LessThan(sacked.End) {
r.End = sacked.End
}
toDelete = append(toDelete, i)
return true
})
for _, i := range toDelete {
if sb := s.ranges.Delete(i); sb != nil {
sb := i.(header.SACKBlock)
s.sacked -= sb.Start.Size(sb.End)
}
}
replaced := s.ranges.ReplaceOrInsert(r)
if replaced == nil {
s.sacked += r.Start.Size(r.End)
}
}
// IsSACKED returns true if the a given range of sequence numbers denoted by r
// are already covered by SACK information in the scoreboard.
func (s *SACKScoreboard) IsSACKED(r header.SACKBlock) bool {
if s.Empty() {
return false
}
found := false
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
sacked := i.(header.SACKBlock)
if sacked.End.LessThan(r.Start) {
return false
}
if sacked.Contains(r) {
found = true
return false
}
return true
})
return found
}
// String returns human-readable state of the scoreboard structure.
func (s *SACKScoreboard) String() string {
var str strings.Builder
str.WriteString("SACKScoreboard: {")
s.ranges.Ascend(func(i btree.Item) bool {
str.WriteString(fmt.Sprintf("%v,", i))
return true
})
str.WriteString("}\n")
return str.String()
}
// Delete removes all SACK information prior to seq.
func (s *SACKScoreboard) Delete(seq seqnum.Value) {
if s.Empty() {
return
}
toDelete := []btree.Item{}
toInsert := []btree.Item{}
r := header.SACKBlock{seq, seq.Add(1)}
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
if i == r {
return true
}
sb := i.(header.SACKBlock)
toDelete = append(toDelete, i)
if sb.End.LessThanEq(seq) {
s.sacked -= sb.Start.Size(sb.End)
} else {
newSB := header.SACKBlock{seq, sb.End}
toInsert = append(toInsert, newSB)
s.sacked -= sb.Start.Size(seq)
}
return true
})
for _, sb := range toDelete {
s.ranges.Delete(sb)
}
for _, sb := range toInsert {
s.ranges.ReplaceOrInsert(sb)
}
}
// Copy provides a copy of the SACK scoreboard.
func (s *SACKScoreboard) Copy() (sackBlocks []header.SACKBlock, maxSACKED seqnum.Value) {
s.ranges.Ascend(func(i btree.Item) bool {
sackBlocks = append(sackBlocks, i.(header.SACKBlock))
return true
})
return sackBlocks, s.maxSACKED
}
// IsRangeLost implements the IsLost(SeqNum) operation defined in RFC 6675
// section 4 but operates on a range of sequence numbers and returns true if
// there are at least nDupAckThreshold SACK blocks greater than the range being
// checked or if at least (nDupAckThreshold-1)*s.smss bytes have been SACKED
// with sequence numbers greater than the block being checked.
func (s *SACKScoreboard) IsRangeLost(r header.SACKBlock) bool {
if s.Empty() {
return false
}
nDupSACK := 0
nDupSACKBytes := seqnum.Size(0)
isLost := false
// We need to check if the immediate lower (if any) sacked
// range contains or partially overlaps with r.
searchMore := true
s.ranges.DescendLessOrEqual(r, func(i btree.Item) bool {
sacked := i.(header.SACKBlock)
if sacked.Contains(r) {
searchMore = false
return false
}
if sacked.End.LessThanEq(r.Start) {
// all sequence numbers covered by sacked are below
// r so we continue searching.
return false
}
// There is a partial overlap. In this case we r.Start is
// between sacked.Start & sacked.End and r.End extends beyond
// sacked.End.
// Move r.Start to sacked.End and continuing searching blocks
// above r.Start.
r.Start = sacked.End
return false
})
if !searchMore {
return isLost
}
s.ranges.AscendGreaterOrEqual(r, func(i btree.Item) bool {
sacked := i.(header.SACKBlock)
if sacked.Contains(r) {
return false
}
nDupSACKBytes += sacked.Start.Size(sacked.End)
nDupSACK++
if nDupSACK >= nDupAckThreshold || nDupSACKBytes >= seqnum.Size((nDupAckThreshold-1)*s.smss) {
isLost = true
return false
}
return true
})
return isLost
}
// IsLost implements the IsLost(SeqNum) operation defined in RFC3517 section
// 4.
//
// This routine returns whether the given sequence number is considered to be
// lost. The routine returns true when either nDupAckThreshold discontiguous
// SACKed sequences have arrived above 'SeqNum' or (nDupAckThreshold * SMSS)
// bytes with sequence numbers greater than 'SeqNum' have been SACKed.
// Otherwise, the routine returns false.
func (s *SACKScoreboard) IsLost(seq seqnum.Value) bool {
return s.IsRangeLost(header.SACKBlock{seq, seq.Add(1)})
}
// Empty returns true if the SACK scoreboard has no entries, false otherwise.
func (s *SACKScoreboard) Empty() bool {
return s.ranges.Len() == 0
}
// Sacked returns the current number of bytes held in the SACK scoreboard.
func (s *SACKScoreboard) Sacked() seqnum.Size {
return s.sacked
}
// MaxSACKED returns the highest sequence number ever inserted in the SACK
// scoreboard.
func (s *SACKScoreboard) MaxSACKED() seqnum.Value {
return s.maxSACKED
}
// SMSS returns the sender's MSS as held by the SACK scoreboard.
func (s *SACKScoreboard) SMSS() uint16 {
return s.smss
}

251
pkg/tcpip/transport/tcp/segment.go Normal file
View file

@ -0,0 +1,251 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"fmt"
"io"
"github.com/sagernet/gvisor/pkg/buffer"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
"github.com/sagernet/gvisor/pkg/tcpip/stack"
)
// queueFlags are used to indicate which queue of an endpoint a particular segment
// belongs to. This is used to track memory accounting correctly.
type queueFlags uint8
const (
// SegOverheadSize is the size of an empty seg in memory including packet
// buffer overhead. It is advised to use SegOverheadSize instead of segSize
// in all cases where accounting for segment memory overhead is important.
SegOverheadSize = segSize + stack.PacketBufferStructSize + header.IPv4MaximumHeaderSize
recvQ queueFlags = 1 << iota
sendQ
)
var segmentPool = sync.Pool{
New: func() any {
return &segment{}
},
}
// segment represents a TCP segment. It holds the payload and parsed TCP segment
// information, and can be added to intrusive lists.
// segment is mostly immutable, the only field allowed to change is data.
//
// +stateify savable
type segment struct {
segmentEntry
segmentRefs
ep *Endpoint
qFlags queueFlags
id stack.TransportEndpointID `state:"manual"`
pkt *stack.PacketBuffer
sequenceNumber seqnum.Value
ackNumber seqnum.Value
flags header.TCPFlags
window seqnum.Size
// csum is only populated for received segments.
csum uint16
// csumValid is true if the csum in the received segment is valid.
csumValid bool
// parsedOptions stores the parsed values from the options in the segment.
parsedOptions header.TCPOptions
options []byte `state:".([]byte)"`
hasNewSACKInfo bool
rcvdTime tcpip.MonotonicTime
// xmitTime is the last transmit time of this segment.
xmitTime tcpip.MonotonicTime
xmitCount uint32
// acked indicates if the segment has already been SACKed.
acked bool
// dataMemSize is the memory used by pkt initially. The value is used for
// memory accounting in the receive buffer instead of pkt.MemSize() because
// packet contents can be modified, so relying on the computed memory size
// to "free" reserved bytes could leak memory in the receiver.
dataMemSize int
// lost indicates if the segment is marked as lost by RACK.
lost bool
}
func newIncomingSegment(id stack.TransportEndpointID, clock tcpip.Clock, pkt *stack.PacketBuffer) (*segment, error) {
hdr := header.TCP(pkt.TransportHeader().Slice())
var srcAddr tcpip.Address
var dstAddr tcpip.Address
switch netProto := pkt.NetworkProtocolNumber; netProto {
case header.IPv4ProtocolNumber:
hdr := header.IPv4(pkt.NetworkHeader().Slice())
srcAddr = hdr.SourceAddress()
dstAddr = hdr.DestinationAddress()
case header.IPv6ProtocolNumber:
hdr := header.IPv6(pkt.NetworkHeader().Slice())
srcAddr = hdr.SourceAddress()
dstAddr = hdr.DestinationAddress()
default:
panic(fmt.Sprintf("unknown network protocol number %d", netProto))
}
csum, csumValid, ok := header.TCPValid(
hdr,
func() uint16 { return pkt.Data().Checksum() },
uint16(pkt.Data().Size()),
srcAddr,
dstAddr,
pkt.RXChecksumValidated)
if !ok {
return nil, fmt.Errorf("header data offset does not respect size constraints: %d < offset < %d, got offset=%d", header.TCPMinimumSize, len(hdr), hdr.DataOffset())
}
s := newSegment()
s.id = id
s.options = hdr[header.TCPMinimumSize:]
s.parsedOptions = header.ParseTCPOptions(hdr[header.TCPMinimumSize:])
s.sequenceNumber = seqnum.Value(hdr.SequenceNumber())
s.ackNumber = seqnum.Value(hdr.AckNumber())
s.flags = hdr.Flags()
s.window = seqnum.Size(hdr.WindowSize())
s.rcvdTime = clock.NowMonotonic()
s.dataMemSize = pkt.MemSize()
s.pkt = pkt.Clone()
s.csumValid = csumValid
if !s.pkt.RXChecksumValidated {
s.csum = csum
}
return s, nil
}
func newOutgoingSegment(id stack.TransportEndpointID, clock tcpip.Clock, buf buffer.Buffer) *segment {
s := newSegment()
s.id = id
s.rcvdTime = clock.NowMonotonic()
s.pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf})
s.dataMemSize = s.pkt.MemSize()
return s
}
func (s *segment) clone() *segment {
t := newSegment()
t.id = s.id
t.sequenceNumber = s.sequenceNumber
t.ackNumber = s.ackNumber
t.flags = s.flags
t.window = s.window
t.rcvdTime = s.rcvdTime
t.xmitTime = s.xmitTime
t.xmitCount = s.xmitCount
t.ep = s.ep
t.qFlags = s.qFlags
t.dataMemSize = s.dataMemSize
t.pkt = s.pkt.Clone()
return t
}
func newSegment() *segment {
s := segmentPool.Get().(*segment)
*s = segment{}
s.InitRefs()
return s
}
// merge merges data in oth and clears oth.
func (s *segment) merge(oth *segment) {
s.pkt.Data().Merge(oth.pkt.Data())
s.dataMemSize = s.pkt.MemSize()
oth.dataMemSize = oth.pkt.MemSize()
}
// setOwner sets the owning endpoint for this segment. Its required
// to be called to ensure memory accounting for receive/send buffer
// queues is done properly.
func (s *segment) setOwner(ep *Endpoint, qFlags queueFlags) {
switch qFlags {
case recvQ:
ep.updateReceiveMemUsed(s.segMemSize())
case sendQ:
// no memory account for sendQ yet.
default:
panic(fmt.Sprintf("unexpected queue flag %b", qFlags))
}
s.ep = ep
s.qFlags = qFlags
}
func (s *segment) DecRef() {
s.segmentRefs.DecRef(func() {
if s.ep != nil {
switch s.qFlags {
case recvQ:
s.ep.updateReceiveMemUsed(-s.segMemSize())
case sendQ:
// no memory accounting for sendQ yet.
default:
panic(fmt.Sprintf("unexpected queue flag %b set for segment", s.qFlags))
}
}
s.pkt.DecRef()
s.pkt = nil
segmentPool.Put(s)
})
}
// logicalLen is the segment length in the sequence number space. It's defined
// as the data length plus one for each of the SYN and FIN bits set.
func (s *segment) logicalLen() seqnum.Size {
l := seqnum.Size(s.payloadSize())
if s.flags.Contains(header.TCPFlagSyn) {
l++
}
if s.flags.Contains(header.TCPFlagFin) {
l++
}
return l
}
// payloadSize is the size of s.data.
func (s *segment) payloadSize() int {
return s.pkt.Data().Size()
}
// segMemSize is the amount of memory used to hold the segment data and
// the associated metadata.
func (s *segment) segMemSize() int {
return segSize + s.dataMemSize
}
// sackBlock returns a header.SACKBlock that represents this segment.
func (s *segment) sackBlock() header.SACKBlock {
return header.SACKBlock{Start: s.sequenceNumber, End: s.sequenceNumber.Add(s.logicalLen())}
}
func (s *segment) TrimFront(ackLeft seqnum.Size) {
s.pkt.Data().TrimFront(int(ackLeft))
}
func (s *segment) ReadTo(dst io.Writer, peek bool) (int, error) {
return s.pkt.Data().ReadTo(dst, peek)
}

51
pkg/tcpip/transport/tcp/segment_heap.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import "container/heap"
type segmentHeap []*segment
var _ heap.Interface = (*segmentHeap)(nil)
// Len returns the length of h.
func (h *segmentHeap) Len() int {
return len(*h)
}
// Less determines whether the i-th element of h is less than the j-th element.
func (h *segmentHeap) Less(i, j int) bool {
return (*h)[i].sequenceNumber.LessThan((*h)[j].sequenceNumber)
}
// Swap swaps the i-th and j-th elements of h.
func (h *segmentHeap) Swap(i, j int) {
(*h)[i], (*h)[j] = (*h)[j], (*h)[i]
}
// Push adds x as the last element of h.
func (h *segmentHeap) Push(x any) {
*h = append(*h, x.(*segment))
}
// Pop removes the last element of h and returns it.
func (h *segmentHeap) Pop() any {
old := *h
n := len(old)
x := old[n-1]
old[n-1] = nil
*h = old[:n-1]
return x
}

99
pkg/tcpip/transport/tcp/segment_queue.go Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
// segmentQueue is a bounded, thread-safe queue of TCP segments.
//
// +stateify savable
type segmentQueue struct {
mu segmentQueueMutex `state:"nosave"`
list segmentList `state:"wait"`
ep *Endpoint
frozen bool
}
// emptyLocked determines if the queue is empty.
// Preconditions: q.mu must be held.
func (q *segmentQueue) emptyLocked() bool {
return q.list.Empty()
}
// empty determines if the queue is empty.
func (q *segmentQueue) empty() bool {
q.mu.Lock()
defer q.mu.Unlock()
return q.emptyLocked()
}
// enqueue adds the given segment to the queue.
//
// Returns true when the segment is successfully added to the queue, in which
// case ownership of the reference is transferred to the queue. And returns
// false if the queue is full, in which case ownership is retained by the
// caller.
func (q *segmentQueue) enqueue(s *segment) bool {
// q.ep.receiveBufferParams() must be called without holding q.mu to
// avoid lock order inversion.
bufSz := q.ep.ops.GetReceiveBufferSize()
used := q.ep.receiveMemUsed()
q.mu.Lock()
defer q.mu.Unlock()
// Allow zero sized segments (ACK/FIN/RSTs etc even if the segment queue
// is currently full).
allow := (used <= int(bufSz) || s.payloadSize() == 0) && !q.frozen
if allow {
s.IncRef()
q.list.PushBack(s)
// Set the owner now that the endpoint owns the segment.
s.setOwner(q.ep, recvQ)
}
return allow
}
// dequeue removes and returns the next segment from queue, if one exists.
// Ownership is transferred to the caller, who is responsible for decrementing
// the ref count when done.
func (q *segmentQueue) dequeue() *segment {
q.mu.Lock()
defer q.mu.Unlock()
s := q.list.Front()
if s != nil {
q.list.Remove(s)
}
return s
}
// freeze prevents any more segments from being added to the queue. i.e all
// future segmentQueue.enqueue will return false and not add the segment to the
// queue till the queue is unfroze with a corresponding segmentQueue.thaw call.
func (q *segmentQueue) freeze() {
q.mu.Lock()
defer q.mu.Unlock()
q.frozen = true
}
// thaw unfreezes a previously frozen queue using segmentQueue.freeze() and
// allows new segments to be queued again.
func (q *segmentQueue) thaw() {
q.mu.Lock()
defer q.mu.Unlock()
q.frozen = false
}

64
pkg/tcpip/transport/tcp/segment_queue_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type segmentQueueMutex struct {
mu sync.Mutex
}
var segmentQueueprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var segmentQueuelockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type segmentQueuelockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *segmentQueueMutex) Lock() {
locking.AddGLock(segmentQueueprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *segmentQueueMutex) NestedLock(i segmentQueuelockNameIndex) {
locking.AddGLock(segmentQueueprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *segmentQueueMutex) Unlock() {
locking.DelGLock(segmentQueueprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *segmentQueueMutex) NestedUnlock(i segmentQueuelockNameIndex) {
locking.DelGLock(segmentQueueprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func segmentQueueinitLockNames() {}
func init() {
segmentQueueinitLockNames()
segmentQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(segmentQueueMutex{}), segmentQueuelockNames)
}

35
pkg/tcpip/transport/tcp/segment_state.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"context"
)
// saveOptions is invoked by stateify.
func (s *segment) saveOptions() []byte {
// We cannot save s.options directly as it may point to s.data's trimmed
// tail, which is not allowed by state framework (in-struct pointer).
b := make([]byte, 0, cap(s.options))
return append(b, s.options...)
}
// loadOptions is invoked by stateify.
func (s *segment) loadOptions(_ context.Context, options []byte) {
// NOTE: We cannot point s.options back into s.data's trimmed tail. But
// it is OK as they do not need to aliased. Plus, options is already
// allocated so there is no cost here.
s.options = options
}

23
pkg/tcpip/transport/tcp/segment_unsafe.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"unsafe"
)
const (
segSize = int(unsafe.Sizeof(segment{}))
)

1905
pkg/tcpip/transport/tcp/snd.go Normal file
View file

File diff suppressed because it is too large Load diff

64
pkg/tcpip/transport/tcp/snd_queue_mutex.go Normal file
View file

@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"github.com/sagernet/gvisor/pkg/sync"
"github.com/sagernet/gvisor/pkg/sync/locking"
)
// Mutex is sync.Mutex with the correctness validator.
type sndQueueMutex struct {
mu sync.Mutex
}
var sndQueueprefixIndex *locking.MutexClass
// lockNames is a list of user-friendly lock names.
// Populated in init.
var sndQueuelockNames []string
// lockNameIndex is used as an index passed to NestedLock and NestedUnlock,
// referring to an index within lockNames.
// Values are specified using the "consts" field of go_template_instance.
type sndQueuelockNameIndex int
// DO NOT REMOVE: The following function automatically replaced with lock index constants.
// LOCK_NAME_INDEX_CONSTANTS
const ()
// Lock locks m.
// +checklocksignore
func (m *sndQueueMutex) Lock() {
locking.AddGLock(sndQueueprefixIndex, -1)
m.mu.Lock()
}
// NestedLock locks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *sndQueueMutex) NestedLock(i sndQueuelockNameIndex) {
locking.AddGLock(sndQueueprefixIndex, int(i))
m.mu.Lock()
}
// Unlock unlocks m.
// +checklocksignore
func (m *sndQueueMutex) Unlock() {
locking.DelGLock(sndQueueprefixIndex, -1)
m.mu.Unlock()
}
// NestedUnlock unlocks m knowing that another lock of the same type is held.
// +checklocksignore
func (m *sndQueueMutex) NestedUnlock(i sndQueuelockNameIndex) {
locking.DelGLock(sndQueueprefixIndex, int(i))
m.mu.Unlock()
}
// DO NOT REMOVE: The following function is automatically replaced.
func sndQueueinitLockNames() {}
func init() {
sndQueueinitLockNames()
sndQueueprefixIndex = locking.NewMutexClass(reflect.TypeOf(sndQueueMutex{}), sndQueuelockNames)
}

480
pkg/tcpip/transport/tcp/state.go Normal file
View file

@ -0,0 +1,480 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"time"
"github.com/sagernet/gvisor/pkg/atomicbitops"
"github.com/sagernet/gvisor/pkg/tcpip"
"github.com/sagernet/gvisor/pkg/tcpip/header"
"github.com/sagernet/gvisor/pkg/tcpip/internal/tcp"
"github.com/sagernet/gvisor/pkg/tcpip/seqnum"
)
// TCPProbeFunc is the expected function type for a TCP probe function to be
// passed to stack.AddTCPProbe.
type TCPProbeFunc func(s *TCPEndpointState)
// TCPCubicState is used to hold a copy of the internal cubic state when the
// TCPProbeFunc is invoked.
//
// +stateify savable
type TCPCubicState struct {
// WLastMax is the previous wMax value.
WLastMax float64
// WMax is the value of the congestion window at the time of the last
// congestion event.
WMax float64
// T is the time when the current congestion avoidance was entered.
T tcpip.MonotonicTime
// TimeSinceLastCongestion denotes the time since the current
// congestion avoidance was entered.
TimeSinceLastCongestion time.Duration
// C is the cubic constant as specified in RFC8312, page 11.
C float64
// K is the time period (in seconds) that the above function takes to
// increase the current window size to WMax if there are no further
// congestion events and is calculated using the following equation:
//
// K = cubic_root(WMax*(1-beta_cubic)/C) (Eq. 2, page 5)
K float64
// Beta is the CUBIC multiplication decrease factor. That is, when a
// congestion event is detected, CUBIC reduces its cwnd to
// WC(0)=WMax*beta_cubic.
Beta float64
// WC is window computed by CUBIC at time TimeSinceLastCongestion. It's
// calculated using the formula:
//
// WC(TimeSinceLastCongestion) = C*(t-K)^3 + WMax (Eq. 1)
WC float64
// WEst is the window computed by CUBIC at time
// TimeSinceLastCongestion+RTT i.e WC(TimeSinceLastCongestion+RTT).
WEst float64
// EndSeq is the sequence number that, when cumulatively ACK'd, ends the
// HyStart round.
EndSeq seqnum.Value
// CurrRTT is the minimum round-trip time from the current round.
CurrRTT time.Duration
// LastRTT is the minimum round-trip time from the previous round.
LastRTT time.Duration
// SampleCount is the number of samples from the current round.
SampleCount uint
// LastAck is the time we received the most recent ACK (or start of round if
// more recent).
LastAck tcpip.MonotonicTime
// RoundStart is the time we started the most recent HyStart round.
RoundStart tcpip.MonotonicTime
}
// TCPRACKState is used to hold a copy of the internal RACK state when the
// TCPProbeFunc is invoked.
//
// +stateify savable
type TCPRACKState struct {
// XmitTime is the transmission timestamp of the most recent
// acknowledged segment.
XmitTime tcpip.MonotonicTime
// EndSequence is the ending TCP sequence number of the most recent
// acknowledged segment.
EndSequence seqnum.Value
// FACK is the highest selectively or cumulatively acknowledged
// sequence.
FACK seqnum.Value
// RTT is the round trip time of the most recently delivered packet on
// the connection (either cumulatively acknowledged or selectively
// acknowledged) that was not marked invalid as a possible spurious
// retransmission.
RTT time.Duration
// Reord is true iff reordering has been detected on this connection.
Reord bool
// DSACKSeen is true iff the connection has seen a DSACK.
DSACKSeen bool
// ReoWnd is the reordering window time used for recording packet
// transmission times. It is used to defer the moment at which RACK
// marks a packet lost.
ReoWnd time.Duration
// ReoWndIncr is the multiplier applied to adjust reorder window.
ReoWndIncr uint8
// ReoWndPersist is the number of loss recoveries before resetting
// reorder window.
ReoWndPersist int8
// RTTSeq is the SND.NXT when RTT is updated.
RTTSeq seqnum.Value
}
// TCPEndpointID is the unique 4 tuple that identifies a given endpoint.
//
// +stateify savable
type TCPEndpointID struct {
// LocalPort is the local port associated with the endpoint.
LocalPort uint16
// LocalAddress is the local [network layer] address associated with
// the endpoint.
LocalAddress tcpip.Address
// RemotePort is the remote port associated with the endpoint.
RemotePort uint16
// RemoteAddress it the remote [network layer] address associated with
// the endpoint.
RemoteAddress tcpip.Address
}
// TCPFastRecoveryState holds a copy of the internal fast recovery state of a
// TCP endpoint.
//
// +stateify savable
type TCPFastRecoveryState struct {
// Active if true indicates the endpoint is in fast recovery. The
// following fields are only meaningful when Active is true.
Active bool
// First is the first unacknowledged sequence number being recovered.
First seqnum.Value
// Last is the 'recover' sequence number that indicates the point at
// which we should exit recovery barring any timeouts etc.
Last seqnum.Value
// MaxCwnd is the maximum value we are permitted to grow the congestion
// window during recovery. This is set at the time we enter recovery.
// It exists to avoid attacks where the receiver intentionally sends
// duplicate acks to artificially inflate the sender's cwnd.
MaxCwnd int
// HighRxt is the highest sequence number which has been retransmitted
// during the current loss recovery phase. See: RFC 6675 Section 2 for
// details.
HighRxt seqnum.Value
// RescueRxt is the highest sequence number which has been
// optimistically retransmitted to prevent stalling of the ACK clock
// when there is loss at the end of the window and no new data is
// available for transmission. See: RFC 6675 Section 2 for details.
RescueRxt seqnum.Value
}
// TCPReceiverState holds a copy of the internal state of the receiver for a
// given TCP endpoint.
//
// +stateify savable
type TCPReceiverState struct {
// RcvNxt is the TCP variable RCV.NXT.
RcvNxt seqnum.Value
// RcvAcc is one beyond the last acceptable sequence number. That is,
// the "largest" sequence value that the receiver has announced to its
// peer that it's willing to accept. This may be different than RcvNxt
// + (last advertised receive window) if the receive window is reduced;
// in that case we have to reduce the window as we receive more data
// instead of shrinking it.
RcvAcc seqnum.Value
// RcvWndScale is the window scaling to use for inbound segments.
RcvWndScale uint8
// PendingBufUsed is the number of bytes pending in the receive queue.
PendingBufUsed int
}
// TCPRTTState holds a copy of information about the endpoint's round trip
// time.
//
// +stateify savable
type TCPRTTState struct {
// SRTT is the smoothed round trip time defined in section 2 of RFC
// 6298.
SRTT time.Duration
// RTTVar is the round-trip time variation as defined in section 2 of
// RFC 6298.
RTTVar time.Duration
// SRTTInited if true indicates that a valid RTT measurement has been
// completed.
SRTTInited bool
}
// TCPSenderState holds a copy of the internal state of the sender for a given
// TCP Endpoint.
//
// +stateify savable
type TCPSenderState struct {
// LastSendTime is the timestamp at which we sent the last segment.
LastSendTime tcpip.MonotonicTime
// DupAckCount is the number of Duplicate ACKs received. It is used for
// fast retransmit.
DupAckCount int
// SndCwnd is the size of the sending congestion window in packets.
SndCwnd int
// Ssthresh is the threshold between slow start and congestion
// avoidance.
Ssthresh int
// SndCAAckCount is the number of packets acknowledged during
// congestion avoidance. When enough packets have been ack'd (typically
// cwnd packets), the congestion window is incremented by one.
SndCAAckCount int
// Outstanding is the number of packets that have been sent but not yet
// acknowledged.
Outstanding int
// SackedOut is the number of packets which have been selectively
// acked.
SackedOut int
// SndWnd is the send window size in bytes.
SndWnd seqnum.Size
// SndUna is the next unacknowledged sequence number.
SndUna seqnum.Value
// SndNxt is the sequence number of the next segment to be sent.
SndNxt seqnum.Value
// RTTMeasureSeqNum is the sequence number being used for the latest
// RTT measurement.
RTTMeasureSeqNum seqnum.Value
// RTTMeasureTime is the time when the RTTMeasureSeqNum was sent.
RTTMeasureTime tcpip.MonotonicTime
// Closed indicates that the caller has closed the endpoint for
// sending.
Closed bool
// RTO is the retransmit timeout as defined in section of 2 of RFC
// 6298.
RTO time.Duration
// RTTState holds information about the endpoint's round trip time.
RTTState TCPRTTState
// MaxPayloadSize is the maximum size of the payload of a given
// segment. It is initialized on demand.
MaxPayloadSize int
// SndWndScale is the number of bits to shift left when reading the
// send window size from a segment.
SndWndScale uint8
// MaxSentAck is the highest acknowledgement number sent till now.
MaxSentAck seqnum.Value
// FastRecovery holds the fast recovery state for the endpoint.
FastRecovery TCPFastRecoveryState
// Cubic holds the state related to CUBIC congestion control.
Cubic TCPCubicState
// RACKState holds the state related to RACK loss detection algorithm.
RACKState TCPRACKState
// RetransmitTS records the timestamp used to detect spurious recovery.
RetransmitTS uint32
// SpuriousRecovery indicates if the sender entered recovery spuriously.
SpuriousRecovery bool
}
// TCPSACKInfo holds TCP SACK related information for a given TCP endpoint.
//
// +stateify savable
type TCPSACKInfo struct {
// Blocks is the list of SACK Blocks that identify the out of order
// segments held by a given TCP endpoint.
Blocks []header.SACKBlock
// ReceivedBlocks are the SACK blocks received by this endpoint from
// the peer endpoint.
ReceivedBlocks []header.SACKBlock
// MaxSACKED is the highest sequence number that has been SACKED by the
// peer.
MaxSACKED seqnum.Value
}
// RcvBufAutoTuneParams holds state related to TCP receive buffer auto-tuning.
//
// +stateify savable
type RcvBufAutoTuneParams struct {
// MeasureTime is the time at which the current measurement was
// started.
MeasureTime tcpip.MonotonicTime
// CopiedBytes is the number of bytes copied to user space since this
// measure began.
CopiedBytes int
// PrevCopiedBytes is the number of bytes copied to userspace in the
// previous RTT period.
PrevCopiedBytes int
// RcvBufSize is the auto tuned receive buffer size.
RcvBufSize int
// RTT is the smoothed RTT as measured by observing the time between
// when a byte is first acknowledged and the receipt of data that is at
// least one window beyond the sequence number that was acknowledged.
RTT time.Duration
// RTTVar is the "round-trip time variation" as defined in section 2 of
// RFC6298.
RTTVar time.Duration
// RTTMeasureSeqNumber is the highest acceptable sequence number at the
// time this RTT measurement period began.
RTTMeasureSeqNumber seqnum.Value
// RTTMeasureTime is the absolute time at which the current RTT
// measurement period began.
RTTMeasureTime tcpip.MonotonicTime
// Disabled is true if an explicit receive buffer is set for the
// endpoint.
Disabled bool
}
// TCPRcvBufState contains information about the state of an endpoint's receive
// socket buffer.
//
// +stateify savable
type TCPRcvBufState struct {
// RcvBufUsed is the amount of bytes actually held in the receive
// socket buffer for the endpoint.
RcvBufUsed int
// RcvBufAutoTuneParams is used to hold state variables to compute the
// auto tuned receive buffer size.
RcvAutoParams RcvBufAutoTuneParams
// RcvClosed if true, indicates the endpoint has been closed for
// reading.
RcvClosed bool
}
// TCPSndBufState contains information about the state of an endpoint's send
// socket buffer.
//
// +stateify savable
type TCPSndBufState struct {
// SndBufSize is the size of the socket send buffer.
SndBufSize int
// SndBufUsed is the number of bytes held in the socket send buffer.
SndBufUsed int
// SndClosed indicates that the endpoint has been closed for sends.
SndClosed bool
// PacketTooBigCount is used to notify the main protocol routine how
// many times a "packet too big" control packet is received.
PacketTooBigCount int
// SndMTU is the smallest MTU seen in the control packets received.
SndMTU int
// AutoTuneSndBufDisabled indicates that the auto tuning of send buffer
// is disabled.
AutoTuneSndBufDisabled atomicbitops.Uint32
}
// TCPEndpointStateInner contains the members of TCPEndpointState used directly
// (that is, not within another containing struct) within the endpoint's
// internal implementation.
//
// +stateify savable
type TCPEndpointStateInner struct {
// TSOffset is a randomized offset added to the value of the TSVal
// field in the timestamp option.
TSOffset tcp.TSOffset
// SACKPermitted is set to true if the peer sends the TCPSACKPermitted
// option in the SYN/SYN-ACK.
SACKPermitted bool
// SendTSOk is used to indicate when the TS Option has been negotiated.
// When sendTSOk is true every non-RST segment should carry a TS as per
// RFC7323#section-1.1.
SendTSOk bool
// RecentTS is the timestamp that should be sent in the TSEcr field of
// the timestamp for future segments sent by the endpoint. This field
// is updated if required when a new segment is received by this
// endpoint.
RecentTS uint32
}
// TCPEndpointState is a copy of the internal state of a TCP endpoint.
//
// +stateify savable
type TCPEndpointState struct {
// TCPEndpointStateInner contains the members of TCPEndpointState used
// by the endpoint's internal implementation.
TCPEndpointStateInner
// ID is a copy of the TransportEndpointID for the endpoint.
ID TCPEndpointID
// SegTime denotes the absolute time when this segment was received.
SegTime tcpip.MonotonicTime
// RcvBufState contains information about the state of the endpoint's
// receive socket buffer.
RcvBufState TCPRcvBufState
// SndBufState contains information about the state of the endpoint's
// send socket buffer.
SndBufState TCPSndBufState
// SACK holds TCP SACK related information for this endpoint.
SACK TCPSACKInfo
// Receiver holds variables related to the TCP receiver for the
// endpoint.
Receiver TCPReceiverState
// Sender holds state related to the TCP Sender for the endpoint.
Sender TCPSenderState
}

239
pkg/tcpip/transport/tcp/tcp_endpoint_list.go Normal file
View file

@ -0,0 +1,239 @@
package tcp
// ElementMapper provides an identity mapping by default.
//
// This can be replaced to provide a struct that maps elements to linker
// objects, if they are not the same. An ElementMapper is not typically
// required if: Linker is left as is, Element is left as is, or Linker and
// Element are the same type.
type endpointElementMapper struct{}
// linkerFor maps an Element to a Linker.
//
// This default implementation should be inlined.
//
//go:nosplit
func (endpointElementMapper) linkerFor(elem *Endpoint) *Endpoint { return elem }
// List is an intrusive list. Entries can be added to or removed from the list
// in O(1) time and with no additional memory allocations.
//
// The zero value for List is an empty list ready to use.
//
// To iterate over a list (where l is a List):
//
// for e := l.Front(); e != nil; e = e.Next() {
// // do something with e.
// }
//
// +stateify savable
type endpointList struct {
head *Endpoint
tail *Endpoint
}
// Reset resets list l to the empty state.
func (l *endpointList) Reset() {
l.head = nil
l.tail = nil
}
// Empty returns true iff the list is empty.
//
//go:nosplit
func (l *endpointList) Empty() bool {
return l.head == nil
}
// Front returns the first element of list l or nil.
//
//go:nosplit
func (l *endpointList) Front() *Endpoint {
return l.head
}
// Back returns the last element of list l or nil.
//
//go:nosplit
func (l *endpointList) Back() *Endpoint {
return l.tail
}
// Len returns the number of elements in the list.
//
// NOTE: This is an O(n) operation.
//
//go:nosplit
func (l *endpointList) Len() (count int) {
for e := l.Front(); e != nil; e = (endpointElementMapper{}.linkerFor(e)).Next() {
count++
}
return count
}
// PushFront inserts the element e at the front of list l.
//
//go:nosplit
func (l *endpointList) PushFront(e *Endpoint) {
linker := endpointElementMapper{}.linkerFor(e)
linker.SetNext(l.head)
linker.SetPrev(nil)
if l.head != nil {
endpointElementMapper{}.linkerFor(l.head).SetPrev(e)
} else {
l.tail = e
}
l.head = e
}
// PushFrontList inserts list m at the start of list l, emptying m.
//
//go:nosplit
func (l *endpointList) PushFrontList(m *endpointList) {
if l.head == nil {
l.head = m.head
l.tail = m.tail
} else if m.head != nil {
endpointElementMapper{}.linkerFor(l.head).SetPrev(m.tail)
endpointElementMapper{}.linkerFor(m.tail).SetNext(l.head)
l.head = m.head
}
m.head = nil
m.tail = nil
}
// PushBack inserts the element e at the back of list l.
//
//go:nosplit
func (l *endpointList) PushBack(e *Endpoint) {
linker := endpointElementMapper{}.linkerFor(e)
linker.SetNext(nil)
linker.SetPrev(l.tail)
if l.tail != nil {
endpointElementMapper{}.linkerFor(l.tail).SetNext(e)
} else {
l.head = e
}
l.tail = e
}
// PushBackList inserts list m at the end of list l, emptying m.
//
//go:nosplit
func (l *endpointList) PushBackList(m *endpointList) {
if l.head == nil {
l.head = m.head
l.tail = m.tail
} else if m.head != nil {
endpointElementMapper{}.linkerFor(l.tail).SetNext(m.head)
endpointElementMapper{}.linkerFor(m.head).SetPrev(l.tail)
l.tail = m.tail
}
m.head = nil
m.tail = nil
}
// InsertAfter inserts e after b.
//
//go:nosplit
func (l *endpointList) InsertAfter(b, e *Endpoint) {
bLinker := endpointElementMapper{}.linkerFor(b)
eLinker := endpointElementMapper{}.linkerFor(e)
a := bLinker.Next()
eLinker.SetNext(a)
eLinker.SetPrev(b)
bLinker.SetNext(e)
if a != nil {
endpointElementMapper{}.linkerFor(a).SetPrev(e)
} else {
l.tail = e
}
}
// InsertBefore inserts e before a.
//
//go:nosplit
func (l *endpointList) InsertBefore(a, e *Endpoint) {
aLinker := endpointElementMapper{}.linkerFor(a)
eLinker := endpointElementMapper{}.linkerFor(e)
b := aLinker.Prev()
eLinker.SetNext(a)
eLinker.SetPrev(b)
aLinker.SetPrev(e)
if b != nil {
endpointElementMapper{}.linkerFor(b).SetNext(e)
} else {
l.head = e
}
}
// Remove removes e from l.
//
//go:nosplit
func (l *endpointList) Remove(e *Endpoint) {
linker := endpointElementMapper{}.linkerFor(e)
prev := linker.Prev()
next := linker.Next()
if prev != nil {
endpointElementMapper{}.linkerFor(prev).SetNext(next)
} else if l.head == e {
l.head = next
}
if next != nil {
endpointElementMapper{}.linkerFor(next).SetPrev(prev)
} else if l.tail == e {
l.tail = prev
}
linker.SetNext(nil)
linker.SetPrev(nil)
}
// Entry is a default implementation of Linker. Users can add anonymous fields
// of this type to their structs to make them automatically implement the
// methods needed by List.
//
// +stateify savable
type endpointEntry struct {
next *Endpoint
prev *Endpoint
}
// Next returns the entry that follows e in the list.
//
//go:nosplit
func (e *endpointEntry) Next() *Endpoint {
return e.next
}
// Prev returns the entry that precedes e in the list.
//
//go:nosplit
func (e *endpointEntry) Prev() *Endpoint {
return e.prev
}
// SetNext assigns 'entry' as the entry that follows e in the list.
//
//go:nosplit
func (e *endpointEntry) SetNext(elem *Endpoint) {
e.next = elem
}
// SetPrev assigns 'entry' as the entry that precedes e in the list.
//
//go:nosplit
func (e *endpointEntry) SetPrev(elem *Endpoint) {
e.prev = elem
}

239
pkg/tcpip/transport/tcp/tcp_segment_list.go Normal file
View file

@ -0,0 +1,239 @@
package tcp
// ElementMapper provides an identity mapping by default.
//
// This can be replaced to provide a struct that maps elements to linker
// objects, if they are not the same. An ElementMapper is not typically
// required if: Linker is left as is, Element is left as is, or Linker and
// Element are the same type.
type segmentElementMapper struct{}
// linkerFor maps an Element to a Linker.
//
// This default implementation should be inlined.
//
//go:nosplit
func (segmentElementMapper) linkerFor(elem *segment) *segment { return elem }
// List is an intrusive list. Entries can be added to or removed from the list
// in O(1) time and with no additional memory allocations.
//
// The zero value for List is an empty list ready to use.
//
// To iterate over a list (where l is a List):
//
// for e := l.Front(); e != nil; e = e.Next() {
// // do something with e.
// }
//
// +stateify savable
type segmentList struct {
head *segment
tail *segment
}
// Reset resets list l to the empty state.
func (l *segmentList) Reset() {
l.head = nil
l.tail = nil
}
// Empty returns true iff the list is empty.
//
//go:nosplit
func (l *segmentList) Empty() bool {
return l.head == nil
}
// Front returns the first element of list l or nil.
//
//go:nosplit
func (l *segmentList) Front() *segment {
return l.head
}
// Back returns the last element of list l or nil.
//
//go:nosplit
func (l *segmentList) Back() *segment {
return l.tail
}
// Len returns the number of elements in the list.
//
// NOTE: This is an O(n) operation.
//
//go:nosplit
func (l *segmentList) Len() (count int) {
for e := l.Front(); e != nil; e = (segmentElementMapper{}.linkerFor(e)).Next() {
count++
}
return count
}
// PushFront inserts the element e at the front of list l.
//
//go:nosplit
func (l *segmentList) PushFront(e *segment) {
linker := segmentElementMapper{}.linkerFor(e)
linker.SetNext(l.head)
linker.SetPrev(nil)
if l.head != nil {
segmentElementMapper{}.linkerFor(l.head).SetPrev(e)
} else {
l.tail = e
}
l.head = e
}
// PushFrontList inserts list m at the start of list l, emptying m.
//
//go:nosplit
func (l *segmentList) PushFrontList(m *segmentList) {
if l.head == nil {
l.head = m.head
l.tail = m.tail
} else if m.head != nil {
segmentElementMapper{}.linkerFor(l.head).SetPrev(m.tail)
segmentElementMapper{}.linkerFor(m.tail).SetNext(l.head)
l.head = m.head
}
m.head = nil
m.tail = nil
}
// PushBack inserts the element e at the back of list l.
//
//go:nosplit
func (l *segmentList) PushBack(e *segment) {
linker := segmentElementMapper{}.linkerFor(e)
linker.SetNext(nil)
linker.SetPrev(l.tail)
if l.tail != nil {
segmentElementMapper{}.linkerFor(l.tail).SetNext(e)
} else {
l.head = e
}
l.tail = e
}
// PushBackList inserts list m at the end of list l, emptying m.
//
//go:nosplit
func (l *segmentList) PushBackList(m *segmentList) {
if l.head == nil {
l.head = m.head
l.tail = m.tail
} else if m.head != nil {
segmentElementMapper{}.linkerFor(l.tail).SetNext(m.head)
segmentElementMapper{}.linkerFor(m.head).SetPrev(l.tail)
l.tail = m.tail
}
m.head = nil
m.tail = nil
}
// InsertAfter inserts e after b.
//
//go:nosplit
func (l *segmentList) InsertAfter(b, e *segment) {
bLinker := segmentElementMapper{}.linkerFor(b)
eLinker := segmentElementMapper{}.linkerFor(e)
a := bLinker.Next()
eLinker.SetNext(a)
eLinker.SetPrev(b)
bLinker.SetNext(e)
if a != nil {
segmentElementMapper{}.linkerFor(a).SetPrev(e)
} else {
l.tail = e
}
}
// InsertBefore inserts e before a.
//
//go:nosplit
func (l *segmentList) InsertBefore(a, e *segment) {
aLinker := segmentElementMapper{}.linkerFor(a)
eLinker := segmentElementMapper{}.linkerFor(e)
b := aLinker.Prev()
eLinker.SetNext(a)
eLinker.SetPrev(b)
aLinker.SetPrev(e)
if b != nil {
segmentElementMapper{}.linkerFor(b).SetNext(e)
} else {
l.head = e
}
}
// Remove removes e from l.
//
//go:nosplit
func (l *segmentList) Remove(e *segment) {
linker := segmentElementMapper{}.linkerFor(e)
prev := linker.Prev()
next := linker.Next()
if prev != nil {
segmentElementMapper{}.linkerFor(prev).SetNext(next)
} else if l.head == e {
l.head = next
}
if next != nil {
segmentElementMapper{}.linkerFor(next).SetPrev(prev)
} else if l.tail == e {
l.tail = prev
}
linker.SetNext(nil)
linker.SetPrev(nil)
}
// Entry is a default implementation of Linker. Users can add anonymous fields
// of this type to their structs to make them automatically implement the
// methods needed by List.
//
// +stateify savable
type segmentEntry struct {
next *segment
prev *segment
}
// Next returns the entry that follows e in the list.
//
//go:nosplit
func (e *segmentEntry) Next() *segment {
return e.next
}
// Prev returns the entry that precedes e in the list.
//
//go:nosplit
func (e *segmentEntry) Prev() *segment {
return e.prev
}
// SetNext assigns 'entry' as the entry that follows e in the list.
//
//go:nosplit
func (e *segmentEntry) SetNext(elem *segment) {
e.next = elem
}
// SetPrev assigns 'entry' as the entry that precedes e in the list.
//
//go:nosplit
func (e *segmentEntry) SetPrev(elem *segment) {
e.prev = elem
}

141
pkg/tcpip/transport/tcp/tcp_segment_refs.go Normal file
View file

@ -0,0 +1,141 @@
package tcp
import (
"context"
"fmt"
"github.com/sagernet/gvisor/pkg/atomicbitops"
"github.com/sagernet/gvisor/pkg/refs"
)
// enableLogging indicates whether reference-related events should be logged (with
// stack traces). This is false by default and should only be set to true for
// debugging purposes, as it can generate an extremely large amount of output
// and drastically degrade performance.
const segmentenableLogging = false
// obj is used to customize logging. Note that we use a pointer to T so that
// we do not copy the entire object when passed as a format parameter.
var segmentobj *segment
// Refs implements refs.RefCounter. It keeps a reference count using atomic
// operations and calls the destructor when the count reaches zero.
//
// NOTE: Do not introduce additional fields to the Refs struct. It is used by
// many filesystem objects, and we want to keep it as small as possible (i.e.,
// the same size as using an int64 directly) to avoid taking up extra cache
// space. In general, this template should not be extended at the cost of
// performance. If it does not offer enough flexibility for a particular object
// (example: b/187877947), we should implement the RefCounter/CheckedObject
// interfaces manually.
//
// +stateify savable
type segmentRefs struct {
// refCount is composed of two fields:
//
// [32-bit speculative references]:[32-bit real references]
//
// Speculative references are used for TryIncRef, to avoid a CompareAndSwap
// loop. See IncRef, DecRef and TryIncRef for details of how these fields are
// used.
refCount atomicbitops.Int64
}
// InitRefs initializes r with one reference and, if enabled, activates leak
// checking.
func (r *segmentRefs) InitRefs() {
r.refCount.RacyStore(1)
refs.Register(r)
}
// RefType implements refs.CheckedObject.RefType.
func (r *segmentRefs) RefType() string {
return fmt.Sprintf("%T", segmentobj)[1:]
}
// LeakMessage implements refs.CheckedObject.LeakMessage.
func (r *segmentRefs) LeakMessage() string {
return fmt.Sprintf("[%s %p] reference count of %d instead of 0", r.RefType(), r, r.ReadRefs())
}
// LogRefs implements refs.CheckedObject.LogRefs.
func (r *segmentRefs) LogRefs() bool {
return segmentenableLogging
}
// ReadRefs returns the current number of references. The returned count is
// inherently racy and is unsafe to use without external synchronization.
func (r *segmentRefs) ReadRefs() int64 {
return r.refCount.Load()
}
// IncRef implements refs.RefCounter.IncRef.
//
//go:nosplit
func (r *segmentRefs) IncRef() {
v := r.refCount.Add(1)
if segmentenableLogging {
refs.LogIncRef(r, v)
}
if v <= 1 {
panic(fmt.Sprintf("Incrementing non-positive count %p on %s", r, r.RefType()))
}
}
// TryIncRef implements refs.TryRefCounter.TryIncRef.
//
// To do this safely without a loop, a speculative reference is first acquired
// on the object. This allows multiple concurrent TryIncRef calls to distinguish
// other TryIncRef calls from genuine references held.
//
//go:nosplit
func (r *segmentRefs) TryIncRef() bool {
const speculativeRef = 1 << 32
if v := r.refCount.Add(speculativeRef); int32(v) == 0 {
r.refCount.Add(-speculativeRef)
return false
}
v := r.refCount.Add(-speculativeRef + 1)
if segmentenableLogging {
refs.LogTryIncRef(r, v)
}
return true
}
// DecRef implements refs.RefCounter.DecRef.
//
// Note that speculative references are counted here. Since they were added
// prior to real references reaching zero, they will successfully convert to
// real references. In other words, we see speculative references only in the
// following case:
//
// A: TryIncRef [speculative increase => sees non-negative references]
// B: DecRef [real decrease]
// A: TryIncRef [transform speculative to real]
//
//go:nosplit
func (r *segmentRefs) DecRef(destroy func()) {
v := r.refCount.Add(-1)
if segmentenableLogging {
refs.LogDecRef(r, v)
}
switch {
case v < 0:
panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %s", r, r.RefType()))
case v == 0:
refs.Unregister(r)
if destroy != nil {
destroy()
}
}
}
func (r *segmentRefs) afterLoad(context.Context) {
if r.ReadRefs() > 0 {
refs.Register(r)
}
}

1935
pkg/tcpip/transport/tcp/tcp_state_autogen.go Normal file
View file

File diff suppressed because it is too large Load diff

3
pkg/tcpip/transport/tcp/tcp_unsafe_state_autogen.go Normal file
View file

@ -0,0 +1,3 @@
// automatically generated by stateify.
package tcp

160
pkg/tcpip/transport/tcp/timer.go Normal file
View file

@ -0,0 +1,160 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"time"
"github.com/sagernet/gvisor/pkg/tcpip"
)
type timerState int
const (
// The timer has not been initialized yet or has been cleaned up.
timerUninitialized timerState = iota
// The timer is disabled.
timerStateDisabled
// The timer is enabled, but the clock timer may be set to an earlier
// expiration time due to a previous orphaned state.
timerStateEnabled
// The timer is disabled, but the clock timer is enabled, which means that
// it will cause a spurious wakeup unless the timer is enabled before the
// clock timer fires.
timerStateOrphaned
)
// timer is a timer implementation that reduces the interactions with the
// clock timer infrastructure by letting timers run (and potentially
// eventually expire) even if they are stopped. It makes it cheaper to
// disable/reenable timers at the expense of spurious wakes. This is useful for
// cases when the same timer is disabled/reenabled repeatedly with relatively
// long timeouts farther into the future.
//
// TCP retransmit timers benefit from this because they the timeouts are long
// (currently at least 200ms), and get disabled when acks are received, and
// reenabled when new pending segments are sent.
//
// It is advantageous to avoid interacting with the clock because it acquires
// a global mutex and performs O(log n) operations, where n is the global number
// of timers, whenever a timer is enabled or disabled, and may make a syscall.
//
// This struct is thread-compatible.
type timer struct {
state timerState
clock tcpip.Clock
// target is the expiration time of the current timer. It is only
// meaningful in the enabled state.
target tcpip.MonotonicTime
// clockTarget is the expiration time of the clock timer. It is
// meaningful in the enabled and orphaned states.
clockTarget tcpip.MonotonicTime
// timer is the clock timer used to wait on.
timer tcpip.Timer
// callback is the function that's called when the timer expires.
callback func()
}
// init initializes the timer. Once it expires the function callback
// passed will be called.
func (t *timer) init(clock tcpip.Clock, f func()) {
t.state = timerStateDisabled
t.clock = clock
t.callback = f
}
// cleanup frees all resources associated with the timer.
func (t *timer) cleanup() {
if t.timer == nil {
// No cleanup needed.
return
}
t.timer.Stop()
*t = timer{}
}
// isUninitialized returns true if the timer is in the uninitialized state. This
// is only true if init() has never been called or if cleanup has been called.
func (t *timer) isUninitialized() bool {
return t.state == timerUninitialized
}
// checkExpiration checks if the given timer has actually expired, it should be
// called whenever the callback function is called, and is used to check if it's
// a spurious timer expiration (due to a previously orphaned timer) or a
// legitimate one.
func (t *timer) checkExpiration() bool {
// Transition to fully disabled state if we're just consuming an
// orphaned timer.
if t.state == timerStateOrphaned {
t.state = timerStateDisabled
return false
}
// The timer is enabled, but it may have expired early. Check if that's
// the case, and if so, reset the runtime timer to the correct time.
now := t.clock.NowMonotonic()
if now.Before(t.target) {
t.clockTarget = t.target
t.timer.Reset(t.target.Sub(now))
return false
}
// The timer has actually expired, disable it for now and inform the
// caller.
t.state = timerStateDisabled
return true
}
// disable disables the timer, leaving it in an orphaned state if it wasn't
// already disabled.
func (t *timer) disable() {
if t.state != timerStateDisabled {
t.state = timerStateOrphaned
}
}
// enabled returns true if the timer is currently enabled, false otherwise.
func (t *timer) enabled() bool {
return t.state == timerStateEnabled
}
// enable enables the timer, programming the runtime timer if necessary.
func (t *timer) enable(d time.Duration) {
t.target = t.clock.NowMonotonic().Add(d)
// Check if we need to set the runtime timer.
if t.state == timerStateDisabled || t.target.Before(t.clockTarget) {
t.clockTarget = t.target
t.resetOrStart(d)
}
t.state = timerStateEnabled
}
// resetOrStart creates the timer if it doesn't already exist or resets it with
// the given duration if it does.
func (t *timer) resetOrStart(d time.Duration) {
if t.timer == nil {
t.timer = t.clock.AfterFunc(d, t.callback)
} else {
t.timer.Reset(d)
}
}