system stack: self-heal the TCP forwarder accept loop (sing-box-lx SPEC 040)

Upstream acceptLoop treats any Accept error as terminal and silently
returns, leaving the stack alive but every new TCP SYN NAT-rewritten onto
a dead port (instant RST) until a full restart. When the listener fd is
closed out from under the stack (a stray close on a reused fd number from
another runtime in the same process), all new TCP dies forever while
UDP/QUIC/DNS keep working.

- System.Close() now marks a deliberate shutdown first; acceptLoop still
  exits quietly on it.
- Any other Accept error is logged (the errno names the killer path),
  the listener is recreated on the same address, the forwarder port is
  republished atomically, and the loop keeps serving.
- If the rebind fails, the loop logs an error and gives up - no worse
  than upstream.
- acceptRecoveries counter is kept as telemetry.

tcpPort/tcpPort6 become atomic (written by the heal path, read from the
tunLoop dispatch/NAT path); listener replacement is serialized against
Close() with a mutex.
This commit is contained in:
Leadaxe 2026-07-31 23:54:20 +03:00
parent da24acaf4d
commit d31d20ba58
2 changed files with 244 additions and 39 deletions

View file

@ -7,6 +7,8 @@ import (
"net/netip" "net/netip"
"os" "os"
"slices" "slices"
"sync"
"sync/atomic"
"syscall" "syscall"
"time" "time"
@ -45,17 +47,25 @@ type System struct {
icmpTimeout time.Duration icmpTimeout time.Duration
tcpListener net.Listener tcpListener net.Listener
tcpListener6 net.Listener tcpListener6 net.Listener
tcpPort uint16 // lx/040: ports are written by acceptLoop on self-heal relisten and read
tcpPort6 uint16 // concurrently from the tunLoop path (dispatch filter + NAT rewrite) —
tcpNat *TCPNat // they must be atomic. listenAccess serializes listener replacement
udpNat *UDPNat // against Close(); closing marks a deliberate shutdown so acceptLoop can
udpNATOptions UDPNatOptions // tell it apart from the listener dying out from under the stack.
dispatcher *ForwardDispatcher tcpPort atomic.Uint32
bindInterface bool tcpPort6 atomic.Uint32
interfaceFinder control.InterfaceFinder closing atomic.Bool
frontHeadroom int listenAccess sync.Mutex
txChecksumOffload bool acceptRecoveries atomic.Uint32
multiPendingPackets bool tcpNat *TCPNat
udpNat *UDPNat
udpNATOptions UDPNatOptions
dispatcher *ForwardDispatcher
bindInterface bool
interfaceFinder control.InterfaceFinder
frontHeadroom int
txChecksumOffload bool
multiPendingPackets bool
} }
type Session struct { type Session struct {
@ -124,10 +134,15 @@ func (s *System) ResetNetwork() {
} }
func (s *System) Close() error { func (s *System) Close() error {
// lx/040: mark the deliberate shutdown BEFORE closing the listeners so
// acceptLoop exits quietly instead of treating it as a foreign kill.
s.closing.Store(true)
s.dispatcher.Close() s.dispatcher.Close()
if s.udpNat != nil { if s.udpNat != nil {
s.udpNat.Close() s.udpNat.Close()
} }
s.listenAccess.Lock()
defer s.listenAccess.Unlock()
return common.Close( return common.Close(
s.tcpListener, s.tcpListener,
s.tcpListener6, s.tcpListener6,
@ -143,8 +158,10 @@ func (s *System) Start() error {
return nil return nil
} }
func (s *System) start() error { // lx/040: TCP forwarder bind, shared by start() and the acceptLoop self-heal
_ = fixWindowsFirewall() // relisten path. isIPv6 selects the address family; the bind-to-interface
// Control and the EADDRNOTAVAIL retry loop match the original start() code.
func (s *System) listenTCP(isIPv6 bool) (net.Listener, error) {
var listener net.ListenConfig var listener net.ListenConfig
if s.bindInterface { if s.bindInterface {
listener.Control = control.Append(listener.Control, func(network, address string, conn syscall.RawConn) error { listener.Control = control.Append(listener.Control, func(network, address string, conn syscall.RawConn) error {
@ -155,37 +172,50 @@ func (s *System) start() error {
return nil return nil
}) })
} }
network := "tcp4"
address := s.inet4Address
if isIPv6 {
network = "tcp6"
address = s.inet6Address
}
var (
tcpListener net.Listener
err error
)
for range 3 {
tcpListener, err = listenNetworkNamespace(s.ctx, s.netNs, listener, network, net.JoinHostPort(address.String(), "0"))
if !retryableListenError(err) {
break
}
time.Sleep(time.Second)
}
if err != nil {
return nil, err
}
return tcpListener, nil
}
func (s *System) start() error {
_ = fixWindowsFirewall()
var tcpListener net.Listener var tcpListener net.Listener
var err error var err error
if s.inet4NextAddress.IsValid() { if s.inet4NextAddress.IsValid() {
for range 3 { tcpListener, err = s.listenTCP(false)
tcpListener, err = listenNetworkNamespace(s.ctx, s.netNs, listener, "tcp4", net.JoinHostPort(s.inet4Address.String(), "0"))
if !retryableListenError(err) {
break
}
time.Sleep(time.Second)
}
if err != nil { if err != nil {
return err return err
} }
s.tcpListener = tcpListener s.tcpListener = tcpListener
s.tcpPort = M.SocksaddrFromNet(tcpListener.Addr()).Port s.tcpPort.Store(uint32(M.SocksaddrFromNet(tcpListener.Addr()).Port))
go s.acceptLoop(tcpListener) go s.acceptLoop(tcpListener, false)
} }
if s.inet6NextAddress.IsValid() { if s.inet6NextAddress.IsValid() {
for range 3 { tcpListener, err = s.listenTCP(true)
tcpListener, err = listenNetworkNamespace(s.ctx, s.netNs, listener, "tcp6", net.JoinHostPort(s.inet6Address.String(), "0"))
if !retryableListenError(err) {
break
}
time.Sleep(time.Second)
}
if err != nil { if err != nil {
return err return err
} }
s.tcpListener6 = tcpListener s.tcpListener6 = tcpListener
s.tcpPort6 = M.SocksaddrFromNet(tcpListener.Addr()).Port s.tcpPort6.Store(uint32(M.SocksaddrFromNet(tcpListener.Addr()).Port))
go s.acceptLoop(tcpListener) go s.acceptLoop(tcpListener, true)
} }
s.tcpNat = NewNat(s.ctx, s.udpTimeout) s.tcpNat = NewNat(s.ctx, s.udpTimeout)
udpNATOptions := s.udpNATOptions udpNATOptions := s.udpNATOptions
@ -367,11 +397,28 @@ func (s *System) processPacket(packet []byte) bool {
return writeBack return writeBack
} }
func (s *System) acceptLoop(listener net.Listener) { func (s *System) acceptLoop(listener net.Listener, isIPv6 bool) {
for { for {
conn, err := listener.Accept() conn, err := listener.Accept()
if err != nil { if err != nil {
return // lx/040 (SPECS/TASKS/040): upstream silently returns on ANY Accept
// error, leaving the stack alive but every new TCP SYN NAT-rewritten
// onto a dead port (instant RST) until a VPN restart — the LxBox §047
// "browser dead, QUIC alive" failure. A deliberate System.Close is the
// only quiet exit; anything else means the listener died out from
// under us (e.g. a foreign close on a reused fd number from the
// Java side of the shared Android process) — log it (the errno names
// the killer) and recreate the listener.
if s.closing.Load() {
return
}
newListener, healErr := s.healListener(listener, isIPv6, err)
if healErr != nil {
s.logger.Error("system stack: tcp", ipVersionSuffix(isIPv6), " accept loop died: ", err, "; relisten failed: ", healErr)
return
}
listener = newListener
continue
} }
connPort := M.SocksaddrFromNet(conn.RemoteAddr()).Port connPort := M.SocksaddrFromNet(conn.RemoteAddr()).Port
session := s.tcpNat.LookupBack(connPort) session := s.tcpNat.LookupBack(connPort)
@ -383,6 +430,47 @@ func (s *System) acceptLoop(listener net.Listener) {
} }
} }
// lx/040: recreate a TCP forwarder listener that died out from under the
// stack. Returns the replacement listener after publishing it (listener field
// + atomic port) under listenAccess, or an error if the stack is closing or
// the bind failed.
func (s *System) healListener(dead net.Listener, isIPv6 bool, cause error) (net.Listener, error) {
port := &s.tcpPort
if isIPv6 {
port = &s.tcpPort6
}
oldPort := port.Load()
s.logger.Warn("system stack: tcp", ipVersionSuffix(isIPv6), " listener (port ", oldPort, ") accept failed: ", cause, " — recreating listener")
_ = dead.Close() // release netpoll state; harmless if already closed
newListener, err := s.listenTCP(isIPv6)
if err != nil {
return nil, err
}
s.listenAccess.Lock()
defer s.listenAccess.Unlock()
if s.closing.Load() {
_ = newListener.Close()
return nil, net.ErrClosed
}
if isIPv6 {
s.tcpListener6 = newListener
} else {
s.tcpListener = newListener
}
newPort := uint32(M.SocksaddrFromNet(newListener.Addr()).Port)
port.Store(newPort)
recoveries := s.acceptRecoveries.Add(1)
s.logger.Warn("system stack: tcp", ipVersionSuffix(isIPv6), " listener recreated (port ", oldPort, " → ", newPort, ", recoveries: ", recoveries, ")")
return newListener, nil
}
func ipVersionSuffix(isIPv6 bool) string {
if isIPv6 {
return "6"
}
return "4"
}
func (s *System) dispatchIPv4(ipHdr header.IPv4, destination netip.Addr) bool { func (s *System) dispatchIPv4(ipHdr header.IPv4, destination netip.Addr) bool {
switch ipHdr.TransportProtocol() { switch ipHdr.TransportProtocol() {
case header.TCPProtocolNumber: case header.TCPProtocolNumber:
@ -392,7 +480,7 @@ func (s *System) dispatchIPv4(ipHdr header.IPv4, destination netip.Addr) bool {
if ipHdr.SourceAddr() == s.inet4Address && if ipHdr.SourceAddr() == s.inet4Address &&
ipHdr.FragmentOffset() == 0 && ipHdr.FragmentOffset() == 0 &&
len(ipHdr.Payload()) >= header.TCPMinimumSize && len(ipHdr.Payload()) >= header.TCPMinimumSize &&
header.TCP(ipHdr.Payload()).SourcePort() == s.tcpPort { header.TCP(ipHdr.Payload()).SourcePort() == uint16(s.tcpPort.Load()) {
return false return false
} }
case header.ICMPv4ProtocolNumber: case header.ICMPv4ProtocolNumber:
@ -411,7 +499,7 @@ func (s *System) dispatchIPv6(ipHdr header.IPv6, destination netip.Addr) bool {
} }
if ipHdr.SourceAddr() == s.inet6Address && if ipHdr.SourceAddr() == s.inet6Address &&
len(ipHdr.Payload()) >= header.TCPMinimumSize && len(ipHdr.Payload()) >= header.TCPMinimumSize &&
header.TCP(ipHdr.Payload()).SourcePort() == s.tcpPort6 { header.TCP(ipHdr.Payload()).SourcePort() == uint16(s.tcpPort6.Load()) {
return false return false
} }
case header.ICMPv6ProtocolNumber: case header.ICMPv6ProtocolNumber:
@ -475,7 +563,7 @@ func (s *System) processIPv4TCP(ipHdr header.IPv4, tcpHdr header.TCP) (bool, err
destination := netip.AddrPortFrom(ipHdr.DestinationAddr(), tcpHdr.DestinationPort()) destination := netip.AddrPortFrom(ipHdr.DestinationAddr(), tcpHdr.DestinationPort())
if !destination.Addr().IsGlobalUnicast() { if !destination.Addr().IsGlobalUnicast() {
return false, nil return false, nil
} else if source.Addr() == s.inet4Address && source.Port() == s.tcpPort { } else if source.Addr() == s.inet4Address && source.Port() == uint16(s.tcpPort.Load()) {
session := s.tcpNat.LookupBack(destination.Port()) session := s.tcpNat.LookupBack(destination.Port())
if session == nil { if session == nil {
return false, E.New("ipv4: tcp: session not found: ", destination.Port()) return false, E.New("ipv4: tcp: session not found: ", destination.Port())
@ -501,7 +589,7 @@ func (s *System) processIPv4TCP(ipHdr header.IPv4, tcpHdr header.TCP) (bool, err
} }
rewriteIPv4TCP(ipHdr, tcpHdr, s.txChecksumOffload, rewriteIPv4TCP(ipHdr, tcpHdr, s.txChecksumOffload,
s.inet4NextAddress, natPort, true, s.inet4NextAddress, natPort, true,
s.inet4Address, s.tcpPort, true) s.inet4Address, uint16(s.tcpPort.Load()), true)
} }
} }
return true, nil return true, nil
@ -512,7 +600,7 @@ func (s *System) processIPv6TCP(ipHdr header.IPv6, tcpHdr header.TCP) (bool, err
destination := netip.AddrPortFrom(ipHdr.DestinationAddr(), tcpHdr.DestinationPort()) destination := netip.AddrPortFrom(ipHdr.DestinationAddr(), tcpHdr.DestinationPort())
if !destination.Addr().IsGlobalUnicast() { if !destination.Addr().IsGlobalUnicast() {
return false, nil return false, nil
} else if source.Addr() == s.inet6Address && source.Port() == s.tcpPort6 { } else if source.Addr() == s.inet6Address && source.Port() == uint16(s.tcpPort6.Load()) {
session := s.tcpNat.LookupBack(destination.Port()) session := s.tcpNat.LookupBack(destination.Port())
if session == nil { if session == nil {
return false, E.New("ipv6: tcp: session not found: ", destination.Port()) return false, E.New("ipv6: tcp: session not found: ", destination.Port())
@ -538,7 +626,7 @@ func (s *System) processIPv6TCP(ipHdr header.IPv6, tcpHdr header.TCP) (bool, err
} }
rewriteIPv6TCP(ipHdr, tcpHdr, s.txChecksumOffload, rewriteIPv6TCP(ipHdr, tcpHdr, s.txChecksumOffload,
s.inet6NextAddress, natPort, true, s.inet6NextAddress, natPort, true,
s.inet6Address, s.tcpPort6, true) s.inet6Address, uint16(s.tcpPort6.Load()), true)
} }
} }
return true, nil return true, nil

View file

@ -0,0 +1,117 @@
package tun
// lx/040 (SPECS/TASKS/040-SINGTUN_ACCEPTLOOP_SELFHEAL): acceptLoop self-heal.
//
// Red/green против апстрима 2d9b8aed5fe2: там acceptLoop(listener) при любой
// ошибке Accept молча выходит навсегда — восстановления нет, порт не меняется,
// новый connect вечно бьётся в мёртвый сокет. Для red-прогона на чистом
// апстрим-чекауте достаточно адаптировать хелперы ниже (currentTCPPort →
// s.tcpPort, spawnAcceptLoop → go s.acceptLoop(ln)): тест упадёт по таймауту
// ожидания восстановления.
import (
"context"
"fmt"
"net"
"net/netip"
"testing"
"time"
"github.com/sagernet/sing/common/logger"
)
func newSelfHealTestSystem(t *testing.T) *System {
t.Helper()
s := &System{
ctx: context.Background(),
logger: logger.NOP(),
inet4Address: netip.MustParseAddr("127.0.0.1"),
udpTimeout: time.Minute,
}
s.tcpNat = NewNat(s.ctx, s.udpTimeout)
ln, err := s.listenTCP(false)
if err != nil {
t.Fatalf("listenTCP: %v", err)
}
s.tcpListener = ln
s.tcpPort.Store(uint32(ln.Addr().(*net.TCPAddr).Port))
spawnAcceptLoop(s, ln)
return s
}
func currentTCPPort(s *System) uint32 {
return s.tcpPort.Load()
}
func spawnAcceptLoop(s *System, ln net.Listener) {
go s.acceptLoop(ln, false)
}
func dialForwarder(t *testing.T, port uint32) error {
t.Helper()
conn, err := net.DialTimeout("tcp4", fmt.Sprintf("127.0.0.1:%d", port), time.Second)
if err == nil {
_ = conn.Close()
}
return err
}
// Убийство listener'а мимо System.Close (эмуляция чужого close по
// переиспользованному fd-номеру) должно приводить к пересозданию listener'а
// и продолжению приёма TCP, а не к вечной смерти петли.
func TestSystemAcceptLoopSelfHeal(t *testing.T) {
s := newSelfHealTestSystem(t)
oldPort := currentTCPPort(s)
if err := dialForwarder(t, oldPort); err != nil {
t.Fatalf("healthy listener refused connect: %v", err)
}
// Убить listener из-под стека: closing НЕ выставлен.
_ = s.tcpListener.Close()
deadline := time.Now().Add(5 * time.Second)
healed := false
for time.Now().Before(deadline) {
if s.acceptRecoveries.Load() > 0 {
healed = true
break
}
time.Sleep(10 * time.Millisecond)
}
if !healed {
t.Fatalf("acceptLoop did not recover within 5s (upstream behavior: silent permanent death)")
}
newPort := currentTCPPort(s)
if newPort == oldPort {
t.Fatalf("recovered port equals dead port %d — relisten did not publish a new port", oldPort)
}
if err := dialForwarder(t, newPort); err != nil {
t.Fatalf("connect to recreated listener (port %d) failed: %v", newPort, err)
}
if got := s.acceptRecoveries.Load(); got != 1 {
t.Fatalf("acceptRecoveries = %d, want 1", got)
}
s.closing.Store(true)
_ = s.tcpListener.Close()
}
// Штатное закрытие (closing выставлен, как это делает System.Close) обязано
// оставаться тихим: без пересозданий и без роста счётчика.
func TestSystemAcceptLoopQuietOnClose(t *testing.T) {
s := newSelfHealTestSystem(t)
oldPort := currentTCPPort(s)
s.closing.Store(true)
_ = s.tcpListener.Close()
time.Sleep(300 * time.Millisecond)
if got := s.acceptRecoveries.Load(); got != 0 {
t.Fatalf("deliberate close triggered %d recoveries, want 0", got)
}
if port := currentTCPPort(s); port != oldPort {
t.Fatalf("deliberate close changed port %d → %d", oldPort, port)
}
}