eburet-oss/wireguard-go-awg2-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rework outbound buffer management

Browse source 
Outbound element buffers now come from the sing allocator sized to the
actual packet instead of the bounded MaxMessageSize pool, element and
container pools become plain sync.Pools, and the bounded message buffer
pool serves only the receive path. Packets injected via
InputPacket/InputPackets are dropped before they are copied once a peer
has 2048 packets queued: injection runs on the caller's read loop, which
must never block on pool exhaustion, and the queues are bounded in
containers, so a flood was buffered instead of dropped.
This commit is contained in:
世界 2026-07-06 21:05:53 +08:00
parent 9de6dc32df
commit 8403cdb937
No known key found for this signature in database
GPG key ID: CD109927C34A63C4
5 changed files with 78 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

2
device/channels.go
View file

@ -126,7 +126,7 @@ func (device *Device) flushOutboundQueue(q *autodrainingOutboundQueue) {
case elemsContainer := <-q.c: case elemsContainer := <-q.c:
elemsContainer.Lock() elemsContainer.Lock()
for _, elem := range elemsContainer.elems { for _, elem := range elemsContainer.elems {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
} }
device.PutOutboundElementsContainer(elemsContainer) device.PutOutboundElementsContainer(elemsContainer)

8
device/device.go
View file

@ -71,11 +71,11 @@ type Device struct {
cookieChecker CookieChecker cookieChecker CookieChecker
pool struct { pool struct {
inboundElementsContainer *WaitPool inboundElementsContainer *sync.Pool
outboundElementsContainer *WaitPool outboundElementsContainer *sync.Pool
messageBuffers *WaitPool messageBuffers *WaitPool
inboundElements *WaitPool inboundElements *sync.Pool
outboundElements *WaitPool outboundElements *sync.Pool
} }
queue struct { queue struct {

3
device/peer.go
View file

@ -25,6 +25,8 @@ type Peer struct {
rxBytes atomic.Uint64 // bytes received from peer rxBytes atomic.Uint64 // bytes received from peer
lastHandshakeNano atomic.Int64 // nano seconds since epoch lastHandshakeNano atomic.Int64 // nano seconds since epoch
queuedOutboundPackets atomic.Int32 // packets in staged+outbound queues, for input backpressure
endpoint struct { endpoint struct {
sync.Mutex sync.Mutex
val conn.Endpoint val conn.Endpoint
@ -193,6 +195,7 @@ func (peer *Peer) Start() {
// reset routine state // reset routine state
peer.stopping.Wait() peer.stopping.Wait()
peer.stopping.Add(2) peer.stopping.Add(2)
peer.queuedOutboundPackets.Store(0)
peer.handshake.mutex.Lock() peer.handshake.mutex.Lock()
peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second)) peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second))

32
device/pools.go
View file

@ -7,6 +7,8 @@ package device
import ( import (
"sync" "sync"
"github.com/sagernet/sing/common/buf"
) )
type WaitPool struct { type WaitPool struct {
@ -47,23 +49,23 @@ func (p *WaitPool) Put(x any) {
} }
func (device *Device) PopulatePools() { func (device *Device) PopulatePools() {
device.pool.inboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any { device.pool.inboundElementsContainer = &sync.Pool{New: func() any {
s := make([]*QueueInboundElement, 0, device.BatchSize()) s := make([]*QueueInboundElement, 0, device.BatchSize())
return &QueueInboundElementsContainer{elems: s} return &QueueInboundElementsContainer{elems: s}
}) }}
device.pool.outboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any { device.pool.outboundElementsContainer = &sync.Pool{New: func() any {
s := make([]*QueueOutboundElement, 0, device.BatchSize()) s := make([]*QueueOutboundElement, 0, device.BatchSize())
return &QueueOutboundElementsContainer{elems: s} return &QueueOutboundElementsContainer{elems: s}
}) }}
device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() any { device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() any {
return new([MaxMessageSize]byte) return new([MaxMessageSize]byte)
}) })
device.pool.inboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any { device.pool.inboundElements = &sync.Pool{New: func() any {
return new(QueueInboundElement) return new(QueueInboundElement)
}) }}
device.pool.outboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any { device.pool.outboundElements = &sync.Pool{New: func() any {
return new(QueueOutboundElement) return new(QueueOutboundElement)
}) }}
} }
func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer { func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer {
@ -102,6 +104,20 @@ func (device *Device) PutMessageBuffer(msg *[MaxMessageSize]byte) {
device.pool.messageBuffers.Put(msg) device.pool.messageBuffers.Put(msg)
} }
// Outbound buffers come from the sing allocator instead of the bounded
// messageBuffers pool: the injection paths (InputPacket/InputPackets) run on
// the caller's shared read loop, which must never block on pool exhaustion,
// and their packets are far smaller than MaxMessageSize, so they are allocated
// by actual size. This also keeps the bounded pool exclusively for the receive
// path, so outbound backlog can no longer starve it.
func (device *Device) GetOutboundBuffer(size int) []byte {
return buf.Get(size)
}
func (device *Device) PutOutboundBuffer(buffer []byte) {
_ = buf.Put(buffer)
}
func (device *Device) GetInboundElement() *QueueInboundElement { func (device *Device) GetInboundElement() *QueueInboundElement {
return device.pool.inboundElements.Get().(*QueueInboundElement) return device.pool.inboundElements.Get().(*QueueInboundElement)
} }

68
device/send.go
View file

@ -45,7 +45,7 @@ import (
*/ */
type QueueOutboundElement struct { type QueueOutboundElement struct {
buffer *[MaxMessageSize]byte // slice holding the packet data buffer []byte // sing-allocated buffer holding the packet data
// packet is always a slice of "buffer". The starting offset in buffer // packet is always a slice of "buffer". The starting offset in buffer
// is either: // is either:
// a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext) // a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext)
@ -63,7 +63,7 @@ type QueueOutboundElementsContainer struct {
func (device *Device) NewOutboundElement() *QueueOutboundElement { func (device *Device) NewOutboundElement() *QueueOutboundElement {
elem := device.GetOutboundElement() elem := device.GetOutboundElement()
elem.buffer = device.GetMessageBuffer() elem.buffer = device.GetOutboundBuffer(MaxMessageSize)
elem.nonce = 0 elem.nonce = 0
// keypair and peer were cleared (if necessary) by clearPointers. // keypair and peer were cleared (if necessary) by clearPointers.
return elem return elem
@ -89,9 +89,10 @@ func (peer *Peer) SendKeepalive() {
elemsContainer.elems = append(elemsContainer.elems, elem) elemsContainer.elems = append(elemsContainer.elems, elem)
select { select {
case peer.queue.staged <- elemsContainer: case peer.queue.staged <- elemsContainer:
peer.queuedOutboundPackets.Add(1)
peer.device.log.Verbosef("%v - Sending keepalive packet", peer) peer.device.log.Verbosef("%v - Sending keepalive packet", peer)
default: default:
peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundBuffer(elem.buffer)
peer.device.PutOutboundElement(elem) peer.device.PutOutboundElement(elem)
peer.device.PutOutboundElementsContainer(elemsContainer) peer.device.PutOutboundElementsContainer(elemsContainer)
} }
@ -238,7 +239,7 @@ func (device *Device) RoutineReadFromTUN() {
defer func() { defer func() {
for _, elem := range elems { for _, elem := range elems {
if elem != nil { if elem != nil {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
} }
} }
@ -295,7 +296,7 @@ func (device *Device) RoutineReadFromTUN() {
peer.SendStagedPackets() peer.SendStagedPackets()
} else { } else {
for _, elem := range elemsForPeer.elems { for _, elem := range elemsForPeer.elems {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
} }
device.PutOutboundElementsContainer(elemsForPeer) device.PutOutboundElementsContainer(elemsForPeer)
@ -322,22 +323,33 @@ func (device *Device) RoutineReadFromTUN() {
} }
} }
// maxQueuedInputPackets bounds the staged+outbound backlog of a peer fed via
// InputPacket/InputPackets. Injected packets beyond it are dropped before they
// are copied into pooled message buffers, like a full qdisc: injection has no
// flow control, and the queues are bounded in containers (up to a full batch
// each), so without this cap a flood is buffered instead of dropped.
const maxQueuedInputPackets = 2048
func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) { func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) {
peer := device.allowedips.Lookup(destination) peer := device.allowedips.Lookup(destination)
if peer == nil { if peer == nil {
return return
} }
elem := device.NewOutboundElement() if peer.queuedOutboundPackets.Load() >= maxQueuedInputPackets {
packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:] return
}
var totalLength int var totalLength int
for _, packetSlice := range packetSlices { for _, packetSlice := range packetSlices {
totalLength += len(packetSlice) totalLength += len(packetSlice)
} }
if totalLength > len(packet) { allocLength := MessageEncapsulatingTransportSize + MessageTransportHeaderSize + totalLength + PaddingMultiple + chacha20poly1305.Overhead
device.PutMessageBuffer(elem.buffer) if allocLength > MaxMessageSize {
device.PutOutboundElement(elem)
return return
} }
elem := device.GetOutboundElement()
elem.buffer = device.GetOutboundBuffer(allocLength)
elem.nonce = 0
packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:]
var n int var n int
for _, packetSlice := range packetSlices { for _, packetSlice := range packetSlices {
n += copy(packet[n:], packetSlice) n += copy(packet[n:], packetSlice)
@ -349,7 +361,7 @@ func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) {
peer.StagePackets(elemsForPeer) peer.StagePackets(elemsForPeer)
peer.SendStagedPackets() peer.SendStagedPackets()
} else { } else {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
device.PutOutboundElementsContainer(elemsForPeer) device.PutOutboundElementsContainer(elemsForPeer)
} }
@ -369,17 +381,21 @@ func (device *Device) InputPackets(packets []*InputPacketRef) []*InputPacketRef
unmatched = append(unmatched, packetRef) unmatched = append(unmatched, packetRef)
continue continue
} }
elem := device.NewOutboundElement() if peer.queuedOutboundPackets.Load() >= maxQueuedInputPackets {
packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:] continue
}
var totalLength int var totalLength int
for _, packetSlice := range packetRef.PacketSlices { for _, packetSlice := range packetRef.PacketSlices {
totalLength += len(packetSlice) totalLength += len(packetSlice)
} }
if totalLength > len(packet) { allocLength := MessageEncapsulatingTransportSize + MessageTransportHeaderSize + totalLength + PaddingMultiple + chacha20poly1305.Overhead
device.PutMessageBuffer(elem.buffer) if allocLength > MaxMessageSize {
device.PutOutboundElement(elem)
continue continue
} }
elem := device.GetOutboundElement()
elem.buffer = device.GetOutboundBuffer(allocLength)
elem.nonce = 0
packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:]
var n int var n int
for _, packetSlice := range packetRef.PacketSlices { for _, packetSlice := range packetRef.PacketSlices {
n += copy(packet[n:], packetSlice) n += copy(packet[n:], packetSlice)
@ -398,7 +414,7 @@ func (device *Device) InputPackets(packets []*InputPacketRef) []*InputPacketRef
peer.SendStagedPackets() peer.SendStagedPackets()
} else { } else {
for _, elem := range elemsForPeer.elems { for _, elem := range elemsForPeer.elems {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
} }
device.PutOutboundElementsContainer(elemsForPeer) device.PutOutboundElementsContainer(elemsForPeer)
@ -408,6 +424,7 @@ func (device *Device) InputPackets(packets []*InputPacketRef) []*InputPacketRef
} }
func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) { func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) {
peer.queuedOutboundPackets.Add(int32(len(elems.elems)))
for { for {
select { select {
case peer.queue.staged <- elems: case peer.queue.staged <- elems:
@ -416,8 +433,9 @@ func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) {
} }
select { select {
case tooOld := <-peer.queue.staged: case tooOld := <-peer.queue.staged:
peer.queuedOutboundPackets.Add(-int32(len(tooOld.elems)))
for _, elem := range tooOld.elems { for _, elem := range tooOld.elems {
peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundBuffer(elem.buffer)
peer.device.PutOutboundElement(elem) peer.device.PutOutboundElement(elem)
} }
peer.device.PutOutboundElementsContainer(tooOld) peer.device.PutOutboundElementsContainer(tooOld)
@ -464,6 +482,8 @@ top:
elemsContainer.elems = elemsContainer.elems[:i] elemsContainer.elems = elemsContainer.elems[:i]
if elemsContainerOOO != nil { if elemsContainerOOO != nil {
// Already counted at their original staging; StagePackets will count them again.
peer.queuedOutboundPackets.Add(-int32(len(elemsContainerOOO.elems)))
peer.StagePackets(elemsContainerOOO) // XXX: Out of order, but we can't front-load go chans peer.StagePackets(elemsContainerOOO) // XXX: Out of order, but we can't front-load go chans
} }
@ -477,8 +497,9 @@ top:
peer.queue.outbound.c <- elemsContainer peer.queue.outbound.c <- elemsContainer
peer.device.queue.encryption.c <- elemsContainer peer.device.queue.encryption.c <- elemsContainer
} else { } else {
peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
for _, elem := range elemsContainer.elems { for _, elem := range elemsContainer.elems {
peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundBuffer(elem.buffer)
peer.device.PutOutboundElement(elem) peer.device.PutOutboundElement(elem)
} }
peer.device.PutOutboundElementsContainer(elemsContainer) peer.device.PutOutboundElementsContainer(elemsContainer)
@ -497,8 +518,9 @@ func (peer *Peer) FlushStagedPackets() {
for { for {
select { select {
case elemsContainer := <-peer.queue.staged: case elemsContainer := <-peer.queue.staged:
peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
for _, elem := range elemsContainer.elems { for _, elem := range elemsContainer.elems {
peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundBuffer(elem.buffer)
peer.device.PutOutboundElement(elem) peer.device.PutOutboundElement(elem)
} }
peer.device.PutOutboundElementsContainer(elemsContainer) peer.device.PutOutboundElementsContainer(elemsContainer)
@ -592,8 +614,9 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) {
// TODO: rework peer shutdown order to ensure // TODO: rework peer shutdown order to ensure
// that we never accidentally keep timers alive longer than necessary. // that we never accidentally keep timers alive longer than necessary.
elemsContainer.Lock() elemsContainer.Lock()
peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
for _, elem := range elemsContainer.elems { for _, elem := range elemsContainer.elems {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
} }
device.PutOutboundElementsContainer(elemsContainer) device.PutOutboundElementsContainer(elemsContainer)
@ -615,8 +638,9 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) {
if dataSent { if dataSent {
peer.timersDataSent() peer.timersDataSent()
} }
peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
for _, elem := range elemsContainer.elems { for _, elem := range elemsContainer.elems {
device.PutMessageBuffer(elem.buffer) device.PutOutboundBuffer(elem.buffer)
device.PutOutboundElement(elem) device.PutOutboundElement(elem)
} }
device.PutOutboundElementsContainer(elemsContainer) device.PutOutboundElementsContainer(elemsContainer)