snapshot: sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 + SPEC 048 guard
Обновление снапшота с v0.0.0-20250811.0 на пин, которого требует sing-box после мержа 235 коммитов (upstream d620bbbf2 "Update gvisor to 20260727.0"). Прежний снапшот был взят 2026-08-04 ровно с той версии, на которой тогда стоял апстрим; разрыв возник 2026-08-05 вместе с его бампом. За год апстрим-gvisor изменил ~14 000 строк в 292 файлах. Значимое для нас — сетевой стек: tcp/connect.go (PMTU-discovery + исправление начального RTT/RTO: раньше задержка ACK внутри стека завышала стартовый таймаут на несколько RTT), tcp/snd.go, tcp/rcv.go, stack/conntrack.go, stack/packet_buffer.go. Всего 30 файлов в TCP и 37 в stack. Баг SPEC 048 апстрим НЕ исправил — проверено по коду новой версии: handleConnecting по-прежнему проверяет состояние endpoint'а, но не ep.h, а performHandshake так же зануляет h и отпускает мьютекс до Close(). Поэтому guard перенесён (12 строк) вместе со своим тестом (45 строк). Red/green проверен на новой базе: без guard'а тест падает с той же nil-паникой, что в полевом крашдампе; с ним зелёный.
This commit is contained in:
Leadaxe
parent
ffebe42860
commit
117243aa02
293 changed files with 16413 additions and 2842 deletions
442
pkg/rdma/collect.go
Normal file
442
pkg/rdma/collect.go
Normal file
|
|
@ -0,0 +1,442 @@
|
|||
// Copyright 2026 The gVisor Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rdma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// UverbsSpec identifies one /dev/infiniband/uverbs* device from the
|
||||
// container's OCI spec.
|
||||
type UverbsSpec struct {
|
||||
// Name is the device file name, e.g. "uverbs0".
|
||||
Name string
|
||||
// Major and Minor are the char device numbers from the spec.
|
||||
Major int64
|
||||
Minor int64
|
||||
}
|
||||
|
||||
// Static PCI attribute files captured at every level of the hierarchy.
|
||||
// uevent is captured verbatim (NCCL parses PCI_SLOT_NAME out of it;
|
||||
// libibverbs matches drivers on modalias).
|
||||
var pciAttrNames = []string{
|
||||
"class", "vendor", "device", "subsystem_vendor", "subsystem_device",
|
||||
"revision", "numa_node", "local_cpus", "local_cpulist",
|
||||
"max_link_speed", "max_link_width", "current_link_speed",
|
||||
"current_link_width", "modalias", "uevent",
|
||||
}
|
||||
|
||||
// Static identity attributes of /sys/class/infiniband/<ibdev>/.
|
||||
var ibAttrNames = []string{
|
||||
"node_type", "node_guid", "sys_image_guid", "fw_ver", "board_id",
|
||||
"hca_type", "hw_rev", "node_desc",
|
||||
}
|
||||
|
||||
// Per-port attributes served live (they change at runtime: the RoCE GID
|
||||
// table repopulates when netdevs move namespaces and acquire addresses;
|
||||
// link state and rate can change on retrain).
|
||||
var portLiveAttrNames = []string{
|
||||
"state", "phys_state", "rate", "lid", "sm_lid", "sm_sl",
|
||||
}
|
||||
|
||||
// Per-port attributes that are fixed for the sandbox lifetime.
|
||||
var portStaticAttrNames = []string{"link_layer", "cap_mask"}
|
||||
|
||||
// Curated netdev attribute set (see NetDev for why these are static).
|
||||
var netAttrNames = []string{
|
||||
"address", "addr_len", "type", "dev_id", "dev_port", "mtu", "speed",
|
||||
"duplex", "operstate", "carrier", "ifindex",
|
||||
}
|
||||
|
||||
// NUMA aggregate range files.
|
||||
var numaAggregateNames = []string{
|
||||
"online", "possible", "has_cpu", "has_memory", "has_normal_memory",
|
||||
}
|
||||
|
||||
// GPU/accelerator PCI classes included as leaves (beyond the NIC leaves
|
||||
// derived from the spec): 3D controller, VGA controller, NVSwitch bridge.
|
||||
var gpuClassPrefixes = []string{"0x0302", "0x0300", "0x0680"}
|
||||
|
||||
// Collect builds a snapshot for the given spec devices by reading host
|
||||
// sysfs rooted at sysRoot (normally "/sys"; overridable for tests).
|
||||
//
|
||||
// The snapshot is collected by runsc during sandbox creation (while host
|
||||
// sysfs is still reachable, before pivot_root) and consumed by the sentry's
|
||||
// sysfs implementation to build a virtual /sys tree that is path-identical
|
||||
// to the host layout: one canonical /sys/devices/pci... subtree per device,
|
||||
// with /sys/class/* and /sys/bus/pci/devices as pure symlink farms. Path
|
||||
// identity matters because consumers (libibverbs, NCCL) resolve symlinks
|
||||
// with realpath() and walk the resolved paths.
|
||||
//
|
||||
// Only devices derived from the container's OCI spec are captured: the
|
||||
// closure starts at the spec's /dev/infiniband/uverbs* entries and expands
|
||||
// to their ibdevs, associated netdevs, PCI ancestor chains, GPU PCI
|
||||
// functions, and the NUMA node topology. Nothing outside that closure is
|
||||
// exposed to the sandbox.
|
||||
//
|
||||
// It must run while the RDMA netdevs are still in the host netns.
|
||||
func Collect(sysRoot string, uverbs []UverbsSpec) (*Snapshot, error) {
|
||||
if len(uverbs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
abi, err := mustReadAttr(path.Join(sysRoot, "class/infiniband_verbs/abi_version"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading verbs abi_version: %w", err)
|
||||
}
|
||||
s := &Snapshot{VerbsABIVersion: abi}
|
||||
|
||||
// pciPaths accumulates every PCI directory (relative to sysRoot) in
|
||||
// the closure; expanded with ancestors below.
|
||||
pciPaths := make(map[string]bool)
|
||||
|
||||
for _, u := range uverbs {
|
||||
if !SafeName(u.Name) {
|
||||
return nil, fmt.Errorf("unsafe uverbs device name %q", u.Name)
|
||||
}
|
||||
uvDir := path.Join(sysRoot, "class/infiniband_verbs", u.Name)
|
||||
// dev is reproduced verbatim as the uverbs "dev" file, but the
|
||||
// major:minor check needs the value without its trailing newline.
|
||||
dev, err := mustReadAttr(path.Join(uvDir, "dev"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uverbs device %q: reading dev: %w", u.Name, err)
|
||||
}
|
||||
if want := fmt.Sprintf("%d:%d", u.Major, u.Minor); strings.TrimSpace(dev) != want {
|
||||
return nil, fmt.Errorf("uverbs device %q: sysfs dev is %q but the "+
|
||||
"OCI spec says %q; the host may have renumbered devices since "+
|
||||
"the spec was built", u.Name, strings.TrimSpace(dev), want)
|
||||
}
|
||||
// ibdev is a directory name (used in paths and symlink targets), so
|
||||
// keep the trimmed value; its file reproduction re-adds the newline.
|
||||
ibdevRaw, err := mustReadAttr(path.Join(uvDir, "ibdev"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uverbs device %q: reading ibdev: %w", u.Name, err)
|
||||
}
|
||||
ibdev := strings.TrimSpace(ibdevRaw)
|
||||
if !SafeName(ibdev) {
|
||||
return nil, fmt.Errorf("uverbs device %q: unusable ibdev name %q", u.Name, ibdev)
|
||||
}
|
||||
ibDir := path.Join(sysRoot, "class/infiniband", ibdev)
|
||||
|
||||
leaf, err := relRealpath(sysRoot, path.Join(ibDir, "device"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolving PCI device of %s: %w", ibdev, err)
|
||||
}
|
||||
if err := addWithAncestors(pciPaths, leaf); err != nil {
|
||||
return nil, fmt.Errorf("ibdev %s: %w", ibdev, err)
|
||||
}
|
||||
|
||||
abiVersion, err := mustReadAttr(path.Join(uvDir, "abi_version"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uverbs device %q: reading abi_version: %w", u.Name, err)
|
||||
}
|
||||
ibAttrs, err := readAttrs(ibDir, ibAttrNames)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ibdev %s: %w", ibdev, err)
|
||||
}
|
||||
d := Device{
|
||||
Uverbs: u.Name,
|
||||
IBDev: ibdev,
|
||||
LeafPCI: leaf,
|
||||
Dev: dev,
|
||||
ABIVersion: abiVersion,
|
||||
IBAttrs: ibAttrs,
|
||||
Ports: map[string]Port{},
|
||||
}
|
||||
if err := d.collectPorts(ibDir); err != nil {
|
||||
return nil, fmt.Errorf("ibdev %s: %w", ibdev, err)
|
||||
}
|
||||
if err := d.collectNetDevs(sysRoot, path.Join(ibDir, "device/net")); err != nil {
|
||||
return nil, fmt.Errorf("ibdev %s: %w", ibdev, err)
|
||||
}
|
||||
s.Devices = append(s.Devices, d)
|
||||
}
|
||||
|
||||
// GPU/accelerator leaves by PCI class scan; fold each into the PCI
|
||||
// closure so its ancestors are materialized alongside the NIC leaves.
|
||||
gpus, err := gpuLeaves(sysRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, g := range gpus {
|
||||
if err := addWithAncestors(pciPaths, g); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Materialize every PCI node with its static attributes.
|
||||
for p := range pciPaths {
|
||||
attrs, err := readAttrs(path.Join(sysRoot, p), pciAttrNames)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("PCI node %q: %w", p, err)
|
||||
}
|
||||
s.PCINodes = append(s.PCINodes, PCINode{Path: p, Attrs: attrs})
|
||||
}
|
||||
sort.Slice(s.PCINodes, func(i, j int) bool { return s.PCINodes[i].Path < s.PCINodes[j].Path })
|
||||
|
||||
numa, err := collectNUMA(sysRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.NUMA = numa
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// addWithAncestors records leaf and every ancestor directory whose name is
|
||||
// a PCI function or root complex. The walk is textual: no symlinks are
|
||||
// followed (leaf is already fully resolved).
|
||||
func addWithAncestors(set map[string]bool, leaf string) error {
|
||||
if !strings.HasPrefix(leaf, "devices/pci") {
|
||||
return fmt.Errorf("PCI path %q is not under devices/pci*", leaf)
|
||||
}
|
||||
p := leaf
|
||||
for {
|
||||
base := path.Base(p)
|
||||
if bdfRE.MatchString(base) {
|
||||
set[p] = true
|
||||
p = path.Dir(p)
|
||||
continue
|
||||
}
|
||||
if pciRootRE.MatchString(base) {
|
||||
set[p] = true
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unexpected component %q in PCI path %q", base, leaf)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Device) collectPorts(ibDir string) error {
|
||||
portsDir := path.Join(ibDir, "ports")
|
||||
ents, err := os.ReadDir(portsDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading %q: %w", portsDir, err)
|
||||
}
|
||||
for _, e := range ents {
|
||||
num := e.Name()
|
||||
if !SafeName(num) {
|
||||
continue
|
||||
}
|
||||
pDir := path.Join(portsDir, num)
|
||||
staticAttrs, err := readAttrs(pDir, portStaticAttrNames)
|
||||
if err != nil {
|
||||
return fmt.Errorf("port %s: %w", num, err)
|
||||
}
|
||||
// gids is a mandatory part of every IB/RoCE port; its absence
|
||||
// means a broken port tree, so any failure is fatal. counters and
|
||||
// hw_counters vary by device, so a missing directory is tolerated.
|
||||
gids, err := listSafeNames(path.Join(pDir, "gids"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("port %s: %w", num, err)
|
||||
}
|
||||
counters, err := listOptionalNames(path.Join(pDir, "counters"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("port %s: %w", num, err)
|
||||
}
|
||||
hwCounters, err := listOptionalNames(path.Join(pDir, "hw_counters"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("port %s: %w", num, err)
|
||||
}
|
||||
d.Ports[num] = Port{
|
||||
StaticAttrs: staticAttrs,
|
||||
LiveAttrs: append([]string{}, portLiveAttrNames...),
|
||||
GIDNames: gids,
|
||||
CounterNames: counters,
|
||||
HWCounterNames: hwCounters,
|
||||
}
|
||||
}
|
||||
if len(d.Ports) == 0 {
|
||||
return fmt.Errorf("no ports under %q", portsDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Device) collectNetDevs(sysRoot, netDir string) error {
|
||||
ents, err := os.ReadDir(netDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// InfiniBand-link-layer devices may have no netdev; fine.
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("reading %q: %w", netDir, err)
|
||||
}
|
||||
for _, e := range ents {
|
||||
name := e.Name()
|
||||
if !SafeName(name) {
|
||||
continue
|
||||
}
|
||||
attrs, err := readAttrs(path.Join(sysRoot, "class/net", name), netAttrNames)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.NetDevs = append(d.NetDevs, NetDev{Name: name, Attrs: attrs})
|
||||
}
|
||||
sort.Slice(d.NetDevs, func(i, j int) bool { return d.NetDevs[i].Name < d.NetDevs[j].Name })
|
||||
return nil
|
||||
}
|
||||
|
||||
// gpuLeaves returns the leaf paths (relative to sysRoot) of every host GPU,
|
||||
// found by scanning /sys/bus/pci/devices for the GPU PCI classes. NCCL needs
|
||||
// the GPU PCI positions to compute GPU<->NIC distance. We include all host
|
||||
// GPUs: the exposure is read-only PCI metadata (NCCL discovers GPUs via CUDA,
|
||||
// not sysfs, so extra nodes are inert).
|
||||
func gpuLeaves(sysRoot string) ([]string, error) {
|
||||
busDir := path.Join(sysRoot, "bus/pci/devices")
|
||||
ents, err := os.ReadDir(busDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// No PCI bus exposed (e.g. tests); GPU-less snapshot.
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("reading %q: %w", busDir, err)
|
||||
}
|
||||
var leaves []string
|
||||
for _, e := range ents {
|
||||
// This scans unrelated host PCI devices, so a device that vanishes
|
||||
// mid-scan (absent class) is skipped rather than failing collection;
|
||||
// other read errors are still surfaced.
|
||||
class, present, err := readAttr(path.Join(busDir, e.Name(), "class"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading class of PCI device %q: %w", e.Name(), err)
|
||||
}
|
||||
if !present {
|
||||
continue
|
||||
}
|
||||
match := false
|
||||
for _, p := range gpuClassPrefixes {
|
||||
if strings.HasPrefix(strings.TrimSpace(class), p) {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
leaf, err := relRealpath(sysRoot, path.Join(busDir, e.Name()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolving PCI path of GPU %q: %w", e.Name(), err)
|
||||
}
|
||||
leaves = append(leaves, leaf)
|
||||
}
|
||||
return leaves, nil
|
||||
}
|
||||
|
||||
func collectNUMA(sysRoot string) (*NUMA, error) {
|
||||
nodeDir := path.Join(sysRoot, "devices/system/node")
|
||||
if _, err := os.Stat(nodeDir); err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Non-NUMA system (or tests): no node topology to expose.
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("statting %q: %w", nodeDir, err)
|
||||
}
|
||||
agg, err := readAttrs(nodeDir, numaAggregateNames)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &NUMA{Aggregate: agg}, nil
|
||||
}
|
||||
|
||||
// relRealpath resolves p and returns it relative to sysRoot.
|
||||
func relRealpath(sysRoot, p string) (string, error) {
|
||||
r, err := filepath.EvalSymlinks(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, err := filepath.Rel(sysRoot, r)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return "", fmt.Errorf("%q resolves outside %q", p, sysRoot)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
// readAttr reads the attribute file at p and returns its contents. present is
|
||||
// false only when the file is absent (ENOENT). Any other error (permissions,
|
||||
// I/O) is returned rather than swallowed.
|
||||
func readAttr(p string) (content string, present bool, err error) {
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", false, nil
|
||||
}
|
||||
return "", false, err
|
||||
}
|
||||
return string(b), true, nil
|
||||
}
|
||||
|
||||
// mustReadAttr reads an attribute file that is expected to have. Unlike
|
||||
// readAttr, it treats absence (ENOENT) as an error.
|
||||
func mustReadAttr(p string) (string, error) {
|
||||
content, present, err := readAttr(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !present {
|
||||
return "", fmt.Errorf("required sysfs file %q is missing", p)
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// readAttrs reads the named attribute files under dir. Absent files (ENOENT)
|
||||
// are omitted so the sandbox mirrors the host's exact set of files. Any
|
||||
// non-ENOENT read error aborts collection.
|
||||
func readAttrs(dir string, names []string) (map[string]string, error) {
|
||||
m := make(map[string]string)
|
||||
for _, name := range names {
|
||||
p := path.Join(dir, name)
|
||||
content, present, err := readAttr(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %q: %w", p, err)
|
||||
}
|
||||
if present {
|
||||
m[name] = content
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// listSafeNames returns the safe entry names under dir, sorted. The ReadDir
|
||||
// error is returned (never swallowed); callers decide whether an absent
|
||||
// directory is acceptable (see listOptionalNames).
|
||||
func listSafeNames(dir string) ([]string, error) {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %q: %w", dir, err)
|
||||
}
|
||||
var names []string
|
||||
for _, e := range ents {
|
||||
if SafeName(e.Name()) {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// listOptionalNames is listSafeNames for a directory that may legitimately be
|
||||
// absent: a missing directory yields (nil, nil), but a read error is still
|
||||
// surfaced.
|
||||
func listOptionalNames(dir string) ([]string, error) {
|
||||
names, err := listSafeNames(dir)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return names, err
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue