Quarantine unhealthy vGPU VFs via a persisted health store

cmd/api/api/resources.go

@@ -87,9 +87,14 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus {
 
 func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus {
 	result := oapi.GPUResourceStatus{
-		Mode:       oapi.GPUResourceStatusMode(gs.Mode),
-		TotalSlots: gs.TotalSlots,
-		UsedSlots:  gs.UsedSlots,
+		Mode:             oapi.GPUResourceStatusMode(gs.Mode),
+		TotalSlots:       gs.TotalSlots,
+		UsedSlots:        gs.UsedSlots,
+		AllocatableSlots: gs.AllocatableSlots,
+		QuarantinedSlots: gs.QuarantinedSlots,
+	}
+	if gs.PlacementDisabledReason != "" {
+		result.PlacementDisabledReason = &gs.PlacementDisabledReason
 	}
 
 	// Convert profiles (vGPU mode)

cmd/api/config/config.go

@@ -269,7 +269,8 @@ type SnapshotConfig struct {
 
 // GPUConfig holds GPU-related settings.
 type GPUConfig struct {
-	ProfileCacheTTL string `koanf:"profile_cache_ttl"`
+	ProfileCacheTTL       string `koanf:"profile_cache_ttl"`
+	VFQuarantineThreshold int    `koanf:"vf_quarantine_threshold"`
 }
 
 // Config is the top-level Hypeman server configuration.
@@ -494,7 +495,8 @@ func defaultConfig() *Config {
 		},
 
 		GPU: GPUConfig{
-			ProfileCacheTTL: "30m",
+			ProfileCacheTTL:       "30m",
+			VFQuarantineThreshold: 2,
 		},
 	}
 }
@@ -647,6 +649,9 @@ func (c *Config) Validate() error {
 	if c.Build.MaxConcurrentSourceBuilds <= 0 {
 		return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds)
 	}
+	if c.GPU.VFQuarantineThreshold < 1 {
+		return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold)
+	}
 	if c.Limits.MaxConcurrentPushes <= 0 {
 		return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes)
 	}

cmd/api/config/config_test.go

@@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) {
 	}
 }
 
+func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) {
+	for _, threshold := range []int{0, -1} {
+		cfg := defaultConfig()
+		cfg.GPU.VFQuarantineThreshold = threshold
+
+		err := cfg.Validate()
+		if err == nil {
+			t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold)
+		}
+	}
+}
+
 func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) {
 	cfg := defaultConfig()
 	cfg.Otel.MetricExportInterval = "not-a-duration"

cmd/api/main.go

@@ -204,6 +204,9 @@ func run() error {
 
 	// Configure GPU profile cache TTL
 	devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL)
+	if err := devices.InitVFHealth(paths.New(cfg.DataDir).VFHealthState(), cfg.GPU.VFQuarantineThreshold); err != nil {
+		slog.Error("failed to initialize VF health state; vGPU placement is disabled until the state file is repaired or the next write succeeds", "error", err)
+	}
 
 	// Initialize OpenTelemetry (before wire initialization)
 	otelCfg := otel.Config{

config.example.yaml

@@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman
 #   idle_ttl: ""                    # delete builders idle this long (e.g. "24h");
 #                                   # destructive, empty = disabled
 
+# gpu:
+#   profile_cache_ttl: 30m          # vGPU profile metadata cache TTL
+#   vf_quarantine_threshold: 2      # distinct instance assignments that must report
+#                                   # a guest driver init failure before the VF is
+#                                   # quarantined (must be >= 1)
+
 # =============================================================================
 # Resource Limits
 # =============================================================================

lib/devices/GPU.md

@@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu
   "mode": "vgpu",
   "total_slots": 64,
   "used_slots": 5,
+  "allocatable_slots": 57,
+  "quarantined_slots": 2,
   "profiles": [
-    {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59},
+    {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57},
     {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30},
     {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16}
   ]
@@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu
   "mode": "passthrough",
   "total_slots": 4,
   "used_slots": 2,
+  "allocatable_slots": 2,
+  "quarantined_slots": 0,
   "devices": [
     {"name": "NVIDIA L40S", "available": true},
     {"name": "NVIDIA L40S", "available": false}
@@ -185,8 +189,10 @@ Returns GPU status along with other resources:
     "mode": "vgpu",
     "total_slots": 64,
     "used_slots": 5,
+    "allocatable_slots": 57,
+    "quarantined_slots": 2,
     "profiles": [
-      {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}
+      {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}
     ]
   }
 }
@@ -282,10 +288,38 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)
 ```
 
 (0x65 = timeout; the guest's init requests are never answered, and
-`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because
-placement is deterministic least-loaded, an idle host re-picks the same VF for
-every request, so one wedged VF presents as all vGPU instances failing while
-`/resources` reports full capacity.
+`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle).
+
+Hypeman tracks these failures in `<data-dir>/gpu/vf-health.json` (it survives
+restarts): each reported init failure is tallied per instance assignment, and
+once failures accumulate from `gpu.vf_quarantine_threshold` distinct
+assignments (default 2), the VF is quarantined: excluded from placement and
+from advertised profile availability, and its parent GPU becomes
+overflow-only — deprioritized for new placements. Selection among a card's
+equivalent free VFs is randomized so a wedged VF cannot capture every
+placement. A reported init success clears failures only when that exact
+assignment has a recorded failure, removing the match and older tallies; if
+that assignment is the most recent failure recorded (the one that crossed
+the threshold), its later success also rescinds the quarantine. If the state
+file exists but cannot be loaded, or the last write to it failed, placement
+and advertised availability fail closed; the load or write is retried on the
+next placement or `/resources` read, so the store recovers on its own once
+the file is repaired or the disk is writable again.
+Recorded tallies are re-evaluated against the configured
+threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs
+whose persisted failures already meet the new value.
+
+`used_slots` includes quarantined VFs still held by running instances, so it
+can overlap `quarantined_slots`; use `allocatable_slots` for admission. While
+the store is unavailable, `allocatable_slots` is 0 and
+`placement_disabled_reason` carries the load or write error, so a broken
+state file is distinguishable from a full host. The
+`hypeman_resources_gpu_slots` gauge exports the same counts under
+`kind=allocatable` and `kind=quarantined`, and
+`hypeman_resources_gpu_placement_disabled` is 1 while the store is
+unavailable, so the condition is alertable without scraping `/resources`.
+
+Quarantine only removes capacity — it never touches a running instance.
 
 The wedge itself leaves no host-side log: no kernel error, no XID, no plugin
 crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
@@ -303,18 +337,45 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it.
 Confirm by assigning the same profile on a different VF: if that guest
 initializes, the VF is wedged, not the driver stack. Remediate by cycling
 SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it
-requires no vGPU assignments on that GPU):
+requires no vGPU assignments on that GPU). Quiescing the services that hold
+the GPU open is not optional: with `nv-hostengine`/`dcgm-exporter` or
+`nvidia-persistenced` attached, `sriov-manage -d` fails with `Cannot obtain
+unbindLock` on first contact.
+
+Any manual edit to `vf-health.json` needs an immediate hypeman restart: the
+store loads only at startup, and a failure report landing first re-persists
+the in-memory set over your edit. The restart does not disturb running VMs —
+startup reconciliation protects live VFs.
+
+**Draining the parent GPU.** Overflow-only is a preference, not a cordon:
+under capacity pressure new placements still land on the card's healthy VFs
+and refill it. To drain the card, quarantine all of its VFs by hand — add
+records to the versioned `vf-health.json` (`{"version": 1, "records":
+[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running
+instances are untouched and
+drain through their normal lifecycle: standby is blocked for vGPU instances,
+so only a running VM pins a VF, and each stop or delete frees one for good.
+Monitor by listing instances whose `gpu.device_path` sits under the parent
+GPU; once none remain, run the cycle below.
 
 ```bash
+# 1. Quiesce the services holding the GPU (required for the unbind lock).
+systemctl stop nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced
+
+# 2. Cycle SR-IOV on the parent GPU.
 /usr/lib/nvidia/sriov-manage -d <parent-gpu-pci-addr>
 /usr/lib/nvidia/sriov-manage -e <parent-gpu-pci-addr>
+
+# 3. Restart the quiesced services.
+systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter
 ```
 
+After the cycle, remove the card's entries from `vf-health.json`, restart,
+and boot a GPU instance to verify recovery.
+
 Do not unbind/rebind the VF from the nvidia driver — it breaks the
 nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`)
 and the VF stops accepting assignments entirely until the SR-IOV cycle.
-Services holding the GPU (DCGM, persistenced) must be stopped for the cycle
-to obtain the unbind lock.
 
 ### vGPU assignment fails
 

lib/devices/mdev_darwin.go

@@ -23,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) {
 }
 
 // ListGPUProfilesWithVFs returns an empty list on macOS.
-func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) {
+func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) {
 	return []GPUProfile{}, nil
 }
 

lib/devices/vendor_vfio_linux.go

@@ -12,7 +12,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 
 	"github.com/kernel/hypeman/lib/logger"
@@ -30,14 +29,11 @@ type vendorVFIOSysfs struct {
 	openVFIOPathsFunc func() (map[string]struct{}, error)
 }
 
-var (
-	hostVendorVFIO = vendorVFIOSysfs{
-		pciDevicesPath:  pciDevicesPath,
-		procPath:        procPath,
-		vfioDevicesPath: vfioDevicesPath,
-	}
-	vendorVFIOMu sync.Mutex
-)
+var hostVendorVFIO = vendorVFIOSysfs{
+	pciDevicesPath:  pciDevicesPath,
+	procPath:        procPath,
+	vfioDevicesPath: vfioDevicesPath,
+}
 
 func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 	entries, err := os.ReadDir(s.pciDevicesPath)
@@ -94,21 +90,23 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 	return vfs, nil
 }
 
-// listProfiles counts each free VF advertising a type as one creatable
-// instance, matching the driver-reported units that mdev sums through
-// available_instances. This is a best-effort snapshot because creating on one
-// VF may revoke the type from siblings that share its GPU framebuffer.
-func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) {
+// listProfiles counts each free, non-quarantined VF advertising a type as
+// one creatable instance, matching the driver-reported units that mdev sums
+// through available_instances. This is a best-effort snapshot because
+// creating on one VF may revoke the type from siblings that share its GPU
+// framebuffer.
+func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) {
 	profilesByType := make(map[string]VGPUProfileType)
 	creatableVFs := make(map[string]int)
 	profilesByVF, err := s.profileTypes(vfs)
 	if err != nil {
 		return nil, err
 	}
 	for _, vf := range vfs {
+		_, bad := quarantined[vf.PCIAddress]
 		for _, profile := range profilesByVF[vf.PCIAddress] {
 			profilesByType[profile.TypeName] = profile
-			if !vf.Allocated {
+			if !vf.Allocated && !bad {
 				creatableVFs[profile.TypeName]++
 			}
 		}
@@ -159,6 +157,17 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s
 	if profileType == "" || profileType == "0" {
 		return fmt.Errorf("invalid vendor VFIO vGPU profile type %q", profileType)
 	}
+	// Placement filters quarantined VFs from a snapshot taken outside this
+	// lock. Re-checking here, under the lock quarantine mutations take,
+	// closes the window where a VF is quarantined between selection and
+	// configuration.
+	quarantined, err := vfHealth.checkedAddresses()
+	if err != nil {
+		return err
+	}
+	if _, bad := quarantined[vfAddress]; bad {
+		return fmt.Errorf("vendor VFIO vGPU on VF %s is quarantined", vfAddress)
+	}
 	currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type")
 	currentType, err := readCurrentVGPUType(currentTypePath)
 	if err != nil {

lib/devices/vendor_vfio_linux_test.go

@@ -49,7 +49,7 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) {
 
 	vfs, err := sysfs.discoverVFs()
 	require.NoError(t, err)
-	profiles, err := sysfs.listProfiles(vfs)
+	profiles, err := sysfs.listProfiles(vfs, nil)
 	require.NoError(t, err)
 	assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q"))
 }
@@ -178,3 +178,47 @@ func assertFileValue(t *testing.T, path, expected string) {
 	require.NoError(t, err)
 	assert.Equal(t, expected, string(value))
 }
+
+func TestVendorVFIOConfigureRefusesQuarantinedVF(t *testing.T) {
+	resetVFHealthStore(t)
+	quarantineVF(t, "0000:82:00.4")
+
+	sysfs := newTestVendorVFIOSysfs(t)
+	const vfAddress = "0000:82:00.4"
+	sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "0", testCreatableTypes)
+
+	err := sysfs.configure(context.Background(), vfAddress, "1148")
+	require.ErrorContains(t, err, "is quarantined")
+	assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0")
+}
+
+func TestVendorVFIOConfigureFailsClosedWhenVFHealthUnavailable(t *testing.T) {
+	path := resetVFHealthStore(t)
+	require.NoError(t, os.WriteFile(path, []byte("not json"), 0644))
+	require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold))
+
+	sysfs := newTestVendorVFIOSysfs(t)
+	const vfAddress = "0000:82:00.4"
+	sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "0", testCreatableTypes)
+
+	err := sysfs.configure(context.Background(), vfAddress, "1148")
+	require.ErrorContains(t, err, "VF health state unavailable")
+	assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0")
+}
+
+func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) {
+	resetVFHealthStore(t)
+	quarantineVF(t, "0000:82:00.4")
+
+	sysfs := newTestVendorVFIOSysfs(t)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes)
+
+	vfs, err := sysfs.discoverVFs()
+	require.NoError(t, err)
+	availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs)
+	require.NoError(t, err)
+	profiles, err := sysfs.listProfiles(vfs, availability.Quarantined)
+	require.NoError(t, err)
+	assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q"))
+}