diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..db38fd0fc 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,14 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, + } + if gs.PlacementDisabledReason != "" { + result.PlacementDisabledReason = &gs.PlacementDisabledReason } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index 6084ed52f..9eb676622 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,9 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + if err := devices.InitVFHealth(paths.New(cfg.DataDir).VFHealthState(), cfg.GPU.VFQuarantineThreshold); err != nil { + slog.Error("failed to initialize VF health state; vGPU placement is disabled until the state file is repaired or the next write succeeds", "error", err) + } // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index f06af1983..d5539b35f 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -185,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -282,10 +288,38 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) ``` (0x65 = timeout; the guest's init requests are never answered, and -`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because -placement is deterministic least-loaded, an idle host re-picks the same VF for -every request, so one wedged VF presents as all vGPU instances failing while -`/resources` reports full capacity. +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman tracks these failures in `/gpu/vf-health.json` (it survives +restarts): each reported init failure is tallied per instance assignment, and +once failures accumulate from `gpu.vf_quarantine_threshold` distinct +assignments (default 2), the VF is quarantined: excluded from placement and +from advertised profile availability, and its parent GPU becomes +overflow-only — deprioritized for new placements. Selection among a card's +equivalent free VFs is randomized so a wedged VF cannot capture every +placement. A reported init success clears failures only when that exact +assignment has a recorded failure, removing the match and older tallies; if +that assignment is the most recent failure recorded (the one that crossed +the threshold), its later success also rescinds the quarantine. If the state +file exists but cannot be loaded, or the last write to it failed, placement +and advertised availability fail closed; the load or write is retried on the +next placement or `/resources` read, so the store recovers on its own once +the file is repaired or the disk is writable again. +Recorded tallies are re-evaluated against the configured +threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs +whose persisted failures already meet the new value. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. While +the store is unavailable, `allocatable_slots` is 0 and +`placement_disabled_reason` carries the load or write error, so a broken +state file is distinguishable from a full host. The +`hypeman_resources_gpu_slots` gauge exports the same counts under +`kind=allocatable` and `kind=quarantined`, and +`hypeman_resources_gpu_placement_disabled` is 1 while the store is +unavailable, so the condition is alertable without scraping `/resources`. + +Quarantine only removes capacity — it never touches a running instance. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is @@ -303,18 +337,45 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU): +requires no vGPU assignments on that GPU). Quiescing the services that hold +the GPU open is not optional: with `nv-hostengine`/`dcgm-exporter` or +`nvidia-persistenced` attached, `sriov-manage -d` fails with `Cannot obtain +unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. ```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced + +# 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter ``` +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. + Do not unbind/rebind the VF from the nvidia driver — it breaks the nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) and the VF stops accepting assignments entirely until the SR-IOV cycle. -Services holding the GPU (DCGM, persistenced) must be stopped for the cycle -to obtain the unbind lock. ### vGPU assignment fails diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index cc7b0e78c..f0f009d34 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -23,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { return []GPUProfile{}, nil } diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 7b047d5d9..2fb552095 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -12,7 +12,6 @@ import ( "sort" "strconv" "strings" - "sync" "syscall" "github.com/kernel/hypeman/lib/logger" @@ -30,14 +29,11 @@ type vendorVFIOSysfs struct { openVFIOPathsFunc func() (map[string]struct{}, error) } -var ( - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - } - vendorVFIOMu sync.Mutex -) +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, +} func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(s.pciDevicesPath) @@ -94,11 +90,12 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free VF advertising a type as one creatable -// instance, matching the driver-reported units that mdev sums through -// available_instances. This is a best-effort snapshot because creating on one -// VF may revoke the type from siblings that share its GPU framebuffer. -func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { +// listProfiles counts each free, non-quarantined VF advertising a type as +// one creatable instance, matching the driver-reported units that mdev sums +// through available_instances. This is a best-effort snapshot because +// creating on one VF may revoke the type from siblings that share its GPU +// framebuffer. +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { profilesByType := make(map[string]VGPUProfileType) creatableVFs := make(map[string]int) profilesByVF, err := s.profileTypes(vfs) @@ -106,9 +103,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro return nil, err } for _, vf := range vfs { + _, bad := quarantined[vf.PCIAddress] for _, profile := range profilesByVF[vf.PCIAddress] { profilesByType[profile.TypeName] = profile - if !vf.Allocated { + if !vf.Allocated && !bad { creatableVFs[profile.TypeName]++ } } @@ -159,6 +157,17 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s if profileType == "" || profileType == "0" { return fmt.Errorf("invalid vendor VFIO vGPU profile type %q", profileType) } + // Placement filters quarantined VFs from a snapshot taken outside this + // lock. Re-checking here, under the lock quarantine mutations take, + // closes the window where a VF is quarantined between selection and + // configuration. + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return err + } + if _, bad := quarantined[vfAddress]; bad { + return fmt.Errorf("vendor VFIO vGPU on VF %s is quarantined", vfAddress) + } currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") currentType, err := readCurrentVGPUType(currentTypePath) if err != nil { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index dc88951de..3c7438532 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -49,7 +49,7 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { vfs, err := sysfs.discoverVFs() require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs) + profiles, err := sysfs.listProfiles(vfs, nil) require.NoError(t, err) assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) } @@ -178,3 +178,47 @@ func assertFileValue(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(value)) } + +func TestVendorVFIOConfigureRefusesQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "0", testCreatableTypes) + + err := sysfs.configure(context.Background(), vfAddress, "1148") + require.ErrorContains(t, err, "is quarantined") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOConfigureFailsClosedWhenVFHealthUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "0", testCreatableTypes) + + err := sysfs.configure(context.Background(), vfAddress, "1148") + require.ErrorContains(t, err, "VF health state unavailable") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs, availability.Quarantined) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..8de8e23a0 --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,512 @@ +package devices + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +// +// InstanceID and AssignedAt together identify the assignment. AssignedAt is +// the instance's stored GPUClaimedAt rendered with FormatVFAssignedAt; a +// success report only clears a failure whose AssignedAt string matches +// exactly, so every reporter must use that formatting. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +// AssignedAt follows the same format as VFInitFailureReport.AssignedAt. +// +// A success clears the matched failure and every older tally. It rescinds a +// quarantine only when the matched failure is the most recent one recorded: +// the report that crossed the threshold, or the newest tally when a lowered +// threshold quarantined the VF at load. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// FormatVFAssignedAt renders a claim time as the AssignedAt key used in VF +// health reports. +func FormatVFAssignedAt(claimedAt time.Time) string { + return claimedAt.UTC().Format(time.RFC3339Nano) +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error + syncDirFunc func(string) error +} + +// vfHealthAddressPattern is stricter than ValidatePCIAddress on purpose: +// addresses are map keys compared against sysfs entry names, which are +// lowercase with a 0-7 function digit, and this file also builds on macOS +// where ValidatePCIAddress always returns false. +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{ + records: make(map[string]vfHealthRecord), + threshold: defaultVFQuarantineThreshold, + syncDirFunc: syncDir, + } + // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine + // mutations with vendor-VFIO configure and destroy so a VF cannot be + // configured for a new claim while it is being quarantined. + vendorVFIOMu sync.Mutex +) + +// InitVFHealth loads persisted VF health state from path and evaluates the +// loaded tallies against threshold, the number of failed assignments that +// quarantine a VF. A lowered threshold therefore applies to failures +// persisted before the change. An error leaves the store unavailable, which +// fails vGPU placement closed until a later load or write succeeds. An empty +// path keeps state in memory only. +func InitVFHealth(path string, threshold int) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + vfHealth.threshold = threshold + return vfHealth.loadLocked() +} + +// requarantineLocked quarantines records whose failure tallies meet the +// current threshold, so threshold changes and loaded state agree. +// +// Unlike reports, a failed persist here does not roll memory back: the +// tallies already meet the threshold, so dropping the quarantine would +// readmit a VF the store has judged unhealthy. The latched error closes +// placement until a later read or report re-persists the in-memory state. +func (s *vfHealthStore) requarantineLocked() error { + changed := false + for address, record := range s.records { + if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { + continue + } + now := time.Now().UTC() + record.QuarantinedAt = &now + s.records[address] = record + changed = true + } + if !changed { + return nil + } + _, err := s.persistLocked() + return err +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + s.persistErr = nil + if s.path == "" { + return nil + } + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for instance %q assigned at %q", i, failure.InstanceID, failure.AssignedAt) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + return s.requarantineLocked() +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + // A failed write closes placement, which stops the guest reports that + // would otherwise retry it. Retrying here lets the store recover on the + // next placement or /resources read once the disk is writable again. + if err := s.retryPersistLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", err) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// QuarantinedVFAddresses returns the PCI addresses of quarantined VFs. It +// fails while the health store is unavailable so placement fails closed. +func QuarantinedVFAddresses() (map[string]struct{}, error) { + return vfHealth.checkedAddresses() +} + +// VGPUAvailability is one VF health snapshot applied to discovered VFs. +// Pass Quarantined to ListGPUProfilesWithVFs so profile availability is +// computed from the same snapshot without reading the store again. +type VGPUAvailability struct { + AllocatableSlots int // free VFs eligible for placement + QuarantinedSlots int + Quarantined map[string]struct{} // PCI addresses of quarantined VFs +} + +// GetVGPUAvailability counts free allocatable and quarantined VFs. It fails +// while the health store is unavailable so callers fail closed. +func GetVGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (VGPUAvailability, error) { + if framework != VGPUFrameworkVendorVFIO { + return VGPUAvailability{AllocatableSlots: countFreeVFs(vfs, nil)}, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return VGPUAvailability{}, err + } + availability := VGPUAvailability{ + AllocatableSlots: countFreeVFs(vfs, addresses), + Quarantined: addresses, + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + availability.QuarantinedSlots++ + } + } + return availability, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through an exactly matched successful +// assignment. A quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + if err := s.retryPersistLocked(); err != nil { + return VFReportResult{}, err + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + if err := s.retryPersistLocked(); err != nil { + return VFSuccessResult{}, err + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if match < 0 { + return VFSuccessResult{}, nil + } + + // Only the newest failure can rescind a quarantine; see VFInitSuccessReport. + remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil && len(remaining) == 0, + } + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + s.records[report.VFAddress] = previous + } + return VFSuccessResult{}, err + } + return result, nil +} + +func (s *vfHealthStore) retryPersistLocked() error { + if s.persistErr == nil { + return nil + } + _, err := s.persistLocked() + return err +} + +// persistLocked writes the current records to disk. A failure is latched and +// fails placement closed until a retry succeeds. The returned boolean +// reports whether the rename made the new state visible. +func (s *vfHealthStore) persistLocked() (bool, error) { + if s.path == "" { + return false, nil + } + renamed, err := s.writeStateLocked() + s.persistErr = err + return renamed, err +} + +func (s *vfHealthStore) writeStateLocked() (bool, error) { + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return false, fmt.Errorf("marshal VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + if err := os.MkdirAll(dirPath, 0755); err != nil { + return false, fmt.Errorf("create VF health state dir: %w", err) + } + // The first write creates the state directory; syncing its parent makes + // that creation durable. Doing it on every write keeps the path + // stateless and cheap relative to how rarely the store is written. + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return false, fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return false, fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return false, fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return false, fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return false, fmt.Errorf("rename VF health state: %w", err) + } + if err := s.syncDirFunc(dirPath); err != nil { + return true, fmt.Errorf("sync VF health state dir: %w", err) + } + return true, nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..7c2e93d15 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,669 @@ +package devices + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + vfHealth.persistErr = nil + vfHealth.syncDirFunc = syncDir + }) + return path +} + +// setVFHealthThreshold changes the quarantine threshold on the loaded store +// and re-evaluates recorded tallies, as a restart with a new +// gpu.vf_quarantine_threshold would. +func setVFHealthThreshold(n int) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n + return vfHealth.requarantineLocked() +} + +func vfHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil || vfHealth.persistErr != nil +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + require.NoError(t, setVFHealthThreshold(1)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + require.NoError(t, setVFHealthThreshold(defaultVFQuarantineThreshold)) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.6", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, availability.AllocatableSlots, "a below-threshold failure tally must not remove the VF from placement") + assert.Equal(t, 1, availability.QuarantinedSlots) + + availability, err = GetVGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, availability.AllocatableSlots) + assert.Zero(t, availability.QuarantinedSlots) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + + _, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + availability, err := GetVGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, availability.AllocatableSlots) + assert.Zero(t, availability.QuarantinedSlots) + + restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) + availability, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err, "a repaired state file must re-enable placement without a new report") + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) +} + +func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { + resetVFHealthStore(t) + require.NoError(t, setVFHealthThreshold(1)) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0o644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + assert.True(t, vfHealthStoreUnavailable()) + _, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.path = goodPath + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.False(t, vfHealthStoreUnavailable()) + + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) +} + +func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, setVFHealthThreshold(1)) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "instance-2"}) + require.Error(t, err) + assert.True(t, vfHealthStoreUnavailable()) + _, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, nil) + require.ErrorContains(t, err, "last write failed") + + // No report arrives while placement is closed; a read alone must clear the latch. + vfHealth.syncDirFunc = syncDir + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}, {PCIAddress: "0000:e3:00.5"}}) + require.NoError(t, err) + assert.False(t, vfHealthStoreUnavailable()) + assert.Equal(t, 1, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1, "the retried write must persist the rolled-back in-memory state") + assert.Equal(t, "0000:e3:00.4", state.Records[0].VFAddress) +} + +func TestSetThresholdReevaluatesRecordedFailures(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, setVFHealthThreshold(3)) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + require.NoError(t, setVFHealthThreshold(2)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt, "the re-evaluated quarantine must be persisted") +} + +func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, setVFHealthThreshold(3)) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + // Simulate a restart with a lower configured threshold. + require.NoError(t, InitVFHealth(path, 2)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: FormatVFAssignedAt(time.Date(2026, 8, 20, 15, 0, 0, 0, time.UTC)), + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessForOlderAssignmentKeepsQuarantine(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + older := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + } + _, err := ReportVFInitFailure(older) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: older.VFAddress, + InstanceID: older.InstanceID, + AssignedAt: older.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 1, success.Cleared) + assert.False(t, success.Rescinded) + + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + quarantined := quarantinedVFs() + require.Len(t, quarantined, 1) + require.Len(t, quarantined[0].Failures, 1) + assert.Equal(t, trigger.InstanceID, quarantined[0].Failures[0].InstanceID) + + success, err = ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 1, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { + resetVFHealthStore(t) + parentDir := t.TempDir() + vfHealth.path = filepath.Join(parentDir, "gpu", "vf-health.json") + + parentSyncs := 0 + retrySawPersistErr := false + vfHealth.syncDirFunc = func(path string) error { + if path != parentDir { + return syncDir(path) + } + parentSyncs++ + if parentSyncs == 1 { + return errors.New("injected parent sync failure") + } + if parentSyncs == 2 { + retrySawPersistErr = vfHealth.persistErr != nil + } + return syncDir(path) + } + + report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} + _, err := ReportVFInitFailure(report) + require.ErrorContains(t, err, "sync VF health state parent dir") + assert.True(t, vfHealthStoreUnavailable()) + + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 3, parentSyncs) + assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") + assert.False(t, vfHealthStoreUnavailable()) +} + +func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(path string) error { + if path == filepath.Dir(vfHealth.path) { + return errors.New("injected sync failure") + } + return syncDir(path) + } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) + require.ErrorContains(t, err, "sync VF health state dir") + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) + require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") + assert.True(t, vfHealthStoreUnavailable()) + + vfHealth.syncDirFunc = syncDir + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.False(t, vfHealthStoreUnavailable()) + + data, err = os.ReadFile(path) + require.NoError(t, err) + state = vfHealthFile{} + require.NoError(t, json.Unmarshal(data, &state)) + found := false + for _, record := range state.Records { + if record.VFAddress == vf { + found = true + assert.NotNil(t, record.QuarantinedAt, "a later write must not erase the renamed quarantine") + } + } + require.True(t, found) + + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + require.NoError(t, err) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) +} + +func TestReportRetriesFailedThresholdPersistence(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + require.NoError(t, setVFHealthThreshold(3)) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) + require.NoError(t, err) + } + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + require.Error(t, setVFHealthThreshold(2)) + assert.True(t, vfHealthStoreUnavailable()) + + vfHealth.path = path + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.False(t, vfHealthStoreUnavailable()) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: `duplicate failure for instance "instance-1" assigned at "a"`, + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, InitVFHealth(path, defaultVFQuarantineThreshold), tt.wantErr) + assert.True(t, vfHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + +func TestInitVFHealthFailsWhenReevaluatedQuarantineCannotPersist(t *testing.T) { + path := resetVFHealthStore(t) + // Two persisted tallies meet the default threshold, so loading them + // quarantines the VF and must write that back. + state := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[` + + `{"instance_id":"instance-1","reported_at":"2026-08-20T00:00:00Z"},` + + `{"instance_id":"instance-2","reported_at":"2026-08-20T01:00:00Z"}]}]}` + require.NoError(t, os.WriteFile(path, []byte(state), 0o644)) + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + + err := InitVFHealth(path, defaultVFQuarantineThreshold) + require.ErrorContains(t, err, "injected sync failure") + require.Len(t, quarantinedVFs(), 1, "the re-evaluated quarantine must stay in effect in memory") + assert.True(t, vfHealthStoreUnavailable()) + _, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.syncDirFunc = syncDir + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err, "a read must retry the failed write once the disk recovers") + assert.False(t, vfHealthStoreUnavailable()) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index f7ab42ec5..d53659efd 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -38,16 +38,21 @@ func ListGPUProfiles() ([]GPUProfile, error) { if err != nil { return nil, err } - return ListGPUProfilesWithVFs(framework, vfs) + availability, err := GetVGPUAvailability(framework, vfs) + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs, availability.Quarantined) } // ListGPUProfilesWithVFs returns available profiles for discovered VFs. -func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { +// Quarantined VFs are excluded from vendor VFIO counts. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { switch framework { case VGPUFrameworkMdev: return listMdevGPUProfilesWithVFs(vfs) case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.listProfiles(vfs) + return hostVendorVFIO.listProfiles(vfs, quarantined) default: return nil, nil } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index ec0c7c9d2..4c0046c7e 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -638,6 +638,10 @@ func cloneStoredMetadata(src StoredMetadata) StoredMetadata { guestAgentReadyAt := *src.GuestAgentReadyAt dst.GuestAgentReadyAt = &guestAgentReadyAt } + if src.GPUClaimedAt != nil { + gpuClaimedAt := *src.GPUClaimedAt + dst.GPUClaimedAt = &gpuClaimedAt + } if src.ExitCode != nil { exitCode := *src.ExitCode dst.ExitCode = &exitCode diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..98479175a 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -711,6 +711,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { t.Parallel() startedAt := time.Now().Add(-2 * time.Minute) stoppedAt := time.Now().Add(-1 * time.Minute) + gpuClaimedAt := time.Now().Add(-3 * time.Minute) expiresAt := time.Now().Add(time.Hour) notBefore := time.Now().Add(5 * time.Minute) pid := 1234 @@ -731,6 +732,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { ExpiresAt: &expiresAt, StartedAt: &startedAt, StoppedAt: &stoppedAt, + GPUClaimedAt: &gpuClaimedAt, HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, ExitCode: &exitCode, AutoStandby: &autostandby.Policy{ @@ -786,6 +788,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { *cloned.ExpiresAt = now *cloned.StartedAt = now *cloned.StoppedAt = now + *cloned.GPUClaimedAt = now require.Equal(t, "1", src.Env["A"]) require.Equal(t, "x", src.Tags["m"]) @@ -806,6 +809,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { require.Equal(t, expiresAt, *src.ExpiresAt) require.Equal(t, startedAt, *src.StartedAt) require.Equal(t, stoppedAt, *src.StoppedAt) + require.Equal(t, gpuClaimedAt, *src.GPUClaimedAt) } func TestCloneStoredMetadataWithoutPendingStandbyCompression_ClearsPendingPlan(t *testing.T) { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 2faf2c382..de5cbf8ed 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -187,6 +187,8 @@ type manager struct { createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) configureVGPU func(context.Context, string, string) error vendorVFIOProfiles func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) + quarantinedVFs func() (map[string]struct{}, error) + pickVFIndex func(n int) int destroyVGPU func(context.Context, devices.VGPUAssignment) error reconcileVGPUDevices func(context.Context, map[string]struct{}) error vgpuAllocationMu sync.Mutex diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index f68f6f6dd..1058af131 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -311,6 +311,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUClaimedAt = sourceMeta.GPUClaimedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..efa057b31 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -154,7 +154,8 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUClaimedAt *time.Time // when the vendor VFIO claim was persisted; identifies this assignment in VF health reports // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index f036161f9..0296ef34d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -3,6 +3,7 @@ package instances import ( "context" "fmt" + "math/rand/v2" "path/filepath" "sort" @@ -56,7 +57,15 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } - vfAddress, profileType, err := selectVendorVFIOVF(vfs, profilesByVF, allMetadata, profileName) + // Quarantine is read under the allocation lock but mutated under the + // devices lock, so this is a snapshot. configure re-checks it under the + // devices lock before the VF is touched. + quarantined, err := m.quarantinedVFAddresses() + if err != nil { + return nil, err + } + candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, claims: allMetadata, quarantined: quarantined} + vfAddress, profileType, err := selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) if err != nil { // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the @@ -67,10 +76,11 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if _, vfs, err = m.discoverVGPUDevices(); err != nil { return nil, err } - if profilesByVF, err = listProfiles(vfs); err != nil { + if candidates.profilesByVF, err = listProfiles(vfs); err != nil { return nil, err } - if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, profileName); err != nil { + candidates.vfs = vfs + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, err } } @@ -82,7 +92,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // clean siblings. log := logger.FromContext(ctx) for { - vf, ok := vfByAddress(vfs, vfAddress) + vf, ok := vfByAddress(candidates.vfs, vfAddress) if !ok || !vf.Allocated { break } @@ -96,8 +106,8 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str break } log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) - vfs = withoutVF(vfs, vf.PCIAddress) - if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, profileName); err != nil { + candidates.vfs = withoutVF(candidates.vfs, vf.PCIAddress) + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) } } @@ -109,6 +119,11 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str SysfsPath: filepath.Clean(devices.GetDeviceSysfsPath(vfAddress)), } setStoredVGPUDevice(&meta.StoredMetadata, device) + // Only vendor VFIO claims carry an assignment identity, so the claim time + // is set here rather than in setStoredVGPUDevice. clearStoredVGPUDevice + // resets it with the rest of the device fields. + claimedAt := m.nowUTC() + meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { clearStoredVGPUDevice(&meta.StoredMetadata) return nil, fmt.Errorf("save vGPU claim: %w", err) @@ -150,10 +165,31 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt return reset } -func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][]devices.VGPUProfileType, allMetadata []StoredMetadata, profileName string) (string, string, error) { +func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { + quarantined := m.quarantinedVFs + if quarantined == nil { + quarantined = devices.QuarantinedVFAddresses + } + return quarantined() +} + +// vendorVFIOCandidates is the host state a vendor VFIO placement chooses from. +type vendorVFIOCandidates struct { + vfs []devices.VirtualFunction + profilesByVF map[string][]devices.VGPUProfileType + claims []StoredMetadata // every instance; those with a device path hold a VF + quarantined map[string]struct{} +} + +// selectVendorVFIOVF picks the VF to claim for profileName. Quarantined VFs +// are never candidates and count against their parent GPU, so placement +// drifts away from cards carrying a wedged VF. Among equally ranked +// candidates on the chosen GPU, pick selects the index (nil is uniform +// random), so a single VF cannot capture every placement on an idle host. +func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n int) int) (string, string, error) { profilesByName := make(map[string]devices.VGPUProfileType) - advertises := make(map[string]map[string]struct{}, len(profilesByVF)) - for vfAddress, profiles := range profilesByVF { + advertises := make(map[string]map[string]struct{}, len(c.profilesByVF)) + for vfAddress, profiles := range c.profilesByVF { advertises[vfAddress] = make(map[string]struct{}, len(profiles)) for _, profile := range profiles { profilesByName[profile.Name] = profile @@ -162,21 +198,21 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ } requested, found := profilesByName[profileName] if !found { - if len(profilesByName) == 0 && len(vfs) > 0 { + if len(profilesByName) == 0 && len(c.vfs) > 0 { return "", "", fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) } return "", "", fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) } - vfsByAddress := make(map[string]devices.VirtualFunction, len(vfs)) - for _, vf := range vfs { + vfsByAddress := make(map[string]devices.VirtualFunction, len(c.vfs)) + for _, vf := range c.vfs { vfsByAddress[vf.PCIAddress] = vf } claimed := make(map[string]struct{}) usageByGPU := make(map[string]int) unknownUsageByGPU := make(map[string]bool) - for i := range allMetadata { - stored := &allMetadata[i] + for i := range c.claims { + stored := &c.claims[i] if stored.GPUDevicePath == "" { continue } @@ -195,13 +231,18 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ } parentAdvertises := make(map[string]bool) - for _, vf := range vfs { + for _, vf := range c.vfs { if _, ok := advertises[vf.PCIAddress][requested.TypeName]; ok { parentAdvertises[vf.ParentGPU] = true } } + quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]devices.VirtualFunction) - for _, vf := range vfs { + for _, vf := range c.vfs { + if _, bad := c.quarantined[vf.PCIAddress]; bad { + quarantinedByGPU[vf.ParentGPU]++ + continue + } if _, ok := claimed[vf.PCIAddress]; ok { continue } @@ -227,6 +268,9 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { + return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] + } if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { return !unknownUsageByGPU[gpus[i]] } @@ -238,7 +282,17 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ if len(gpus) == 0 { return "", "", fmt.Errorf("no available VF for profile %q", profileName) } - return freeByGPU[gpus[0]][0].PCIAddress, requested.TypeName, nil + // Candidates are sorted clean-first, so the leading run with the same + // Allocated state is the set of equally ranked VFs. + candidates := freeByGPU[gpus[0]] + n := 1 + for n < len(candidates) && candidates[n].Allocated == candidates[0].Allocated { + n++ + } + if pick == nil { + pick = rand.IntN + } + return candidates[pick(n)].PCIAddress, requested.TypeName, nil } func (m *manager) configureClaimedVGPU(ctx context.Context, device *devices.VGPUDevice) error { @@ -270,6 +324,7 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUClaimedAt = nil } // vgpuCleanupGuard checks whether the VF can be safely released: the VMM diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index dcfb08bd2..c93e4b2f5 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -7,6 +7,7 @@ import ( "errors" "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" @@ -72,6 +73,59 @@ func TestConcurrentVGPUClaimsUseDistinctVFs(t *testing.T) { assert.Len(t, claims, 2) } +func TestVGPUClaimSkipsQuarantinedVF(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + m.quarantinedVFs = func() (map[string]struct{}, error) { + return map[string]struct{}{"0000:82:00.4": {}}, nil + } + m.pickVFIndex = pickFirst + meta := saveTestVGPUInstance(t, m, "new") + + device, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVGPUClaimFailsClosedWhenVFHealthUnavailable(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + m.quarantinedVFs = func() (map[string]struct{}, error) { + return nil, errors.New("VF health state unavailable: read failed") + } + meta := saveTestVGPUInstance(t, m, "new") + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.ErrorContains(t, err, "VF health state unavailable") + stored, loadErr := m.loadMetadata("new") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath, "placement must not claim while quarantine state is unknown") +} + +func TestVGPUClaimRecordsClaimTime(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + claimedAt := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + m.now = func() time.Time { return claimedAt } + m.destroyVGPU = func(context.Context, devices.VGPUAssignment) error { return nil } + meta := saveTestVGPUInstance(t, m, "new") + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.NoError(t, err) + stored, err := m.loadMetadata("new") + require.NoError(t, err) + require.NotNil(t, stored.GPUClaimedAt) + assert.True(t, claimedAt.Equal(*stored.GPUClaimedAt)) + + require.NoError(t, m.releaseStoredVGPUPersisted(context.Background(), stored)) + stored, err = m.loadMetadata("new") + require.NoError(t, err) + assert.Nil(t, stored.GPUClaimedAt, "release must clear the assignment identity with the claim") +} + func TestVGPUClaimUsesLeastLoadedGPU(t *testing.T) { vfs := []devices.VirtualFunction{ {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, @@ -244,6 +298,8 @@ func TestVGPUClaimFallsBackWhenDirtyVFRefusesReset(t *testing.T) { {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, } m := newVGPUAllocationManager(t, vfs) + // Both VFs are equally ranked; pin the tiebreak so the fallback order is fixed. + m.pickVFIndex = pickFirst var resets []string m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { resets = append(resets, assignment.DevicePath) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index de697b8b8..be346dee0 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -120,12 +120,97 @@ func TestSelectVendorVFIOVFFailsClosedOnClaimedVF(t *testing.T) { profiles := map[string][]devices.VGPUProfileType{ "0000:82:00.4": {{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048}}, } - _, _, err := selectVendorVFIOVF(vfs, profiles, []StoredMetadata{{ - GPUDevicePath: testVFDevicePath, - }}, testVGPUProfile) + _, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: profiles, + claims: []StoredMetadata{{GPUDevicePath: testVFDevicePath}}, + }, testVGPUProfile, nil) require.ErrorContains(t, err, "no available VF") } +func testVFProfiles(addresses ...string) map[string][]devices.VGPUProfileType { + profiles := make(map[string][]devices.VGPUProfileType, len(addresses)) + for _, address := range addresses { + profiles[address] = []devices.VGPUProfileType{{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048}} + } + return profiles +} + +func pickFirst(int) int { return 0 } + +func pickLast(n int) int { return n - 1 } + +func TestSelectVendorVFIOVFSkipsQuarantinedVF(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + quarantined := map[string]struct{}{"0000:82:00.4": {}} + + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.4", "0000:82:00.5"), quarantined: quarantined}, testVGPUProfile, pickFirst) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", vf) + + _, _, err = selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs[:1], profilesByVF: testVFProfiles("0000:82:00.4"), quarantined: quarantined}, testVGPUProfile, pickFirst) + require.ErrorContains(t, err, "no available VF") +} + +func TestSelectVendorVFIOVFAvoidsGPUWithQuarantinedVF(t *testing.T) { + // Both GPUs are idle; GPU 82 sorts first by name but carries a + // quarantined VF, so the clean card wins. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + quarantined := map[string]struct{}{"0000:82:00.4": {}} + + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.4", "0000:82:00.5", "0000:e3:00.4"), quarantined: quarantined}, testVGPUProfile, pickFirst) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", vf) +} + +func TestSelectVendorVFIOVFPicksAmongEquivalentFreeVFs(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + var offered int + pick := func(n int) int { + offered = n + return n - 1 + } + + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.4", "0000:82:00.5")}, testVGPUProfile, pick) + require.NoError(t, err) + assert.Equal(t, 2, offered) + assert.Equal(t, "0000:82:00.5", vf) +} + +func TestSelectVendorVFIOVFRandomizesOnlyAmongCleanVFs(t *testing.T) { + // The dirty VF is still a candidate of last resort but must never be + // offered to the tiebreak while a clean sibling exists. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + } + var offered int + pick := func(n int) int { + offered = n + return n - 1 + } + + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.5", "0000:82:00.6")}, testVGPUProfile, pick) + require.NoError(t, err) + assert.Equal(t, 2, offered) + assert.Equal(t, "0000:82:00.6", vf) + + vf, _, err = selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs[:1], profilesByVF: testVFProfiles("0000:82:00.4")}, testVGPUProfile, pickLast) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", vf) +} + func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { // GPU 82 carries a claim whose profile is no longer creatable anywhere, // so its load is unknown; GPU e3 has a known 2 GB claim. Known load wins @@ -146,13 +231,13 @@ func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, } - vf, profileType, err := selectVendorVFIOVF(vfs, profiles, claims, testVGPUProfile) + vf, profileType, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: profiles, claims: claims}, testVGPUProfile, nil) require.NoError(t, err) assert.Equal(t, "0000:e3:00.5", vf) assert.Equal(t, testVFProfileType, profileType) // With no alternative, the GPU with unknown load is still used. - vf, _, err = selectVendorVFIOVF(vfs[:2], profiles, claims[:1], testVGPUProfile) + vf, _, err = selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs[:2], profilesByVF: profiles, claims: claims[:1]}, testVGPUProfile, nil) require.NoError(t, err) assert.Equal(t, "0000:82:00.5", vf) } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 7242d5503..b1d2067b8 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1019,19 +1019,28 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` // Mode GPU mode (vgpu for SR-IOV/mdev, passthrough for whole GPU) Mode GPUResourceStatusMode `json:"mode"` + // PlacementDisabledReason Present when allocatable_slots is 0 because the VF health state could not be read or written rather than because the host is full. vGPU placement is refused until the state file is repaired or the next write succeeds. + PlacementDisabledReason *string `json:"placement_disabled_reason,omitempty"` + // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -19370,252 +19379,257 @@ var swaggerSpec = []string{ "bf/VRM1r8gOJjTITPZTyuFbjyb7f7iJ6dfrutGlMRdUl5I9uaU4r4H2Wo10mOTN1nzybCrb1GC3imsHj", "MZ310Y9Eqh6ZTLhQ+zaQBRxSXs1XvECCxLmmBx+ajDMk6TiBM1r0qgV+/YueEACBjPPJhIgq6sCLkCzt", "vT1KA8a9n/RzZF4oYA9OfqzK01pub6u1n1aoAdT2CY4om2623u6AQbA2jXW4gq9O3721ZYOaEJf1Uhal", - "hQzYch/9WhTW0kstS1SlfsBSWEfua0gkO50tJI1wYlo0VTEo8w18cBpaS+in5YfWFBqQ08P12NzJQxvz", - "aZbDuT972zt+834rjcm8WxkTBPLNeEL0uDc99jR3aAZl1luFK82bLC2GMGTbE+utVcEyWi+SxyACq6O4", - "wslIJjwUM3SuHyJ4iDbe/2TSkfUIuiirbKX+3Yca8un7afDEAIJ9Q7dn0GHdZFs54EHdtQ6B1qlOr9Jp", - "6KiYVKplGWu5GCK/rG40v1xfgM800tzvocv2qhnVrSCLTFKYwWt0YRDIfGqs5tY0I0mGBVYkWdTiMauA", - "6mTZo02uSXSDdLOX+vVPpohGLshIzQSRM55U4yB2u8uFWCXEHM+JrT1l5uQZ/hVHKRaXcBM7QR7lzKxA", - "NWR9dx2+zEyp7AaT+vn8/NRo94qIOU7qSQ9yycN/RBK8QGOirghhbipYIuzHvNaTRmVD/R+hRhkRlFfX", - "sLMb6PfMxEGjqcARQeYrV8bZbomE+KK2S2l7CSAWRhGRsmF/t1ftr/10kift9jg0rO21Vc+jm2zw+eGp", - "q2NT1Al2y7yzvMqnRPTMkXMFg1dv7Y5cXVHJdcUMEO2SwDAmUGLJZpr4iaAuvglKSOnPK8mXHnOQfrqg", - "7QdOgVmqrjnnH1pJl/XjHnLkp5jFoXrLJsHApOZPAa8LopxFDqIfjU3wk0ELMHqAnychCI4pI1LW8pqj", - "XCSdbqc3sbPa39pKeIQTgCzc291+vrU6jHRl/LANlxrFdJV+6YKqTNiNy6I1eG8w6SpJbOEsa2GBM+u4", - "5n4A9rQcrwg1lvXd5kl4LoB9MFhK0b7GkXJV4cAkV3G5Yv/YArBwZT7QYGoKvWhR+4V/PgdBH3iG1axK", - "/ltLtA9xMRDTqGnEmDtq62iI/GNQouIiVPqLC2Uz/sbEBT0W96ELKXQQuhXn3OC5P8unT57sPlnHh4DZ", - "1I65PXeBqZq3qznNxFtue35tAxCeVZU53JFeXb1Ur8samtIccYmkVi8ozwi70Xo+2dvdudl6tp3IsQsL", - "q/GlECTM4cmRkYkizhSmjAiUEoVjrHCVyYAtS3MZqC2DSQppQZPvV7OWhvgJH+PltoWxvpT3vaFG3lsH", - "CZ1iRieaIds3/Z7lDO88ebpvKnnGZLL35Gm/378p8sXLEuqi1VZsmSA9DwSjL2eftw93AHDRZi5/dE4P", - "zn/WjCyXwlxaW3JM2b737+Kf5QP4w/xzTFkYGKNN8Vc6WSr6Wo1Hyy3yMIn3UVnf28k9beKDGozREJ0M", - "aDxBmLlKlObd4ckVNE6rZQNukBu8IldWiytvWLJoXJtb13EtK5wrr36rn23WopYr/bjav+7MXfCO7dMA", - "UBdlbpc967cqVCxX1nJcKvWVEVZUb0wS81fEGQD7hko5Vq5I96xFJTC4RmzJr6JL/8eid+/HQ38g3u+u", - "kpj3k63p+OGGITErBdK/Lcuh67mQE0fXHOaw7bG4FdrWz7U4dMFY8Ae+C28TNlbt/c30v37/P/L02d+3", - "f3/9/v1/z1/919Gv9L/fJ6dvPgvjZDUE4YPiCH4x6EBTE97HD2xLSidYRQEbnVb/GlbYPjEWBxXNoOIn", - "GpP9Ieuh11QRYerH1ZIfhx20QUBTgq+0uAvFcUze2ab++NR4NPXHfzgx+FO9jdgmpQu7IQXWiMzHMU8x", - "ZZtDNmS2LeQmIkEv0H/FKMKZKQJHGdL67wKNBVTssy6msvMu+gNn2afNIbNV+g2adoah6NmkyPpizkFs", - "R2XCYO3rpICdMBmJQ1bc1gUGn/Ez9kvAfkqSes5Qw6Ks1t+s5vR8EEIrhHwWvZFQlAZUkIKyNRkViTbo", - "+WBzWZ9bo2MUNLSC/Kwj3iQ8HuQhc3FTkuQRiWkEfMXlCc5sJmmRomkozRrxMsGvF7A3b03yWoxwrmaa", - "F0U2sT7i/JKSLmxpF9xhEPkBXxp//oxnvfGiN+NZAbKAhYl2wcYjXlWy/0/PTrT3ngg6sT0Fc+U1iQSE", - "TjgydmYm5bCwLixN7NzU82Ba9JkT+7opDCNNLRATyq1ywVy5CgJFKwHNo6A+EpLJv0dRQsHqJGc8T2I0", - "A8A9pZsJYeZ1BkXKDB5HMZnU/10Nadh58hQ0WPfv3Z3WGatm6VZRWZ4EdNrUsb4WHNuwSRiAEQ9GzhC+", - "JghJ34DWjwt2CsXhv2fINVSeuIKRGO+UyaGTtvpBIr3suc1gmpM9BhYyZITtaWpzHy2dwkrWVIsWTHQC", - "fJa0ACt5abI1z1+fIUVE6vLnNyK9O3BKDDJFj0qZ20JcB4cnLzf7nSDQUsVVBVu1MquqOugA1oKNVmgK", - "wihtNDglXXR8BNmy9lopdTFIb/iJC5SYW7G8jPYBrKNq7sGmhN/xkRVAk0UZ8mDElmFn07WY1a+3ffS2", - "UAFxMZQi77GkLddkeZlAszYAzuReLLVeS5MF/5hV/+x9DJkWUN/Q8GIAi2y8v9rbHB30lL6oahayG19I", - "fhRKo/3L2/svDav85WX03ZvJ6NYHPMpmWIaoe+Z7NeGlpX333chVdi+aY4Yq/Y4kDZ6tv7kqL941pIi+", - "5yqfh3BEn/S2t8+3925uvrspIm4VCsuDyStAcduj2d4FKmwA45WqUWNwOdKPbSi5s4u8P0EzLNl3Ch7W", - "rCPbu8/aGCWg17Zh2X5ANp+YIRVcyuFqFeHEBmHskiaJEWAknTKcoBdo4+z41S/Hr19voh568+akvhWr", - "vgjuzy3AceEWgHU0GWYBaKUKJAAqcgfPz1/D4UoIpF8YOfzy9pC5a02LLSB03eBenb4Dxz+WIxe42Zyr", - "iMt8X3JNpZLLqGqt4p8/B7LXfNquyL+bpGmjrPW/Gvf35wowbRAmb/MOAHtd8PrScj4Alu1DJgl+fTi6", - "K5FvPxe+1toZ7gi9tvFKCyG/1hARnjTdbrfHob2T4VQAZUJsy5dwXAb3rYFfux0ayF49kPriITE6Pi0r", - "PZXOCNd8bU4vdvrbT59DsdLtQRvGnuJoRd8nB4ftOx/smFtmH4/3o3gfFPbb+qwsYRsVBCdXeAHF/szS", - "DjvmwvS0W+/YWkWyVXzNMr7u7eB062JcA2AuiLMucEmO0pW1RlqkJ9ZB09LcQkKmNEmoJBFnsazKyDMs", - "kcwMEqqpuVFI8EMGA+yiovQxSCkIR5HIS9Ojla6tvJ9nlu6h7mfGmdYBAPj/F7KQKKXgBC26h9BHiYos", - "mHjINoTLmCpSo6DkZ6x/gPyDro1sj/XQqILaIvqDIZOzXGkmttlHh5zJPCXCWmXRmILHaBPJ3Ki0MF5Y", - "jYVmmJLGRAyZfi2AtfpHoZ7sPx0MBoNup9DkdvW/ByFqulPnZ99iCZucX0D7YxZVGGAERc5QzmIiivrf", - "xJBDPUTuho7TzwQRdp+3E6/s56VcFT6Y6zCH24EJfy6CKwy1QT+H6NBbKOdPbi+it8o5cvKrzTayX41u", - "EsFAUMTzJNYa31jfdsYgR2JrhpREGe5s3qUSvTO1N6tTt6HHiqPfcyIW6P3JSSXsQZCJ5gHtJg5comEf", - "eHajbdhZYyNZO5qbuJc9vNv7wLitSyqehPjFEW19D6NLgjYUWjFsVRTnVfY1rVUGs0goM/ukiWbFBEMp", - "MqMyMNJvVS7kxOI7WlEaO3nI4E0X8A+lU18u5NY4l1tZRLds/s0WYHM8B2yOvWDydEzmozwPqUb6kQNj", - "effu+AhtwC+ALQsplFUCxvjp9vPB8xe95+Ptp729eLDdw9u7T3s7T/Bgshs9293e2V2RCNMim+72CXJB", - "jTkQx1xErY9c9HwoqLkpd6Emm9h47CvKYn5Vuf6CAbJ+7zb4dl33y6H1rYcQTMhJsFTGfNHAyU7gkieR", - "btsEpNuMzaLUUtjQ+fR8sP251h8YXMMdcS5yZtyqBmOgcCGk3oD9zaqO83YsHwbkEl/WrZbfeftFG+w/", - "ebH/5HMXzSVvrBtjnZzucXObIsIcBnMtO8RlKHp2JGeg7FiZyFj1bTJJp9sp8l3gbxAGarHUxeNWSVxN", - "B7YbZiOrrpWG5Onjir4CkSoGgy/e15KK00egokORoq9FoMOE5zHybHEGkgz8cMee7qKbAbeYNdEZiFGT", - "jKF1HMC0hsoRlGlGDP5H3YjNtN5Hr+BdeIRTo9bZQZj6Jb7rDccLEy+jz5fr2ihZq4d8ZvUr+EYrW0j/", - "C6atl8GabFc3YaSzffQrh28KbY/xuu3XvA5q1vLrdTvxhoXrdsgZ0JkVNffRT4V4WQioViDdkMT+ObIM", - "qwSs2azABtgd72hqKXfOS4HvdsyKdrodt1CQKr+cNP+upPql8+eTYiiQjOAEznKZI5wrmliYbpgJlYpG", - "0iaP6M1tEntsaSUSj4zy1BSTahJPrYJVfOSkqvcnaAOQGP+CrGFb/2uziF+t3HU7L/ZePH228+JpK7yl", - "coDrReNDSIteHtxaOTnK8pG1jTRN/fD0nbF9RMaqUMS+vD/x4S0ywTXr0TN3Dfqdv+i/8GGmYp6PE8+x", - "aDHpDKotbFgQSa3gRQ1xkL/TZE4nE/b7x+hy5++CptvXT+XOeLsBPtd0FDa7HfvBBUs2ajLumTJJYSQg", - "ICghG8Gy3hIJM0BnRCGgnx7CEag3RTazJTkHqWVXPEhYe7u7u8+fPdlpRVd2dN7BGYERLnAp2xF4Rwze", - "RBtvz87Qlkdwpk2HKQEI58yqvuFzhmyN40FVIO1vD3ZDVNJwcZdUY9uep41L/t6qj3ZSdtEhKbtQLZdO", - "eXC1d3cHz/aePH/S7hhb8/BIXK/mMC5lySyPBeL3d34DpMnzg1MECcETHFVtOy5C7EajUjcaFRSRMODv", - "NxjY82dPn+zt7my3Q30LBZ1YPMPKga3yrsChCxBFYDcCS7HMertNt0VInDIE9pZECabpQeRSLGq3jwF5", - "HwnzWrkJbS4Gq4EvXVwtvm1l3CpMViZBx4gGXKCcFaVF+utdsl/Es9rMtc31sJ6rh9JymF49C09kSqjd", - "YikzQeaU5/ILNMSVyZmdJJyLG33bpLC8JTJPlLHZUInen3wHPEXTGpKKZFUdylLjChCnW07uRue5QiJh", - "Im9arFa70WbrV02423Bqu6sANSrcoBE6LdacK2frgz8PcRLlUEwHF/upZwUYYAAJkGXJwsT2JwnnDEUz", - "zMBJIjzEIzTjSdwPRsLqJ6NJMKqCX6GEG9DnS0IyW2fGDEJ/pkUYOidow6+wZkipVvf0SWqYjK0kUqXG", - "J2m4gCOWoWS1IhVerydW3MMjNp9ULKEJn0pQChVkLfTrMPgZFiYZATNTN2meGl0yEHAdGGKNmYduVHOT", - "8olVcK3IAYnmZiVxJLiUiCR0CjV63p/U8pdX5LwVWczrAzqrg21BusahGbjKDBpW6/JqofsxkM/zOTck", - "0DDkDK4IlXTGyRSzHCrPeIRsDfH91uGQMy7VqMCluuFgpRpBOYlckBItr8i6L+xB7p3gvehY222Wy8Yd", - "3+rrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyMtdKMLASXawOJXUTsLqy/gCV0Cr1YMvQBuS8eGzJg6DbbBMk", - "E1ZZdT9L2qqtDvp6b3DWFtZtNYrbKVazYzbhAeyPG3hOnSXaRqtmRKQUCqqgmDBKYqdLFi5Ua+qChPFE", - "EhTnxK6ckU8FtguOzfEGnxVzNjLKpjVeX++wjXnYjGF1tQno177YJtxJhhNqz0UOa2XiFSXCZWptqyBQ", - "Kkdhd9Zyw4JM8wQLZAEZ2wxZLtKEsss2rctFOuYJjZD+oO4Xn/Ak4Vcj/Uj+AHPZbDU7/cGoqTTRmRmc", - "zQs0G1Lrt5zCD3qWm7WsZLDEbJnvt8Ax2iZ6LBgp/hNNiEX3e8fotUfoVTj2vZ1BU7Z8Q6OVPPllZMib", - "cm5LssETn8tAbuFKKccVVSKxxcg3Yk+WS1PfpcWt5EBYnQvwdh6dauLI50GTHBp+XQMmQWMCeT9uastc", - "owVbbDOVYGmJXM7Q3/m4ahBtG/YbKFi2wUqIDEEmwfh+2NGVBmnzxtKaeLt7EwwKYKt6ovDRDaEd1pV2", - "K+OrmvjJ26UqZzNil4y6OZqKZy0qeLj4jwK+wPbaHsegXo8uEK8MKDVSLaCyK5TTWXhFFiUacyEAgVpL", - "OJy52QDsipZ59Fo73Ct0PiMLJEiKKRsyygojKYCpEcTInAgvS5YLrWRNSdxHf/NUPMDsTjO1sGDwYDz/", - "TiJ+xYoxDpk/SN14LnU7B8xYFkWeqUq5SN0saH2aUCBrGZxgSkD9RKpmaCKInPlzD9XM1DLeFRdxYzGi", - "BXKvQI0b8LEixS8J81lZ0UxQNTQNjcxXy1F8puAtPLX6J6rUoEX1GrOr+8slEWEhsZhS8Uqr0BXvqHjK", - "iQGBAUQUqC1o/zIsvkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6Jgxk+MGbboAk2Mmk8tWCfqidtbagKpMGt", - "QrNZ9iWgDRdC7Qq0VCUBr1BKq3uyXSZePVnAjWZLkqja+97zJ8+etqxU81nOOoPe9aVdc/N0hUuuYadO", - "2vh9nj95/uLF7t6TFzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6", - "oPcn//rHP92obj2gEKNZRghv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDTKZEDCDjsy69crB1GBA", - "2knBOMMRVYsAI8dXJtq9eKWGdN3GHVQdbEjkNW1bVFTNuWQ+LpNON1zn6D+Nb7hGC89bF7yS+bjJD/2m", - "3qvxQpdeCz/GoUWIgSxqrS8buIv5XGFZCejWf0eQd+EyzJazbcwbq1F366kQEMVi67p5oYAhtPaaPGk/", - "8re/tp2e37Ji1qmv+IcV57D5CN7I6hu4kQNG32h9am2NP9gL8HZfjcZ+KbqVtf4qdevKW/fm/bbIHl6u", - "k1DcYDfvz0uYvMmHdUxgoEc7BrvkZdvdCkk0UJOXCxMwoPGE9IpAPZsog2RuPIL6zFuY+UAGZ3TJJ5Mq", - "1u2TZmx0gP2BZC/XC1ZKayZdRK6dzaIOrG0wfoadJ3LY0SrAsLOdDjs1t1UwfTLF1yPbQRXbZbAKrLxM", - "f68NUroZjBMeXZoqa1C8u48GKCWYSZQzOPw1r9r2YLV3qNvJvL0poMGJCXFaYlswpjGZ4TmFihTWpzKt", - "BGKSa6okBIxCO/so5gbtqVJi1s5Qv2aSG/fLScOlg9nCNqwb1O9x5iJay3fBwDeBwrbsIxG8a8EKNMd+", - "8+akawIYIPTQDKwS3+gmakagGWTRRa28Qvl7OH54nJARjLsO158ur6Ofkw6eVUEkUdLid5fkUCMCFPGc", - "qTqOf9pOkaumlS1fSTmDYD8b/gG4bLZ3QyAoJhGcSLl8FquEfgviruUN2JUOJQ7shkgYDgX4ksK+4rfW", - "IVwfgDE2eNWhTTt+XLfxEo6k4racWHGqR+Q6IiSuA36GX2kbK2+/DMbKv8YWI6go3Gzfhnjn5dn17y7B", - "C8batNp+TD/jrAfoJG5LLZKIgQa0WDVVQqtAj3uQFqMQvGrohTYZ1+R69Vr/Sq4V4KPHeWJA78Kka1mV", - "vYzWrfitMxubDjQXZG15vjsoW2fizW9VuM6Gqj9E7Tr71p3Uq1vanTOi3Ltnlowad6ha6KXi0nIB/+6V", - "aoyNIaUushc82k43ayS4NwtbRSwob8scTYZTMsoEmdDrFcRjXjCKcRXWpDxIRQaDwRfdSPE12nuGohkW", - "sjZ2RqczlSyqATh7ASylzyrqKIgizBkK2+x8uZvuw+VoN7udfush4fjMgwZaKmliRdLRKtzsw9LbZq3z", - "GV6AFafRSfhsd28w2N0Z3Ao42w3rBst1WH5iSyBW22lKqfO+s47+SpSq30KRZL1cV/dKUMjVLpZJKkFw", - "ug+JNxmOCErIBEDyioTW9Z7FeterB28FKptFW9C/2yi7b84HXy2ZU3RlMcfdNDrOuVjFIPKfr3GINrCZ", - "aAlSL5Bzt9sbPD3f3t1/8nR/e/suwK6LRWrK9nj2cfvqWbKDJ3vJ88Wz37dnz6Y76W5QD7ukpjJQG1r9", - "Rb/bGGVTXpJVLKMKS0Mbdg4ZEfWCyfVC45IklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjirTtIXIbAq", - "F6dCWQ+Dv2Uns9J3UZ/N8dHqWdwqA6k+kDC91YcC5NVuMFChYrvzmcgMOWt5Db3zXmx9Ea3Milt3FYU8", - "7HDSg7vcsOIh8q4BM3izXnWBL19yAdvplAuqZunq26J4rYARh7jpj1LFVbynPjqeMqiW7v9chMn5SpT+", - "uNPtJB/3qmfG/t4e+csiEBcEaLfalwpahJFBMf7VqwCvlIqHMJHsWlfXY/5hu7f9AuIQko97Pwx6L6oR", - "B12zWv7ybbu3K78O2qyhXwLQlY7afnGjiGu3nqso6BcaKmBX3ssWm9jSeFmb2l0dLuG2ssHl46U9riH5", - "NAqgnyvp2ctt5AtNMUnwIoRN7xlqZU179IkMjcmUMtnGbrs7KAy3T9Jhp48OLEA46LKKF/34zUMNeo9O", - "aJqSmGoZ06j+zRkMOy1tcXVd4ma1SdxXAWmtHxbXXqyHSFiXcLXumux/Rj7uZ2m/7TTeVegdYFdzKipg", - "iMGLXUQnCLNagVLK5jihsU2kh8RIiFfbd0BtJclaHiBLOdDZSbpoyhUqU+hb2tty1mwXLMZPrsHeugIz", - "wxDEzhcBRCkAxOgq9nV8hDLB4zwq80cTGHSJ+CHyGkTbCiF/fUjuXdo3IDF7wgVab99oMmi0s0827XfN", - "NqkJtnmrtwfrt/pOjCLdTp7F63mYeakdB7sRcvuaFMSAiaa67DVJ0JvMhxYc/a2/gss6r7ElR1okyjPn", - "YNE0tUxJAXcLuBhCcb1HJCH6mlpuBPEkLrMkqCy56HqWuv30+azJxQkeqeWB/EJIpnUVwD+C/lLMFsGB", - "ubKjxV2yMXBo39I4vHqmXJFdrergnq2VxBq3yjfhNpVQMFy+ZvM2eCmXnvm7wPj2RbNlBBTH8CtC2tvm", - "EgD2Sxf21mg/vguz3EMKaW+s66EG2+pAhQt0dNd/GQusxboq8e6F3PMhsjjHU6iVt8aP0kgqobqS4OTA", - "SGFb3AoKbOp/IqgiKh1gixMQ4HOX0WuLDSKqEJ7ieuWIprKUxhO9PoaoudDLuTURNGHy1tNhffP7Qe9/", - "jLkdjfr7Wz/85f/uffjPoNm9ZkCQRPRiMoGIq0uy6JkqTApPq1foH6YEhNYqpvbMEJyCMQ3Q3i1X8sf7", - "ZFBwz8WvOF2aAoSqeSWUttdO6C//0Rzo5S3jO7gw1p7dz66QcheVZBV39/JGSsTUBdW7jLrN/pBBkt4l", - "WUjkFWazsp07sd/J4hMvFB9dGHLvEza/QGMKlS7lkGn1HkcRybRaZWv9UFOunQMbFgQnfju2QJw7L9Yz", - "awIrCHp/sgRn/Obd+Y9v3v16NHpz+vLXg+PRLy//G2Jdrnqmh7inaW/vyVNbpN1fye1goZCb17vooxOb", - "r2BjHiY5aPYAWCZRmqscomPIdZTkks6dp1Qlt69ssZy1fPtKEZ8JhaxUEgrPsJDdCZ0QCHCAe9VGF1Hp", - "iJFKqG5vrTyUoWXRxRDOsANXiuJOkgjWFdFbEV7tcmOri/501u4uMSCxgcMOqb/6sAWsqK+pBOAOFwTj", - "vYw2IIXGleB1GcSbNwOtPSgaDIZgfuFKS4MXX6Ia6ruV5U/nPOlpPa+hZETQrG7WIphCAE2Z1IxOk/dl", - "Og4oM9bGPaVTHHC4hBwrX6RqqRvQ2tSxpf1vLN8WTug4qtfTMMfSLFWt/kPNWiJVrznfI9XifQMCMUAs", - "myRe6gUZVjN2U6a2bHXhEHBIzAHVfVXadnnKHExkDz5an428UsH0ZuaNpHlvTpweVdP0VizQqV6aqxkR", - "xNsI+KCsU3DDJbMJSi3gaEx1xoyIMnjXZTdp8Rz87hJtFCYwtwRF2vWyX2B1HYoTfF30AD4lLJccsTCP", - "sg7W9qsfoWbAW1f7k05cEzCMmpYbRsivUtGqNXFUtbwZPlUtz9u8Hzx4llet4H5NZ6tGnGUfFdIM0ePf", - "MFU/cQF6cTP4y50D7cPlHxMBYHh1GP1WGPQ0JfGI52r1+devaenRXPlFfdiyvrCzAWAg4qiS19zECxw8", - "STmG5ZXWy0GiXFC1ONPrZaPaIR/UFfWFhYSO4OeyYyik+ukTWM8ngcyZV4QRQSMoU6vPY4oZaEzo/YlX", - "rdAUrlwCrgUR6M3hsbW7OOxj0KOpAtJzAagHp8edbmdOhLE9dAb93f4ADnNGGM5oZ7+z29/uDzqgVc1g", - "ilvjnCaxTSS3GnWhwR/HVhL60b2kvxQ4JQq++C0AiQABmPZ1UEHw1FMiM0yF1SKzBKAaDMFQ/TXUXXAX", - "6r65lbtm2VsbjyHfGtKASPbGbu4HEJTh7MA0dwYDi/Cu7PULSUwmc2Lr7zaMtuy3lVRnlyhQhmBJzXOy", - "ZbH0n7qdvcH2jca0aihwdkMdv2PYZjMT0M6f3HAhbtXpMTP5iTbb3MaF+ScOCMk/a7990Hsm8zTFYuEW", - "zF+tjMsmwZhIhN27Ro9TEkWaVUCxpD56w4h5jrBC2IRwi5xBjWn3oabQ6ikwbbtNLtCafuTx4ostYaUP", - "Z6P4VGVn+rh8WqLnL0c7BRkvb6R95KDGDdXeAwH9iIsC6Q92UvYGL+6+00POJgmNFOoVBGwDs6mE2KcE", - "gNMdCBMX6PecK4yKvIZHdKStzDouyK1bXkVbf9D4kzneCQn5A06JSDEzWSLmnTWHfuk4G99MeZxX3mqO", - "8I+POvamcmhE5qICQa56RP1rqy4MLl9HewEoCtunmV78gIS/dw8n3E62qJH7kEcOKpOiXJLHdJysr3Fc", - "CiFBWe4VUV8LzQ/u88qy1RT+hKfosRDwK1JIeOVuLV0KW5nImVGAgxLg2zJz0373XVX4Oy+feOFC4NfQ", - "TUNdD2UczDhe9JFbU6P0qwVgTQkC84yXr5VTPbyv5YTt3McJgxkXnqJv19S3a2rVKTfU4qYAB9M75S1s", - "EDeyQPz57A83tj58sz20tz20sjwwcmWtC3/n4z6yobkRjwmSM54nMRoTZICfXBCOwqI//YiwiGZ0TgDd", - "D6rV5YmiGRYQYpOiGCtsfOiNhomVZomiuS3dXM8FZJYLXAf0kGQEITCjJiDOMhSTMkZipD+xUTMlruJS", - "XXVz9oMG9qLB8mpEVzMuSQFsyJR3m0OetzTaMTTbH7Jzi3irFxCiyh2vkSQB3N4V9h/OEB4y+8H3joW4", - "iDiJ05JzYQHgidRAdJptWc7x0yMdyYiHQIfOCcNM9WRGIjqhkZ3WJVnYwNZgg60KUOkBu3G+PykyV9DO", - "Zhi4DkKXwijFR8UzZCmp6r9hEA0eJXlcOrkclhIWY5wkwQol04SPcTIy63NJAj7BV/CGXZTS4VJ6kxiP", - "iamlny3UjDPzdz7OmcrN32PBryQRw85mf8ggI8WuNYm7pYCIrqCiXZpxfc4ET02fW2aIW39cksWn/pAd", - "xClljiLgE5xIjsg1fAdxYwAeYrhXAz2Y0xT2gx/mUvHUh4B1dGeGyXOV5cqm1kiiuiH40yFTHP3hQC4/", - "bf1R9vgJnMUEx5pOvFfMlEC2bhq1HGE9+xG8GnC3E1iAYUdfpCbMYyowUwa/tEDpRFN/SzeKMhFQOra+", - "whFmKOOZKbEBRDXDmuQqbQBoBU4SpOAouW+14A472TAfi0GYjhsBCA1iXO0YUYZOfvQO02Dvefg8SRIJ", - "Eooo+a+zN78iuJX1HpjXynAtk9vCtMCA4hxcp46nvcTRDBlHFVRVHHZoPOwU7tx4E8aaSxsu0+uBT/EH", - "PbQfTDddGv/Q7+umjLtyH/32h2llX5+lLDWAqMPOpy7yHkypmuXj4tmH8II24bidVRgB2jDX3CZwEkwB", - "cse78c0ViVmMuL0FkgXCqORAfuDKmDIsFqsyKgNLb1eQT0wko7cYfwwhcnHY2R+62MVhpzvsEDaH32yA", - "47DzKbwC1mvZXMIP7rPCuVkQ0dPBYHM9JLhd34DPsoVj4AvrgI1aUVF/VO+gxaP9c/kH/q31z8L1g5nu", - "vMRoMoq/M74/QgeEJ7H7mmjABVETuzGLSOLE7vWGnvt3HujNikiS3DeBPhR5Fu6xomTBoyJH2KzyGK00", - "3z8wxQ3u61KpmO0fhn4fnf08YD23tnMyd6HO4YItAMZjVWlkXkZYojMYU+9MK98v4de+/a/T/QBc8iLh", - "04t9o7qjhE9RQpnNB/AClbV4YNcSPjJ4PMV3Fp7HVcvbMJLEv/7xTxgUZdN//eOfFuT+X//4Jxz3LYMz", - "B8W2L2YECzUmWF3so18IyXo4oXPiJgPlcMmciAXaHVibPzxCXs1/K6XJIRuyt0Tlgnl5E6ZwnbQNWleB", - "ng9lOZEWzwjShCa2qo6JbQzYbdxZNkt5rye6G8CFhBl4E9C3oqMBANWjpuK41UQ7YZOpmXPFaFoP01wK", - "1lvPXxS5VoZ6e2aAN2QwsMShcwcP7KTRxtnZy80+Am3LUAVUTgLdoWzGqhH9bzxpPU8yHKXKUGCVDW+K", - "cIbHNKHO5NhQ9sUcwRRHM8pIGV9cgK67JvbdSDWPOTg9RjYQsguvDtmbsy0wsSoSqVyQruUEwkKtlnXh", - "uM1zgR6Af1EF0WE9++6QTQiGPKHjI8MEPDTyIjGyaJgBognEuFJVKUHXHTIDqWshnPXBS3lMEvgI+p9i", - "Ra7woouKor+uTEyClVaIZVe/PGQm1dCuQQ8wW5A3zD7wMzOknovktTlbgkwSrRpDBL6pfw59b0y4QDbC", - "uVvmlbruTLapGZZetBRHb870/KagCXJjD4SW3py53djsIslRlFCghgizIZtCIJBDMeassqtFQtkMi7gX", - "cX0J+KhWl4xfJSSeNvHYQ5/I7lCSqfQTOE4/18n1sQkXs+UJ6ENskPpWe+6O7DvtXHe2xT+T785WxLyB", - "885YcInhN2Z1vznyWjjywuvmnHohz9qRg6K8u4hf08UDBfw62ltec/PEW7KHsOihDYfxA14RLtDp4THC", - "cSyIlJv/3vY+PVNDpaX8p+9HzYofIvTEjoULi35o7S1VAnks7OCtHTXCbl71QsP+/bZVqULUeNMVBYnK", - "K+/ub49apze5Rkqht6S1bzfJ2mBbKiMO9RZLaumBaJSQQnwpzqlPReusyiaMt7hyVopLlj0fH7kDeX/2", - "Zdt1zup3wz0wxaMaQ3xARlhNtfbLhz8man5X7KKD3V5hfv66SHNwf1LQfZuiQ2T+mNTFuLZsmgsaoJPG", - "C/QVUQbe5C71dNtDYOJnRLhTbQa6MLMupmU+RQanBSYElpjVuu+xeaWd6mva+zNpvrA8N5FY7JJ/E1Fa", - "KLvlWq1ScI9tLey7028rSG73HLZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkV", - "etZ9yXUHzC+8rmU6q9dShiYJnc6sEyCmE4jVU34hcxjlzj2MsigYLrAiNkTxMeb9nupFtl7gOREeUqN/", - "pW79AUGr61Ulx7xW3q7v3r7uERbxuHCeNMuk9skXVpgM/Vdyee//1D3CfFbqxIMmgfEz9t8Ek6MCuvN/", - "7fxkwTv/185POMkoI/9r9yDBiki1eWfEMrivm+6+FZhHTHxaf6HVRVtiTVsKT/081CplOvjam1BmBXbW", - "GbuLRJovRn1fXsirY/W2kvDuje61JnXfiCXHriyB2VIuHHpVDZL431voO6sQ9L3b6yrdUwkjAMHuEUYp", - "c+FQV7GFraaOuC1vYlPA9V5jjCjeammPcO//qUwSZtI3MkoU6/rNLtHGLuEv10rThN2KOzVOmD4eyPte", - "EFtoteHRN7ite3DoWIr04LYqHu4ScGvGpYJHjw97wV4ltKA4/9po6ZksD+TK68OR7vFRFxYSyv5D2SGb", - "2nxPfko3jntXvG2/9y/0HKRjOs15Lv2s6RSraEakRRRISJUBPzaTQHk9NxoFvmIqHdzn1XHvOv83ur8j", - "a0R9Qw3zNsEG62R+91Zbmd++r2V+g7ZsURdsbaSuq5u32ZAE4vCW25JxBZZ6OTklNK6QLoLeaUWlVBcQ", - "aBD7Q/a/tf7xmyI4/fCDS+/OB4Odp/A7YfMPP7gMb3biSIUwJagtc3rw6xFEeEwhiB8qoZZgEvVxoDSX", - "ypCeK3nyb6cglUEu7TUkR4XfNKRWGpK3XKs1JLsXd6siVcsm3buO5OgttOC23sGfU0v6k7tuKxqczCcT", - "GlHCoPgUGP/kUqyy0eS+eW1vCZbAbKyEF+hYkURaq5EF11ojoZeF/79kJGG3sQYFR1gpkmYKTQWOyCRP", - "TNUWJGe5ivkVcyUpYIKuuhkt5xO63l1TI9dIOEE2XKK9raZblGW8b1XXdvxIM1R5ZiuMW+WyFG2atcuH", - "Jd671SlbXLX3r1U+ZhIz6tvy0mVaQwiUWDNunjQ36bzFlyU6Yx+dn792qbtaPRGuYJ/irkqfq+Q8ZH6V", - "vj56WZY/NC+4FrT6QGKb6g8JzbbuXUxwnFBGINeByFCWbbW25oMeiy8vAYcLh963i7vNsbQ1sR9OAn4w", - "VnAvsmbh01fWIVscTa+maHFanLwJp+ZR8SvLgAKMJyTrbeFc8Z4FA9iacYMQGQbJPU1wBBi5+jUD32jx", - "Vwxeq98UgKoIniREGFjOLFdO3BqyYnCUKV4UnraS2YVufpQzRZOLrgk1BGwliTBbWGy6Iat0ZmU+wEgA", - "/A8YoSCZGXGtiq4eNOW5hLcAzsDvEuHkCi/kkFlUBfM5lF4XJDIItknSRz9zALQx1aQ9xmtKuX4nh+yC", - "xgkZWTyaC0QlkjMuFGEkRimfE1ntl2CRUCJgEodYr5xEKV4AMKTByDXrwzNiwBcrqDdc/xuzmEJRUN1z", - "MeX9IcNoZzBAKcFMWgwLiSdw4dg2EAyiMqDvEUZ7gxf2q9q+AXi5W/4NfZqEIHMe4XGyQERTsSm2vQkb", - "mNoivabqu96+CRXS7Fdh37TVFysbS6WrORt3Uc5KlA6w9eesANXQ26VywWCe1gtIqCiuQQtMNCYR1uvJ", - "eLUfgITlUZSL0AWpt9qrFv3vKDh60zuDpQpjYCRgMohIDHvOuJrBmeZwlDa/b6Cqkqj+HBdN8JBwgTDy", - "6Lq0aJAoB9a4ARCqF2XpU+ZKmV9sfu/Ojj6+lhG4429ATB/L/QRExCeTygFcfzWZA7wq92yZhP+s5/TQ", - "1bz2WVxM8ZRxqWjkmKFDqPaV5m8KYSuFcPXKBql5wsVlc8DxT1xcttXAXGDk41LE/Bl+hY4IPTwAwX94", - "fwRYw42yoonm3pW0On0VpxSELqpkERKMEs6m+hSVVvl7dxv4Wt2GAbTUl6kwzu4CfkwrISP7oymb7YV1", - "g4shsq0+NC/Svd+DM+pXrhBNs4SkBMpq9wyx6c0uoerGCwulVwC23YxX6lPl4yoYXVCa+IOuE4eArtyG", - "bYD0vrxdQaaa8Ol6QNSic4f+GUBEHbJ30pQquDCupwtU8GAt0JryI+hqRqMZoKOC3qrbN+CpOMsuCmD4", - "zX30Cg6yj48PnW+YoiOa1iRPiAE9nafpxf5y4ej3JyfwkQFGNSWiL/aRKxZd3B9Sv+WjnepZJFgq9KvF", - "cN0olHHY0QuFtb5ZzG/T4qCWwP1DFsJEZeTKNkgn6MKDR71owO5z/PY1n8qvxlVUllsxc1EcWdURaJOw", - "uNMU5EGTsONnezAIVQFoidJqhnHHIK1Lg3nNp0Wplwop4yxrS752mEDF8zRdQcNow4N8lCrmufqLVDER", - "Aj621N1E3GgDR7bMH77UhGoBPt3B3gTyC4YymdoLwaXSTLXT7RCWp5393+y/5mna6XbsePR3VypLvdoN", - "NxDy16De1htcDr3RO+RB234Tz28CWltl+h5qbe0GsWp1s2T+1rzwp/caOtvdA5IhyAk1Y+7XJIp6460a", - "fhgv0HdhZC/uY2QA0YuihEtScfQ8HoA/a/CqyY7NBiO3xj09vDh3FdHaRLKc2U/P3JdfgQ6+LmbEjRm5", - "6d578MjyCB4zWIFcms2Eizoq3Lqokq+ekL7clixNtQ2FfKPNm1sbWxGm1heWWYT9IDYVMnGueIoVjaA6", - "WzTjXHpkX0C4mzqK1ohcUCaYWIy2azMJLjSpXlhz9IVVJ/at6Qxh/5Htow+f2/yD8BfuUfnFT551oOD4", - "XacCQAUTiTAaC0omKMO5JFqqy1OCokWkuaIpx0dwNEMRzlQuCFQaJSiljKZ56mPz6x2bY8ARuthOL7po", - "nCuUYDEF7cw8dEE3EU9TwmICdrohmxE8p1q1FCjBirBo0ZMEKpTPCbri4jLhOAZTQxZj8PhAhVNBNAVC", - "oYOUKBxjhUHQudAnfmSSmS6KouVGvWfkuqSGeMhEzr43VVd0sxduoBeIQFkBKmdFcdsIx4RFQbj9s6+b", - "jX15m/QZUfWJPlCE0K146UOGDPm2VzecryOa6NFCQ7Rg8yuEXtmswlazQBwZ/XseaTNXN8cHcjQVS7zq", - "FH8dHqaC6L4aL9PDu5G4QHFuuvNOJZD5n9U3VDAUP+gKMkzNNt7WQVRU8SyW+UY8b+sP9+fxLWx5Xwkn", - "7DYq9k314spJfw0s167qrXjuAxkxrS3Jt8k9HAt2kV0PJj5x4XG5x2JsrSC0FXzb505KYNC+OPvGtuts", - "2wY+3JZtO9vskmvfY+SU9SBWNMzBrRm3kVVb08G/aVZKbXYey3xwFll6Lu4dbtGxxgwvEo7jP0Ow8Ar/", - "UcSFMDAYAKzxmCCiPauhnyYAtrmyEGXXZW2+PznZbOISQq3kEUI9Yg7hpeboz9J42YD7Zk6EoLGDwTw8", - "ObJhu1QikbM+epNShRRHl4RkZWYLZBf29fwcIEht2HXkj26HMCUWGadMrR1F+erdDKb8gVtclK9QlLQ1", - "B765w1u7w8Gy//jYGXAZyN0wE1itmSqs1tZCpmzCRWrkMjzmuW5d8yC9THo/DWLBhCZELqQiqYlOnOQJ", - "HDeoX2NrlNvvzC53ITZXnxyTNpcRkVIpKWdyyGzOSEaE7lt/rtv3Aq2CDgGFC/56apjk1xHEpwdj4taw", - "alo1gG6C2sed/c4WzrKtGCvcEChmh/cZQ/oJovKQXKRjntAIJZRdSrSR0EujnqC5RIn+Y3NlWN8IvvvS", - "Fdhvf7L0Sh+zCQ/WtzQ0WxDznyq7y7I155h8dGztFfEPi+M/sNFhtra+xrsgOOlBzXQH4INyRRP60bA6", - "3QiVikYm9QgXa/f+pGCq/SE7IUrodzCkuCWJQTYA7XIrEzzaGuaDwW6UUUCB2yUwOGB4zY9T6PHw9J1J", - "RyUpF4vukOl/QMPnB6fGuzvB1prgDdQWd0fHW2/WBDqfwTL9G0cImgmuRDEIbvg3l+DNsUYaz5BsOKI8", - "W6Uq8exPH8JqJbhvdoXHaVcAsKdiNhsFwJdD5QrbEOY8yVP9D/PH8Tp8M4Wj2Xt49auRds1w1nbjJvgo", - "DqWdU0xM/d0HcXqYBXusMat64dwUQIipRAMGb4ED9Wek7i9vvvfX8St0d9oVdbWtv5qzdd83nx2DQ9rw", - "1+OxHHNDaW4miq+2Pl1h2mx9+jHh0aW0kCy+2VDrbYCzrn8scbGtixDEBMgQRRbKyABmEdkdspoB0iD/", - "SISRIiKlDCdbMGfTCCB8OysWnnMKidoR5Kn0JI0BOykBGG+AwdOzAUOVa8Dz6Epb/c9/x3dGKo7GJOIp", - "cajnmyHV7W+Yqp+4qEKYfy188dxbf4AGxBTs7WtQ25t7/CwU9xN8DaHScW4dym5EG694+aMxBXUR7M2w", - "szuQw04XDTs76bCjd+AQgwkVK/QEpZTlisg+OjL2LUjFfTpAkkScxdKBrzsL3u5ANiXmGrJsyPJ8Ct/d", - "p9hjqQqW8q3tJMQe9HtIfw9JO2jDP3D2TMZdOHQx4rky5n57ruxbMVFgHtm8d1+td0a+6fZtOPnf7PGt", - "8CjYZc0uva03nD3L5Yw0m9xem4JGuRoDqLcrgCxn6O98LLuIkStjDRdS9Zf4nv761HRwHwUHdFc3KTZg", - "5/6t0kCLSgPlWoVBG02Apb6SHXUY5EZynXGhAM3R5twbGgJNAhAkoEzhm8PjIYs0KzIQg4KkHLiTxUU3", - "t/DB387Qy8O3XXQExXjRz/l4s4/esGRha9hbH82QGUnMMK8IMzQ2VEvi0PVsxg7Uc5fB4rqDB6pub05G", - "wLPi9soFiXc7M4JjkEj+6LzmprMA+vDb1/oAAQCw+bLY9s5K4aPzliix6B1MFBHLzZ7YPClWYGfYS9pB", - "0VnBzQBg6g6lQ2Ar+zSygYHI2N3pBBAzPn0r/nD3RZzvx0tm4kRM2b1xrh5t/VY4iAVzDLFA/7ouyic0", - "ZQlbXrZSwYAumyK/vyKT+0reVcGY/3c9XTDTR+toyir7pIm4KLuy1tPrkoNnBhbZOqoinOGIqkUX4SSx", - "d5S9CYqIlF4h/o4FwZcxv2L9IXtbFHyxCb3o8PRd1zlqUUzlpWnB+mL76M2cCJmPi8EhOGjGawxrTuIh", - "UxxFOInyRIsbZDIhEeTiQh0X2eDLLYbSucOzU3YSLDrjRbXnj67WXZgmYPdKsqhT3JbZ6i1BogTTtBmE", - "3ApqEHAIoQZj3ShniLJJYkOqIsGlRLapHknolI4TGyAk++h8RpDEKRmyLMGMEYFyaaLi9dB7mSBS5ibB", - "WzcAYL2GorqoBBjMBFc2NCHhXEgTTaAp/P0JkopkK8jsrWn5BOZ8R7Ktadz29EBG6toYmk0h9hWkN8RQ", - "illwTUd54gIY7zUU3QzooaXEx3LwzwWdTonQpwIbJmvC8cyxdstpDn0lY7mx7uVZ8Va7updFq15Wopex", - "txIgblRibsedm0X9BTq/pI0YgvbRzbKIf9Eftey7mq0aHoR99JmzDJXw/HeslnnmJQm2NWCVFP7YzEne", - "yCtHtZJoux5Wq3Vm7V1murbGz3ow2KzHjJaFK+mzTQrv10cIg/tFebjvYmuPm7YqaFcV3bQh5X89qv5X", - "QYF3A6f/wCgnt4DT/6ry7gHv/OHwT4IH9aHy6Cu+Z1d090+PiH9X6fMGFh/g2JrS5w3Xs8GrKxWl9/ad", - "dmqSbfHPJMHbeMcbyO9u2b9p/S1UBm+x1rmgNcGTNFMLF9BmfZVl0JmkH0m/wRFcxK3enSv4FiGdX448", - "HJ02BnT+OWvkP0jMqC0hSCU6PgoUn39kGIP+matcLFv61ulhEc3onDQb3asn2C5RJkgv4xk4V2KzYHY9", - "3F2msOhPPyLbvMVctf+CGpQA1U9iFFNBIpUsTD1QzRFMH99JJLjWBOA5F4vmKBFzRH4SPD2ws1lzH9oz", - "ZY1hZZxhuujFWOHe3HGbFSa0z4judPGUmuEhytCrH9EGuVbCVLpAE635IDoplpRcR4TEEmhy0x/w9qDB", - "skk/ktF03GaUK2qWvLE1YVCUS8VTt/fHR2gDaqBNCdN7oUX9CUiymeBzGpO4MsbOnCdmVbcbFvSmdlct", - "VBQF7JxyYQb3IDJMmwtp+pFmVbZQhMSMKcMwuLVVQapnyiTx6/4wZS4Ax+6RG8W3K8xqfhtO2dGUCPU4", - "7SIqzg3E8+a3a+4xX3N+MpS70yq3nQvPWW28bpcf1TJt6S4KPxS5c/drtn7/9aT0UPkos3ms6XxeKKRN", - "ZvOviwQH93c/3Le5/P0jTgF9RZzy7ZnKoQHdYohgXkNMd0zmJOFZCnXR4d1Ot5OLpLPfmSmV7W9tQez3", - "jEu1v/fi2W7n04dP/38AAAD//+NCURtO9gEA", + "hQzYch/9WhTW0kstS1SlfsBSWKMnE0CtXxjJhIeiUX4ShCB4Vhb01GcPPNgGo7eIGsAxYDRx5soKog2I", + "8oqJRL/nWGCmKCMxev+T/B4NrMv8/U/IZNFYBkSlH9hY1RSfhba0MRvudLaQNMKJWRZT2oMy30oJR7q1", + "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLnJb", + "VdQ4HJV1FesHAdCgrR2zTh16bwZoTCLsAEXqmxcVZYjHEHsBVq8rQZXSkiu2RmeInikbMaB5Ek3yJOlb", + "BuiGbMzME+DDHoKpuao0+cDzDIMmVYhR1wo6JUjmUURIXAtfqY/ao7d9lLMUCznDSX1y+0WodzTDAkf6", + "Pvzu+rtOuI6QHpxsy+g96ixumtZk6d0rAXr0zl3TEX//U/V4mqveJAdbAzllVAHYGSBcbxSDNJjcfXSC", + "F9aOksGdaeml3DE8mRjIz+IuESQhWEL9b4ne/9RfG5ipuMJJ0xzO9UPLpzb0hPQp0sPsoqzCBoB3eVhb", + "frdPg1dGOZ+Amwk6rPss+uiYhdmeif4BtDwDfCUlnTKt20kbABRhVqzk0t5V786gWaiOLtipLlxlOt3A", + "BRCimNBtZbIZl9Wc5Xqk/LLKpvjl+hqYppHmfg9dwmXNr2V1SXduIS/TRSIh86nlGMY6KkmGBVYkWdRC", + "oqs1DchyUAm5JtENMj5f6tc/mTo2uSAjNRNEznhSDUXa7S7XQpYQ9j8ntvybmZPne1McpVhcwilzujTK", + "mVmBatbI7jqIp5lS2Q0m9fP5+akxsCki5jip5x3JpSCbI5LgBRoTdUUIc1PBEmE/7Lyety0bSnAJNcqI", + "oLy6hp3dQL9nJhUBTQWOCDJfuUrqBVvTR6/tUtpeAqChUUSkbNjf7VX7az+d5Em7PQ4Na3vdDqvoJht8", + "fnjqSkkVpbrdMu8sr/IpET1z5FzN7tVbuyNXFzVzXTGDBb0ks48JVDmzyV5+LrYLMYQqbvrzSv6zxxyk", + "n7Fr+4FTYJaqa875h1YKXv24h2JpUsziUMlzk+Nj0DGmAJkHiQYiB+2LxkZSMneydz/bVCUtaVFGpKxB", + "C0S5SDrdTm9iZ7W/taX5fQKooXu728+3VkdyrwzhtxGLo5iuMvG4uEYT+eYS2Q3kIky6ShJbOMtaGMHN", + "Oq65H4A9LYcMQ5lzfbd5SpbLIRkMllASrnGkXGFGsIpXoh6wf2wB27sqyOgGU1NrSWu7L/zzOQiGoWRY", + "zarkv7VE+xCaBmHFmkaMxbG2jobIPwalUy5C1fe4UDbpdkxc3HFxH7qoXodiXfGPD577s3z65Mnuk3V8", + "CJhN7ZjbcxeYqnm7CitAvOW259c2ABGSVZnDHenVBYT1uqyhKc0Rl0hq9YLyjLAbreeTvd2dm61n24kc", + "u8jMGl8KoTIdnhwZmUjr8pgyIlBKFI6xwlUmA+ZkzWWgvBMmKWTmTb5fzVoaQph8mKXb1qb7UgEwDWUq", + "3zpU9hQzOgElybzp9yxneOfJ031TTDcmk70nT/v9/k3BZ16WaDOttmLLxMl6ODR9Ofu8fbgDjJk2c/mj", + "c3pw/rNmZLkU5tLakmPK9r1/F/8sH8Af5p9jysLYNG3qL9PJUt3lakhobsG/SbyPyhL7Tu5pE6LX4A+C", + "BAEAxAoiPVYCpe8O0rGgcVqt3HGD9PwV6epaXHnDkkXj2ty6lDIrbKHKK6Hs2xValFOmH1eHuDiLM7xj", + "+zSWjqLS9HJwy61qhcuV5VSXqu1lhBUFVJPE/BVxBtjaoWqqlSvSPWtRjA+uEVt1r+jS/7Ho3fvx0B+I", + "97sr5uf9ZMuqfrhhVNpKgfRvy3Loei7kxNE1hzls/i9uhbYlrC0UZDAd44HvwttEblZ7fzP9r9//jzx9", + "9vft31+/f//f81f/dfQr/e/3yembz4IZWo0C+qBQnl8MvRPCFSsQnm1J6QSrKGCj0+pfwwrbJ8bioKIZ", + "FN1FY7I/ZD30mioiTAnHWv7xsIM2CGhK8JUWd6E+lUn93NQfn5qgAv3xH04M/lRvI7a4EMJuSAH3I/Nx", + "zFNM2eaQDZltC7mJSNAL9F8xinBm6jBShrT+u0BjAUUzrZe37LyL/sBZ9mlzyMAqS64NoH2Goe7gpPBN", + "MBejYUdlItHt66RAfjFJwUNW3NYFDKZx9ffLmhmUJPW0vYZFWa2/Wc3p+SAEGAopZXojoS4UqCAFZWsy", + "KnLd0PPB5rI+t0bHKGhoBfnZWBiTc3yQh8zFTXnKRySmEfAVl6o7s8ncRZa0oTRrxMsEv17A3rw1+aMx", + "wrmaaV4UWWyLiPNLSrqwpV3wSEPwFXxpQmpmPOuNF70ZzwqcEyxMwBk2QSlVJfv/9OxEe++JoBPbUxCu", + "QpNIQOiEI2NnZrJ+C+vC0sTOTUkdpkWfObGvm9pM0pTjMdkUKhfMVYwhUDcWAHUK6iMhmfx7FCUUrE5y", + "Bj68GWBeKt1MCLayMyiy1vA4ismk/u9qVNHOk6egwbp/7+60Tho3S7eKyvIkoNOmjvW14NiGTcIAjHgw", + "cobwNXGA+ga0oRRgp1Ac/nuGXEPliSsYifH0mTRWaQuQJNJLYN0MZhraY2BRe0bYnqY299HSKawkLrZo", + "wQQIwWdJC7yglyZh+vz1GVJEpA7CYiPSuwOnxIDD9KiUua2Fd3B48nKz3wlinVVcWrBVKxMbq4MOwJ3Y", + "gKGmOKjSRoNT0kXHR5Cwbq+VUheDDKOfuECJuRXLy2gf8HKq5h5sqmgeH1kBNFmUUUdGbBl2Nl2LWf16", + "20dvCxUQF0MpUo9L2nJNlpcJNGtjUE3601LrtUx18I9Z9c/ex5DsBCVGDS8GvNbG+6u9zdGhv+mLqmYh", + "u/GF5AeCNdq/vL3/0sjmX15G372ZjG690KNshmWIume+VxNeWtp335FdZfeiOWyv0u9I0uDZ+psrtORd", + "Q4roe67yeQjK90lve/t8e+/m5rubglJX0eg8pMoCl7o9oPRdADMHYJapGjXmdyD92GZzOLvI+xM0w5J9", + "p+BhzTqyvfusjVECem2bGeHnRPCJGVLBpRy0XRHRb0D+LmmSGAFG0inDCXqBNs6OX/1y/Pr1JuqhN29O", + "6lux6ovg/twCnxpuAVhHExwVQDeroHKgIn33/Pw1HK6EQAaUkcMvb49avda02ALF2g3u1ek7cPxjOXKx", + "083pwrhMuSfXVCq5DGzYKgXhc1CzzaelYazNJE0bNqhyLfT2zxVs6CBS5eYdYGa7/JGl5XwAOOmHzNP9", + "+qCsV4JPfy6CtLUz3BGAdOOVFgJfroGSPGm63W4PBX0nw6lgOoXYli/hOBCFW2Mvdzs0kEB+YOP40PFp", + "WWytdEa45mtzerHT3376HOoFbw/aMPYURyv6Pjk4bN/5YMfcMvt4vB/F+6Cw39ZnZQnbqCA4ucILqLdp", + "lnbYMRemp916x9Yqkq3ia5Yhrm+HaF0X4xowq0GcdYFLcpSuLPfTIkO4jluY5haVNaVJQiWJOItlVUae", + "YYlkZsCITdmbQoIfMhhgFxXVx0FKQTiKRF6aHq10beX9PLN0D6V3M860DgC1N34hC4lSCk7QonsIfZSo", + "SESLh2xDuKTFIjsRqu7G+gdIAera5JK4C1HDUN5HfzBkcpYrzcQ2++iQM5mnRFirLBpT8BhtIpkblRbG", + "C6ux0AxT0piIIdOvBeCO/yjUk/2ng8Fg0O0Umtyu/vcgRE136vzsWzhvk3YPgJvMAntDtLvIGcpZTERR", + "gp8YcqiHyN3QcfqZON7u83bilf28lKvCB3Md7Hc7PO/PBVGGoTbo5xAdegvl/MntRfRWaX9OfrUJf/ar", + "0U0iGGxKhtb4xvq2MwY5ElszpCQ216PIvXlnyt9Wp25DjxVHv+dELND7k5NK2IPN0Gg3ceASDfvAsxtt", + "w84aG8na0dzEvexBTt8HzHRdUvEkxC8OKu17GB0OgaHQimGrojivsq9prTKYA0WZ2SdNNCsmWKuAbFBb", + "ysBIv1W5kBMLsWpFaZdTYSHfCwSW0qkvF3JrnMutLKJbNntsC+BxngM8zl4QvyAm81Geh1Qj/cjhIb17", + "d3yENuAXgHc2CTKV7jF+uv188PxF7/l4+2lvLx5s9/D27tPezhM8mOxGz3a3d3ZXJBW1SGi9fY5qUGMO", + "xDEXUesjFz0fCmpuyl2oySY2HvuKsphfVa6/YICs37sNvl3X/XJofeshBFOCEiyVMV80cLITuORJpNs2", + "Aek2abqodhY2dD49H2x/rvUHBtdwR5yLnBm3qoH5KFwIqTdgf7Oq47wdy4cBucSXdavld95+0Qb7T17s", + "P/ncRXPJG+vGWCene9zcpogwB4Neyw5xScKeHckZKDtWJjJWfZtM0ul2inwX+BuEgVosdfG4VRJX04Ht", + "htnIqmulAb/guKKvQKSKgcGM97Wk4vQRKKpSoGRoEegw4XmMPFucQQUEP9yxp7voZsAtZk10JnHZJGNA", + "ZiSVNp+PMs2Iwf+oG7FgB/voFbwLj3Bq1Do7CFNCyHe94Xhh4mX0+XJdGyVr9ZDPrH4F32hlC+l/wbT1", + "MliT7eomjHS2j37l8E2h7TFet/2a10HNWn69bifesIj5DrwGOrOi5j76qRAvCwHVCqQbktg/R5ZhlZhR", + "mxXkDrvjHU0t5c55KBTdjlnRTrfjFgrQKpZxK96VVL90/nxSDAWSEZzAWS7T9HNFE4uUDzOhUtFI2uQR", + "vblNYo/NzSTxyChPTTGpJvXVKljFR06qen+CNgAM9S/IGrb1vzaL+NXKXbfzYu/F02c7L562gjwrB7he", + "ND4EZILlwa2Vk6MsH1nbSNPUD0/fGdtHZKwKRezL+xMfYSYTXLMePXPXoN/5i/4LH+kt5vk48RyLFhbS", + "AEvDhgXBDAte1BAH+TtN5nQyYb9/jC53/i5oun39VO6MtxsQrE1HYbPbsR9csGSjJuOeqVQWBuMCghKy", + "Ea/uLZEwA3RGFAL66SEcgXpT5FNbknOodnbFg4S1t7u7+/zZk51WdGVH5x2cERjhApeyHYF3xOBNtPH2", + "7AxteQRn2nSwLlkA5qB2zpAtMz6oCqT97cFuiEoaLu6Samzb87Rxyd9b9dFOyi46JG8XquXSKQ+u9u7u", + "4Nnek+dP2h1jax4eievVHMalLJnlsbUw/J3fAGny/OAUQULwBEdV246LELvRqNSNRgV1XEz9hRsM7Pmz", + "p0/2dne22wEvhoJOLKRo5cBWeVfg0AWIIrAbgaVYZr3dptsiJE4ZAntLogTT9CByKRa128fUWRgJ81q5", + "CW0uBquBL11cLb5tZdwqTFYmQceIBlygnBXVffrrXbJfxLPazLXN9bCeq4fScphePYsQZqoY3mIpM0Hm", + "lOfyCzTElcmZnSScixt926SwvCUyT5Sx2VCJ3p98BzxF0xqSimRVHcpS4woctVtO7kbnuUIiYSJvWqxW", + "u9Fm61dNuNtwarurADUq3KARvTDWnCtn64M/D3ES5VDPChf7qWcFMHwACZBlycLE9icJ5wxFM8zASSI8", + "0DE040ncD0bC6iejSTCqgl+hhBvc9UtCMlvqyQxCf6ZFGDonaMMvcmhIqVZ6+ElqmIwt5lOlxidpuIZq", + "GFOpSIXX64kV9yDBzScVS2jCpxKUQgVZC/16JYoMC5OMgJkpXTZPjS4ZCLgODLHGzEM3qrlJ+cQquFbk", + "gERzs5I4Elx6cGDvT2r5yyty3oos5vUBndXBtiBd49AMXGUGkK51hcPQ/RjI5/mcGxJoGHIGV4RKOuNk", + "ilkOxZ88QraG+H7rcMgZl2pUwFzdcLBSjaCiSy5ICVhZZN0X9iD3TvBedKztNstl445v9fUSVYWbahpg", + "M08Nrmh4tboFDYbIeBlXbiWUXQnwVwezugleZFkChEpolXrIgWgDcl48tuShQG62CZIJq6y6nyVt1Rbo", + "fb03OGuLrLgaSPEUq9kxm/AA9scNPKfOEm2jVTMiHLhhTBglsdMlCxeqNXVBwngiCYpzYlfOyKc+/B2E", + "OGA1A3skfEjZtMbr6x22MQ+bMawu+AL92hfbhDvJcELtuchhrUy8okS4TK1tFQRK5SjszlpuWJBpnmCB", + "LCZqmyHLRZpQdtmmdblIxzyhEdIf1P3iE54k/GqkH8kfYC6brWanPxg1VQc7M4OzeYFmQ2r9llP4Qc9y", + "s5aVDJaYLfP9FjhG20SPBSPFf6IJsdiU7xi99gi9WhFhb2fQlC3f0GglT34ZnPWmnNuSbPDE5zKQW7hS", + "ynF1zUhsy1QYsSfLpSmx1OJWcjjIzgV4O49ONXHk86BJDg2/rgGToDGBvB83tWWu0YIttplKsLpLLmfo", + "73xcNYi2DfsN1AzcYCVEhiCTYHw/7OhKg7R5Y2lNvN29CQYFsFU9UfjohtAO66orlvFVTfzk7VKhwRmx", + "S0bdHE3RwRZFdFz8RwFfYHttj2NQLwkZiFcGlBqpFlBcGSpaLbw6pxKNuRAAAq8lHM7cbAB2Rcs8eq0d", + "7hU6n5EFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHsDmp5la2HoMYDz/TiJ+xYoxDpk/SN14", + "LnU7B8xYFkWeqUrFVt0saH2aUCBrGZxgSkAJU6pmaCKInPlzD5Wt1TLeFRdxYz2wBXKvQJkp8LEixS8J", + "81lZ0UxQNTQNjcxXy1F8puY0PLX6J6qUgUb1Ms+r+8slEWEhsZhS8Uqr0BXvqHjKiQGBAUQUKO9p/zIs", + "vkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6JQ/o+MGbboAk2Mmk8tWCfqidtbagKpMGtQrNZ9iWgDRdC7Wok", + "VSUBr1ZRq3uyXSZePVnAjWZLkqja+97zJ8+etiwW9VnOOoPe9aVdc/N0hUuuYadO2vh9nj95/uLF7t6T", + "Fzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6oPcn//rHP92obj2g", + "EKNZBulv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDQJQ3HRORmbdeuVgajAg7aRgnOGIqkWAkeMr", + "E+1evFLD2m7jDqoONiTymrYtKqrmXDIfl0mnG65z9J/GN1yjheeta87JfNzkh35T79V4oUuvhR/j0CLE", + "wFBE2MBdzOcKy0pAt/47grwLl2G2nG1j3liNultPhYAoFlta0QsFDNUaqMmT9iN/+2vb6fktK2ad+op/", + "WHEOm4/gjay+gRs5YPSN1qfW1viDvQBv99Vo7FeDXFlus1I6srx1b95vi+zh5VIlxQ128/68hMmbfFjH", + "BAZ6tGOwS1623a2QRAM1ebkwAQMaT0jPK15gSm3J3HgE9Zm3MPOBDM7okk8mVazbJ83Y6AD7A8lerhes", + "lNZMuohcO5tFHVjbYPwMO0/ksKNVgGFnOx12am6rYPpkiq9HtoMqtstgFVh5mf5eG6R0MxgnPLo0hQ6h", + "fn4fDVBKMJMoZ3D4a1617cFq71C3k3l7U0CDExPitMS2YExjMsNzCvVUrE9lWgnEJNdUSQgYhXb2UcwN", + "2lOlyrOdoX7NJDful5OGSwezhW1YN6jf48xFtJbvgoFvArWl2UcieNeCFWiO/ebNSdcEMEDooRlYJb7R", + "TdSMQDPIootaeYXy93D88DghIxh3Ha4/XV5HPycdPKuCSKKkxe8uyaFGBCjiOVN1HP+0nSJXTStbvpJy", + "BsF+NvwDcNls74ZAUEwiOJFy+SxWCf0WxF3LG7ArHUoc2A2RMByKFfV33lqHcH0AxtjgFWg37fhx3cZL", + "OJKK24p+xakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKJ2un0b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", + "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtRUy", + "76BypIk3v1XtSBuq/hDlI+1bd1Iycml3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", + "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmU/pK1sTM6nalkUQ3A2QtgKX1W", + "XVVBFGHOUNhm58vddB8uR7vZ7fRbDwnHZx400FJJEyuSjlbhZh+W3jZrnc/wAqw4jU7CZ7t7g8HuzuBW", + "wNluWDdYrsPyE1uFtNpOU0qd95119FeiVP0WiiTr5dLWplyd55SUShCc7kPiTYYjghIyAZC8IqF1vWex", + "3vXqwVuBymbRFvTvNsrum/PBV0vmFF1ZzHE3jY5zLlYxiPznaxyiDWwmWoLUC+Tc7fYGT8+3d/efPN3f", + "3r4LsOtikZqyPZ593L56luzgyV7yfPHs9+3Zs+lOuhvUwy6pqQzUhlZ/0e82RtmUl2QVy6jC0tCGnUNG", + "RL1meb3WvyQJZaQniwyp9WmKK3iB8b+vPf83s/ObGayUHc6qk/RFCKzKxalQ1sPgb9nJrPRd1GdzfLR6", + "FrfKQKoPJExv9aEAebUbDFSo2O58JjJDzlpeQ++8F1tfRCuz4tZdRSEPO5z04C43rHiIvGvADN6sV13g", + "y5dcwHY65YKqWbr6tiheK2DEIW76o1RxFe+pj46njGvtyf+5CJPzlSj9cafbST7uVc+M/b098pdFIC4I", + "0G61LxW0CCNLyJwkq1cBXikVD2Ei2bWursf8w3Zv+wXEISQf934Y9F5UIw66ZrX85dt2b1d+HbRZQ78E", + "oCsdtf3iRhHXbj1XUdAvNFTArryXLTaxpfGyPLy7OlzCbWWDy8dLe1xD8mkUQD9X0rOX28gXmmKS4EUI", + "m94z1Mqa9ugTGRqTKWWyjd12d1AYbp+kw04fHViAcNBlFS/68ZvXtOLTCU1TElMtYxrVvzmDYaelLa6u", + "S9ysNon7KiCt9cPi2ov1EAnrEq7WXZP9z8jH/Sztt53Guwq9A+xqTkUFDDF4sYvoBGFWK1DqilUbpQMS", + "IyFebd8BtZUka3mALOVAZyfpoilXqEyhb2lvy1mzXbAYP7kGe+sKzAxDEDtfBBClABCjq9jX8RHKBI/z", + "qMwfTWDQJeKHyGsQbSuE/PUhuXdp34DE7AkXaL19o8mg0c4+2bTfNdukJtjmrd4erN/qOzGKdDt5Fq/n", + "YealdhzsRsjta1IQAyaa6rLXJEFvMh9acPS3/gou67zGlhxpkSjPnINF09QyJQXcLeBiCMX1HpGE6Gtq", + "uRHEk7jMkqCy5KLrWer20+ezJhcneKSWB/ILIZnWVQD/CPpLMVsEB+bKjhZ3ycbAoX1L4/DqmXJFdrWq", + "g3u2VhJr3CrfhNtUQsFw+ZrN2+ClXHrm7wLj2xfNlhFQHMOvCGlvm0sA2C9d2Fuj/fguzHIPKaS9sa6H", + "GmyrAxUu0NFd/2UssBbrqsS7F3LPh8jiHE+hVt4aP0ojqYTqSoKTAyOFbXErKLCp/4mgiqh0gC1OQIDP", + "XUavLTaIqEJ4iuuVI5rKUhpP9PoYouZCL+fWRNCEyVtPh/XN7we9/zHmdjTq72/98Jf/u/fhP4Nm95oB", + "QRLRi8kEIq4uyaJnqjApPK1eoX+YEhBaq5jaM0NwCsY0QHu3XMkf75NBwT0Xv+J0aQoQquaVUNpeO6G/", + "/EdzoJe3jO/gwlh7dj+7QspdVJJV3N3LGykRUxdU7zLqNvtDBkl6l2QhkVeYzcp27sR+J4tPvFB8dGHI", + "vU/Y/AKNKVS6lEOm1XscRSTTapWt9UNNuXYObFgQnPjt2AJx7rxYz6wJrCDo/ckSnPGbd+c/vnn369Ho", + "zenLXw+OR7+8/G+IdbnqmR7inqa9vSdPbZF2fyW3g4VCbl7voo9ObL6CjXmY5KDZA2CZRGmucoiOIddR", + "kks6d55Sldy+ssVy1vLtK0V8JhSyUkkoPMNCdid0QiDAAe5VG11EpSNGKqG6vbXyUIaWRRdDOMMOXCmK", + "O0kiWFdEb0V4tcuNrS7601m7u8SAxAYOO6T+6sMWsKK+phKAO1wQjPcy2oAUGleC12UQb94MtPagaDAY", + "gvmFKy0NXnyJaqjvVpY/nfOkp/W8hpIRQbO6WYtgCgE0ZVIzOk3el+k4oMxYG/eUTnHA4RJyrHyRqqVu", + "QGtTx5b2v7F8Wzih46heT8McS7NUtfoPNWuJVL3mfI9Ui/cNCMQAsWySeKkXZFjN2E2Z2rLVhUPAITEH", + "VPdVadvlKXMwkT34aH028koF05uZN5LmvTlxelRN01uxQKd6aa5mRBBvI+CDsk7BDZfMJii1gKMx1Rkz", + "IsrgXZfdpMVz8LtLtFGYwNwSFGnXy36B1XUoTvB10QP4lLBccsTCPMo6WNuvfoSaAW9d7U86cU3AMGpa", + "bhghv0pFq9bEUdXyZvhUtTxv837w4FletYL7NZ2tGnGWfVRIM0SPf8NU/cQF6MXN4C93DrQPl39MBIDh", + "1WH0W2HQ05TEI56r1edfv6alR3PlF/Vhy/rCzgaAgYijSl5zEy9w8CTlGJZXWi8HiXJB1eJMr5eNaod8", + "UFfUFxYSOoKfy46hkOqnT2A9nwQyZ14RRgSNoEytPo8pZqAxofcnXrVCU7hyCbgWRKA3h8fW7uKwj0GP", + "pgpIzwWgHpwed7qdORHG9tAZ9Hf7AzjMGWE4o539zm5/uz/ogFY1gylujXOaxDaR3GrUhQZ/HFtJ6Ef3", + "kv5S4JQo+OK3ACQCBGDa10EFwVNPicwwFVaLzBKAajAEQ/XXUHfBXaj75lbummVvbTyGfGtIAyLZG7u5", + "H0BQhrMD09wZDCzCu7LXLyQxmcyJrb/bMNqy31ZSnV2iQBmCJTXPyZbF0n/qdvYG2zca06qhwNkNdfyO", + "YZvNTEA7f3LDhbhVp8fM5CfabHMbF+afOCAk/6z99kHvmczTFIuFWzB/tTIumwRjIhF27xo9TkkUaVYB", + "xZL66A0j5jnCCmETwi1yBjWm3YeaQqunwLTtNrlAa/qRx4svtoSVPpyN4lOVnenj8mmJnr8c7RRkvLyR", + "9pGDGjdUew8E9CMuCqQ/2EnZG7y4+04POZskNFKoVxCwDcymEmKfEgBOdyBMXKDfc64wKvIaHtGRtjLr", + "uCC3bnkVbf1B40/meCck5A84JSLFzGSJmHfWHPql42x8M+VxXnmrOcI/PurYm8qhEZmLCgS56hH1r626", + "MLh8He0FoChsn2Z68QMS/t49nHA72aJG7kMeOahMinJJHtNxsr7GcSmEBGW5V0R9LTQ/uM8ry1ZT+BOe", + "osdCwK9IIeGVu7V0KWxlImdGAQ5KgG/LzE373XdV4e+8fOKFC4FfQzcNdT2UcTDjeNFHbk2N0q8WgDUl", + "CMwzXr5WTvXwvpYTtnMfJwxmXHiKvl1T366pVafcUIubAhxM75S3sEHcyALx57M/3Nj68M320N720Mry", + "wMiVtS78nY/7yIbmRjwmSM54nsRoTJABfnJBOAqL/vQjwiKa0TkBdD+oVpcnimZYQIhNimKssPGhNxom", + "Vpoliua2dHM9F5BZLnAd0EOSEYTAjJqAOMtQTMoYiZH+xEbNlLiKS3XVzdkPGtiLBsurEV3NuCQFsCFT", + "3m0Oed7SaMfQbH/Izi3irV5AiCp3vEaSBHB7V9h/OEN4yOwH3zsW4iLiJE5LzoUFgCdSA9FptmU5x0+P", + "dCQjHgIdOicMM9WTGYnohEZ2WpdkYQNbgw22KkClB+zG+f6kyFxBO5th4DoIXQqjFB8Vz5ClpKr/hkE0", + "eJTkcenkclhKWIxxkgQrlEwTPsbJyKzPJQn4BF/BG3ZRSodL6U1iPCamln62UDPOzN/5OGcqN3+PBb+S", + "RAw7m/0hg4wUu9Yk7pYCIrqCinZpxvU5Ezw1fW6ZIW79cUkWn/pDdhCnlDmKgE9wIjki1/AdxI0BeIjh", + "Xg30YE5T2A9+mEvFUx8C1tGdGSbPVZYrm1ojieqG4E+HTHH0hwO5/LT1R9njJ3AWExxrOvFeMVMC2bpp", + "1HKE9exH8GrA3U5gAYYdfZGaMI+pwEwZ/NICpRNN/S3dKMpEQOnY+gpHmKGMZ6bEBhDVDGuSq7QBoBU4", + "SZCCo+S+1YI77GTDfCwGYTpuBCA0iHG1Y0QZOvnRO0yDvefh8yRJJEgoouS/zt78iuBW1ntgXivDtUxu", + "C9MCA4pzcJ06nvYSRzNkHFVQVXHYofGwU7hz400Yay5tuEyvBz7FH/TQfjDddGn8Q7+vmzLuyn302x+m", + "lX19lrLUAKIOO5+6yHswpWqWj4tnH8IL2oTjdlZhBGjDXHObwEkwBcgd78Y3VyRmMeL2FkgWCKOSA/mB", + "K2PKsFisyqgMLL1dQT4xkYzeYvwxhMjFYWd/6GIXh53usEPYHH6zAY7DzqfwClivZXMJP7jPCudmQURP", + "B4PN9ZDgdn0DPssWjoEvrAM2akVF/VG9gxaP9s/lH/i31j8L1w9muvMSo8ko/s74/ggdEJ7E7muiARdE", + "TezGLCKJE7vXG3ru33mgNysiSXLfBPpQ5Fm4x4qSBY+KHGGzymO00nz/wBQ3uK9LpWK2fxj6fXT284D1", + "3NrOydyFOocLtgAYj1WlkXkZYYnOYEy9M618v4Rf+/a/TvcDcMmLhE8v9o3qjhI+RQllNh/AC1TW4oFd", + "S/jI4PEU31l4Hlctb8NIEv/6xz9hUJRN//WPf1qQ+3/9459w3LcMzhwU276YESzUmGB1sY9+ISTr4YTO", + "iZsMlMMlcyIWaHdgbf7wCHk1/62UJodsyN4SlQvm5U2YwnXSNmhdBXo+lOVEWjwjSBOa2Ko6JrYxYLdx", + "Z9ks5b2e6G4AFxJm4E1A34qOBgBUj5qK41YT7YRNpmbOFaNpPUxzKVhvPX9R5FoZ6u2ZAd6QwcASh84d", + "PLCTRhtnZy83+wi0LUMVUDkJdIeyGatG9L/xpPU8yXCUKkOBVTa8KcIZHtOEOpNjQ9kXcwRTHM0oI2V8", + "cQG67prYdyPVPObg9BjZQMguvDpkb862wMSqSKRyQbqWEwgLtVrWheM2zwV6AP5FFUSH9ey7QzYhGPKE", + "jo8ME/DQyIvEyKJhBogmEONKVaUEXXfIDKSuhXDWBy/lMUngI+h/ihW5wosuKor+ujIxCVZaIZZd/fKQ", + "mVRDuwY9wGxB3jD7wM/MkHouktfmbAkySbRqDBH4pv459L0x4QLZCOdumVfqujPZpmZYetFSHL050/Ob", + "gibIjT0QWnpz5nZjs4skR1FCgRoizIZsCoFADsWYs8quFgllMyziXsT1JeCjWl0yfpWQeNrEYw99IrtD", + "SabST+A4/Vwn18cmXMyWJ6APsUHqW+25O7LvtHPd2Rb/TL47WxHzBs47Y8Elht+Y1f3myGvhyAuvm3Pq", + "hTxrRw6K8u4ifk0XDxTw62hvec3NE2/JHsKihzYcxg94RbhAp4fHCMexIFJu/nvb+/RMDZWW8p++HzUr", + "fojQEzsWLiz6obW3VAnksbCDt3bUCLt51QsN+/fbVqUKUeNNVxQkKq+8u789ap3e5Bophd6S1r7dJGuD", + "bamMONRbLKmlB6JRQgrxpTinPhWtsyqbMN7iylkpLln2fHzkDuT92Zdt1zmr3w33wBSPagzxARlhNdXa", + "Lx/+mKj5XbGLDnZ7hfn56yLNwf1JQfdtig6R+WNSF+PasmkuaIBOGi/QV0QZeJO71NNtD4GJnxHhTrUZ", + "6MLMupiW+RQZnBaYEFhiVuu+x+aVdqqvae/PpPnC8txEYrFL/k1EaaHslmu1SsE9trWw706/rSC53XPY", + "iiWwwCKDFXXs3E5gWd3AcsGizW+RK1+cok1cY6nECjdvEheWbIOmVOhZ9yXXHTC/8LqW6axeSxmaJHQ6", + "s06AmE4gVk/5hcxhlDv3MMqiYLjAitgQxceY93uqF9l6gedEeEiN/pW69QcEra5XlRzzWnm7vnv7ukdY", + "xOPCedIsk9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOCujO/7XzkwXv/F87P+Eko4z8r92DBCsi", + "1eadEcvgvm66+1ZgHjHxaf2FVhdtiTVtKTz181CrlOnga29CmRXYWWfsLhJpvhj1fXkhr47V20rCuze6", + "15rUfSOWHLuyBGZLuXDoVTVI4n9voe+sQtD3bq+rdE8ljAAEu0cYpcyFQ13FFraaOuK2vIlNAdd7jTGi", + "eKulPcK9/6cySZhJ38goUazrN7tEG7uEv1wrTRN2K+7UOGH6eCDve0FsodWGR9/gtu7BoWMp0oPbqni4", + "S8CtGZcKHj0+7AV7ldCC4vxro6VnsjyQK68PR7rHR11YSCj7D2WHbGrzPfkp3TjuXfG2/d6/0HOQjuk0", + "57n0s6ZTrKIZkRZRICFVBvzYTALl9dxoFPiKqXRwn1fHvev83+j+jqwR9Q01zNsEG6yT+d1bbWV++76W", + "+Q3askVdsLWRuq5u3mZDEojDW25LxhVY6uXklNC4QroIeqcVlVJdQKBB7A/Z/9b6x2+K4PTDDy69Ox8M", + "dp7C74TNP/zgMrzZiSMVwpSgtszpwa9HEOExhSB+qIRagknUx4HSXCpDeq7kyb+dglQGubTXkBwVftOQ", + "WmlI3nKt1pDsXtytilQtm3TvOpKjt9CC23oHf04t6U/uuq1ocDKfTGhECYPiU2D8k0uxykaT++a1vSVY", + "ArOxEl6gY0USaa1GFlxrjYReFv7/kpGE3cYaFBxhpUiaKTQVOCKTPDFVW5Cc5SrmV8yVpIAJuupmtJxP", + "6Hp3TY1cI+EE2XCJ9raablGW8b5VXdvxI81Q5ZmtMG6Vy1K0adYuH5Z471anbHHV3r9W+ZhJzKhvy0uX", + "aQ0hUGLNuHnS3KTzFl+W6Ix9dH7+2qXuavVEuIJ9irsqfa6S85D5Vfr66GVZ/tC84FrQ6gOJbao/JDTb", + "uncxwXFCGYFcByJDWbbV2poPeiy+vAQcLhx63y7uNsfS1sR+OAn4wVjBvciahU9fWYdscTS9mqLFaXHy", + "JpyaR8WvLAMKMJ6QrLeFc8V7Fgxga8YNQmQYJPc0wRFg5OrXDHyjxV8xeK1+UwCqIniSEGFgObNcOXFr", + "yIrBUaZ4UXjaSmYXuvlRzhRNLrom1BCwlSTCbGGx6Yas0pmV+QAjAfA/YISCZGbEtSq6etCU5xLeAjgD", + "v0uEkyu8kENmURXM51B6XZDIINgmSR/9zAHQxlST9hivKeX6nRyyCxonZGTxaC4QlUjOuFCEkRilfE5k", + "tV+CRUKJgEkcYr1yEqV4AcCQBiPXrA/PiAFfrKDecP1vzGIKRUF1z8WU94cMo53BAKUEM2kxLCSewIVj", + "20AwiMqAvkcY7Q1e2K9q+wbg5W75N/RpEoLMeYTHyQIRTcWm2PYmbGBqi/Saqu96+yZUSLNfhX3TVl+s", + "bCyVruZs3EU5K1E6wNafswJUQ2+XygWDeVovIKGiuAYtMNGYRFivJ+PVfgASlkdRLkIXpN5qr1r0v6Pg", + "6E3vDJYqjIGRgMkgIjHsOeNqBmeaw1Ha/L6Bqkqi+nNcNMFDwgXCyKPr0qJBohxY4wZAqF6UpU+ZK2V+", + "sfm9Ozv6+FpG4I6/ATF9LPcTEBGfTCoHcP3VZA7wqtyzZRL+s57TQ1fz2mdxMcVTxqWikWOGDqHaV5q/", + "KYStFMLVKxuk5gkXl80Bxz9xcdlWA3OBkY9LEfNn+BU6IvTwAAT/4f0RYA03yoommntX0ur0VZxSELqo", + "kkVIMEo4m+pTVFrl791t4Gt1GwbQUl+mwji7C/gxrYSM7I+mbLYX1g0uhsi2+tC8SPd+D86oX7lCNM0S", + "khIoq90zxKY3u4SqGy8slF4B2HYzXqlPlY+rYHRBaeIPuk4cArpyG7YB0vvydgWZasKn6wFRi84d+mcA", + "EXXI3klTquDCuJ4uUMGDtUBryo+gqxmNZoCOCnqrbt+Ap+IsuyiA4Tf30Ss4yD4+PnS+YYqOaFqTPCEG", + "9HSephf7y4Wj35+cwEcGGNWUiL7YR65YdHF/SP2Wj3aqZ5FgqdCvFsN1o1DGYUcvFNb6ZjG/TYuDWgL3", + "D1kIE5WRK9sgnaALDx71ogG7z/Hb13wqvxpXUVluxcxFcWRVR6BNwuJOU5AHTcKOn+3BIFQFoCVKqxnG", + "HYO0Lg3mNZ8WpV4qpIyzrC352mECFc/TdAUNow0P8lGqmOfqL1LFRAj42FJ3E3GjDRzZMn/4UhOqBfh0", + "B3sTyC8YymRqLwSXSjPVTrdDWJ529n+z/5qnaafbsePR312pLPVqN9xAyF+DeltvcDn0Ru+QB237TTy/", + "CWhtlel7qLW1G8Sq1c2S+Vvzwp/ea+hsdw9IhiAn1Iy5X5Mo6o23avhhvEDfhZG9uI+RAUQvihIuScXR", + "83gA/qzBqyY7NhuM3Br39PDi3FVEaxPJcmY/PXNffgU6+LqYETdm5KZ778EjyyN4zGAFcmk2Ey7qqHDr", + "okq+ekL6cluyNNU2FPKNNm9ubWxFmFpfWGYR9oPYVMjEueIpVjSC6mzRjHPpkX0B4W7qKFojckGZYGIx", + "2q7NJLjQpHphzdEXVp3Yt6YzhP1Hto8+fG7zD8JfuEflFz951oGC43edCgAVTCTCaCwomaAM55JoqS5P", + "CYoWkeaKphwfwdEMRThTuSBQaZSglDKa5qmPza93bI4BR+hiO73oonGuUILFFLQz89AF3UQ8TQmLCdjp", + "hmxG8Jxq1VKgBCvCokVPEqhQPifoiovLhOMYTA1ZjMHjAxVOBdEUCIUOUqJwjBUGQedCn/iRSWa6KIqW", + "G/WekeuSGuIhEzn73lRd0c1euIFeIAJlBaicFcVtIxwTFgXh9s++bjb25W3SZ0TVJ/pAEUK34qUPGTLk", + "217dcL6OaKJHCw3Rgs2vEHplswpbzQJxZPTveaTNXN0cH8jRVCzxqlP8dXiYCqL7arxMD+9G4gLFuenO", + "O5VA5n9W31DBUPygK8gwNdt4WwdRUcWzWOYb8bytP9yfx7ew5X0lnLDbqNg31YsrJ/01sFy7qrfiuQ9k", + "xLS2JN8m93As2EV2PZj4xIXH5R6LsbWC0FbwbZ87KYFB++LsG9uus20b+HBbtu1ss0uufY+RU9aDWNEw", + "B7dm3EZWbU0H/6ZZKbXZeSzzwVlk6bm4d7hFxxozvEg4jv8MwcIr/EcRF8LAYACwxmOCiPashn6aANjm", + "ykKUXZe1+f7kZLOJSwi1kkcI9Yg5hJeaoz9L42UD7ps5EYLGDgbz8OTIhu1SiUTO+uhNShVSHF0SkpWZ", + "LZBd2Nfzc4AgtWHXkT+6HcKUWGScMrV2FOWrdzOY8gducVG+QlHS1hz45g5v7Q4Hy/7jY2fAZSB3w0xg", + "tWaqsFpbC5myCRepkcvwmOe6dc2D9DLp/TSIBROaELmQiqQmOnGSJ3DcoH6NrVFuvzO73IXYXH1yTNpc", + "RkRKpaScySGzOSMZEbpv/blu3wu0CjoEFC7466lhkl9HEJ8ejIlbw6pp1QC6CWofd/Y7WzjLtmKscEOg", + "mB3eZwzpJ4jKQ3KRjnlCI5RQdinRRkIvjXqC5hIl+o/NlWF9I/juS1dgv/3J0it9zCY8WN/S0GxBzH+q", + "7C7L1pxj8tGxtVfEPyyO/8BGh9na+hrvguCkBzXTHYAPyhVN6EfD6nQjVCoamdQjXKzd+5OCqfaH7IQo", + "od/BkOKWJAbZALTLrUzwaGuYDwa7UUYBBW6XwOCA4TU/TqHHw9N3Jh2VpFwsukOm/wENnx+cGu/uBFtr", + "gjdQW9wdHW+9WRPofAbL9G8cIWgmuBLFILjh31yCN8caaTxDsuGI8myVqsSzP30Iq5XgvtkVHqddAcCe", + "itlsFABfDpUrbEOY8yRP9T/MH8fr8M0Ujmbv4dWvRto1w1nbjZvgoziUdk4xMfV3H8TpYRbsscas6oVz", + "UwAhphINGLwFDtSfkbq/vPneX8ev0N1pV9TVtv5qztZ933x2DA5pw1+Px3LMDaW5mSi+2vp0hWmz9enH", + "hEeX0kKy+GZDrbcBzrr+scTFti5CEBMgQxRZKCMDmEVkd8hqBkiD/CMRRoqIlDKcbMGcTSOA8O2sWHjO", + "KSRqR5Cn0pM0BuykBGC8AQZPzwYMVa4Bz6MrbfU//x3fGak4GpOIp8Shnm+GVLe/Yap+4qIKYf618MVz", + "b/0BGhBTsLevQW1v7vGzUNxP8DWESse5dSi7EW284uWPxhTURbA3w87uQA47XTTs7KTDjt6BQwwmVKzQ", + "E5RSlisi++jI2LcgFffpAEkScRZLB77uLHi7A9mUmGvIsiHL8yl8d59ij6UqWMq3tpMQe9DvIf09JO2g", + "Df/A2TMZd+HQxYjnypj77bmyb8VEgXlk8959td4Z+abbt+Hkf7PHt8KjYJc1u/S23nD2LJcz0mxye20K", + "GuVqDKDergCynKG/87HsIkaujDVcSNVf4nv661PTwX0UHNBd3aTYgJ37t0oDLSoNlGsVBm00AZb6SnbU", + "YZAbyXXGhQI0R5tzb2gINAlAkIAyhW8Oj4cs0qzIQAwKknLgThYX3dzCB387Qy8P33bRERTjRT/n480+", + "esOSha1hb300Q2YkMcO8IszQ2FAtiUPXsxk7UM9dBovrDh6our05GQHPitsrFyTe7cwIjkEi+aPzmpvO", + "AujDb1/rAwQAwObLYts7K4WPzluixKJ3MFFELDd7YvOkWIGdYS9pB0VnBTcDgKk7lA6BrezTyAYGImN3", + "pxNAzPj0rfjD3Rdxvh8vmYkTMWX3xrl6tPVb4SAWzDHEAv3ruiif0JQlbHnZSgUDumyK/P6KTO4reVcF", + "Y/7f9XTBTB+toymr7JMm4qLsylpPr0sOnhlYZOuoinCGI6oWXYSTxN5R9iYoIlJ6hfg7FgRfxvyK9Yfs", + "bVHwxSb0osPTd13nqEUxlZemBeuL7aM3cyJkPi4Gh+CgGa8xrDmJh0xxFOEkyhMtbpDJhESQiwt1XGSD", + "L7cYSucOz07ZSbDojBfVnj+6WndhmoDdK8miTnFbZqu3BIkSTNNmEHIrqEHAIYQajHWjnCHKJokNqYoE", + "lxLZpnokoVM6TmyAkOyj8xlBEqdkyLIEM0YEyqWJitdD72WCSJmbBG/dAID1GorqohJgMBNc2dCEhHMh", + "TTSBpvD3J0gqkq0gs7em5ROY8x3JtqZx29MDGalrY2g2hdhXkN4QQylmwTUd5YkLYLzXUHQzoIeWEh/L", + "wT8XdDolQp8KbJisCcczx9otpzn0lYzlxrqXZ8Vb7epeFq16WYlext5KgLhRibkdd24W9Rfo/JI2Ygja", + "RzfLIv5Ff9Sy72q2angQ9tFnzjJUwvPfsVrmmZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/r", + "wWCzHjNaFq6kzzYpvF8fIQzuF+XhvoutPW7aqqBdVXTThpT/9aj6XwUF3g2c/gOjnNwCTv+ryrsHvPOH", + "wz8JHtSHyqOv+J5d0d0/PSL+XaXPG1h8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3", + "UBm8xVrngtYET9JMLVxAm/VVlkFnkn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eeskf8gMaO2hCCV6Pgo", + "UHz+kWEM+meucrFs6Vunh0U0o3PSbHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pQAlQ/", + "iVFMBYlUsjD1QDVHMH18J5HgWhOA51wsmqNEzBH5SfD0wM5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4j", + "utPFU2qGhyhDr35EG+RaCVPpAk205oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oWfLG1oRBUS4V", + "T93eHx+hDaiBNiVM74UW9ScgyWaCz2lM4soYO3OemFXdbljQm9pdtVBRFLBzyoUZ3IPIMG0upOlHmlXZ", + "QhESM6YMw+DWVgWpnimTxK/7w5S5ABy7R24U364wq/ltOGVHUyLU47SLqDg3EM+b3665x3zN+clQ7k6r", + "3HYuPGe18bpdflTLtKW7KPxQ5M7dr9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9BVx", + "yrdnKocGdIshgnkNMd0xmZOEZynURYd3O91OLpLOfmemVLa/tQWx3zMu1f7ei2e7nU8fPv3/AQAA//8A", + "qZIc0fkBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index a6218cadd..fc3f221eb 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -354,6 +354,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 054e3744e..6dfd537ae 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -10,16 +10,25 @@ import ( // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + + // PlacementDisabledReason is set when AllocatableSlots is 0 because the + // VF health state could not be read or written, not because the host is + // full. + PlacementDisabledReason string `json:"placement_disabled_reason,omitempty"` } -// GetGPUStatus returns the current GPU resource status. -// Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) *GPUResourceStatus { +// GetGPUStatus returns the current GPU resource status and any error that +// prevents determining allocatable vGPU capacity. The status is still +// returned alongside such an error, with PlacementDisabledReason set. It +// returns nil if no GPU is available or the mode is "none". +func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -27,16 +36,16 @@ func GetGPUStatus(ctx context.Context) *GPUResourceStatus { // expose the PFs/VFs as available passthrough slots while active vGPU // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return nil, nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) } - return getPassthroughStatus() + return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -45,19 +54,29 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d } } + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + } + // One VF health snapshot serves both the slot counts and the profile + // listing, so a status read touches the store once. + availability, err := devices.GetVGPUAvailability(framework, vfs) + if err != nil { + status.PlacementDisabledReason = err.Error() + return status, err + } + status.AllocatableSlots = availability.AllocatableSlots + status.QuarantinedSlots = availability.QuarantinedSlots + // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs, availability.Quarantined) if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, - } + status.Profiles = profiles + return status, nil } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -92,9 +111,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..e15def07f --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,104 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testVFQuarantineThreshold = 2 + +// initVFHealthForTest points the VF health store at a state file for one test +// and detaches it from disk again afterwards. +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + path := paths.New(t.TempDir()).VFHealthState() + if state != nil { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + err := devices.InitVFHealth(path, testVFQuarantineThreshold) + if state == nil { + require.NoError(t, err) + } + t.Cleanup(func() { require.NoError(t, devices.InitVFHealth("", testVFQuarantineThreshold)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, err, "VF health state unavailable") + assert.Equal(t, err.Error(), status.PlacementDisabledReason) +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + require.NoError(t, err) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + disabled := *status + disabled.PlacementDisabledReason = "VF health state unavailable: read failed" + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return &disabled, errors.New(disabled.PlacementDisabledReason) + }) + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + statusMgr, _, _ := monitoringTestManager(t) + full, err := statusMgr.GetFullStatus(ctx) + require.NoError(t, err) + require.NotNil(t, full.GPU) + assert.Equal(t, "VF health state unavailable: read failed", full.GPU.PlacementDisabledReason) + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + full, err = statusMgr.GetFullStatus(ctx) + require.NoError(t, err) + assert.Empty(t, full.GPU.PlacementDisabledReason) + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index d69b1e2a4..59851658d 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -191,7 +191,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_slots", - metric.WithDescription("Total and used GPU slots"), + metric.WithDescription("Total, used, allocatable, and quarantined GPU slots"), ) if err != nil { return err @@ -205,6 +205,14 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { return err } + gpuPlacementDisabled, err := meter.Int64ObservableGauge( + "hypeman_resources_gpu_placement_disabled", + metric.WithDescription("1 while vGPU placement is refused because the VF health state is unavailable, otherwise 0"), + ) + if err != nil { + return err + } + if _, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error { snapshot, ok := mgr.currentMonitoringSnapshot() if !ok { @@ -242,6 +250,13 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { if snapshot.status.GPU != nil { o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.UsedSlots), metric.WithAttributes(attribute.String("kind", "used"))) o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.TotalSlots), metric.WithAttributes(attribute.String("kind", "total"))) + o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.AllocatableSlots), metric.WithAttributes(attribute.String("kind", "allocatable"))) + o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.QuarantinedSlots), metric.WithAttributes(attribute.String("kind", "quarantined"))) + var placementDisabled int64 + if snapshot.status.GPU.PlacementDisabledReason != "" { + placementDisabled = 1 + } + o.ObserveInt64(gpuPlacementDisabled, placementDisabled) for _, profile := range snapshot.status.GPU.Profiles { o.ObserveInt64(gpuProfileSlots, int64(profile.Available), metric.WithAttributes( @@ -253,7 +268,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { } return nil - }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots); err != nil { + }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots, gpuPlacementDisabled); err != nil { return err } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index bef0740dc..357e1c79c 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -3,6 +3,7 @@ package resources import ( "bytes" "context" + "errors" "os" "path/filepath" "sync" @@ -198,16 +199,18 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ - Mode: "vgpu", - TotalSlots: 8, - UsedSlots: 3, + Mode: "vgpu", + TotalSlots: 8, + UsedSlots: 3, + AllocatableSlots: 4, + QuarantinedSlots: 1, Profiles: []devices.GPUProfile{ {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, }, - } + }, nil }) defer func() { setGPUStatusProvider(originalProvider) @@ -225,8 +228,41 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { rm := collectMonitoringMetrics(t, reader) require.Equal(t, int64(3), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "used"})) require.Equal(t, int64(8), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "total"})) + require.Equal(t, int64(4), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "allocatable"})) + require.Equal(t, int64(1), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "quarantined"})) require.Equal(t, int64(5), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-1Q", "kind": "available"})) require.Equal(t, int64(2), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-2Q", "kind": "available"})) + require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_resources_gpu_placement_disabled", nil)) +} + +func TestStartMonitoringPublishesGPUPlacementDisabled(t *testing.T) { + mgr, _, _ := monitoringTestManager(t) + + originalProvider := currentGPUStatusProvider() + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return &GPUResourceStatus{ + Mode: "vgpu", + TotalSlots: 8, + UsedSlots: 3, + PlacementDisabledReason: "VF health state unavailable: read failed", + }, errors.New("VF health state unavailable: read failed") + }) + defer func() { + setGPUStatusProvider(originalProvider) + }() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + require.NoError(t, mgr.StartMonitoring(ctx, provider.Meter("test"), time.Hour)) + waitForMonitoringSnapshot(t, mgr) + + rm := collectMonitoringMetrics(t, reader) + require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "allocatable"})) + require.Equal(t, int64(1), int64GaugeValue(t, rm, "hypeman_resources_gpu_placement_disabled", nil)) } func TestStartMonitoringPublishesDiskUtilizationFromCachedSnapshot(t *testing.T) { diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 86f644bda..d674c5d01 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -426,8 +426,9 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } } - // Get GPU status - gpuStatus := currentGPUStatusProvider()(ctx) + // A GPU status error only means vGPU placement is disabled. The status + // carries the reason, so it is reported rather than failing the read. + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -620,7 +621,23 @@ func (m *Manager) admissionStatusLocked(rt ResourceType, visibleAllocated int64, return status, nil } -func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string, req pendingAllocation) error { +// gpuAdmission is the GPU status an admission check consumes. It is read +// before the manager lock is taken: the provider walks sysfs and may retry a +// failed VF health write, and neither should stall CPU or memory admission. +type gpuAdmission struct { + status *GPUResourceStatus + err error +} + +func (m *Manager) gpuAdmissionFor(ctx context.Context, req pendingAllocation) gpuAdmission { + if req.GPUSlots == 0 { + return gpuAdmission{} + } + status, err := currentGPUStatusProvider()(ctx) + return gpuAdmission{status: status, err: err} +} + +func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string, req pendingAllocation, gpu gpuAdmission) error { usage, err := m.collectAdmissionUsageLocked(ctx) if err != nil { return err @@ -691,15 +708,18 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := gpu.status, gpu.err if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatusErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatusErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) @@ -713,20 +733,22 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Returns nil if allocation is allowed, or a detailed error describing // which resource is insufficient and the current capacity/usage. func (m *Manager) ValidateAllocation(ctx context.Context, vcpus int, memoryBytes int64, networkDownloadBps int64, networkUploadBps int64, diskIOBps int64, diskBytes int64, needsGPU bool) error { + req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) + gpu := m.gpuAdmissionFor(ctx, req) + m.mu.RLock() defer m.mu.RUnlock() - - req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) - return m.validateAllocationLocked(ctx, "", req) + return m.validateAllocationLocked(ctx, "", req, gpu) } // ReserveAllocation tentatively reserves resources for an in-flight operation. func (m *Manager) ReserveAllocation(ctx context.Context, instanceID string, vcpus int, memoryBytes int64, networkDownloadBps int64, networkUploadBps int64, diskIOBps int64, diskBytes int64, needsGPU bool) error { + req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) + gpu := m.gpuAdmissionFor(ctx, req) + m.mu.Lock() defer m.mu.Unlock() - - req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) - if err := m.validateAllocationLocked(ctx, instanceID, req); err != nil { + if err := m.validateAllocationLocked(ctx, instanceID, req, gpu); err != nil { return err } if existing, ok := m.pending[instanceID]; ok { diff --git a/openapi.yaml b/openapi.yaml index edcc53a3c..3621c3e8c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1809,7 +1809,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1822,8 +1822,20 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 + placement_disabled_reason: + type: string + description: Present when allocatable_slots is 0 because the VF health state could not be read or written rather than because the host is full. vGPU placement is refused until the state file is repaired or the next write succeeds. + example: "VF health state unavailable: unmarshal VF health state: invalid character 'x'" profiles: type: array description: Available vGPU profiles (only in vGPU mode)