From fb18c850974b285dc81a579320056a22874c0c6c Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Fri, 30 Oct 2026 16:33:03 +0800 Subject: [PATCH] fix(vmm): reject undersized pci64 windows before gpu initialization Signed-off-by: Leechael Yim --- dstack/vmm/src/app.rs | 11 ++++- dstack/vmm/src/app/qemu.rs | 98 ++++++++++++++++++++++++++++++++++++++ dstack/vmm/src/one_shot.rs | 5 +- 3 files changed, 109 insertions(+), 5 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 379bea90a..9c3c475bf 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -58,7 +58,7 @@ mod id_pool; mod image; mod mr_config; pub(crate) mod network; -mod qemu; +pub(crate) mod qemu; mod vm_info; mod workdir; @@ -542,6 +542,14 @@ impl App { vm_state.config.clone() }; if !is_running { + let devices = self.try_allocate_gpus(&vm_config.manifest)?; + if let Err(err) = + qemu::check_passthrough_pci64(&devices, self.config.cvm.qemu_pci_hole64_size) + { + self.set_started(id, false)?; + self.vm_event_report(vm_config.cid, "boot.error", format!("{err:#}"))?; + return Err(err); + } let work_dir = self.work_dir(id)?; for path in [work_dir.serial_pty(), work_dir.qmp_socket()] { if path.symlink_metadata().is_ok() { @@ -560,7 +568,6 @@ impl App { } let mut runtime_networks = self.runtime_networks(&vm_config.manifest); - let devices = self.try_allocate_gpus(&vm_config.manifest)?; let gpu_host_config = self.config.cvm.gpu.clone(); let devices_to_sanitize = devices.clone(); tokio::task::spawn_blocking(move || { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index e7594da57..3ca2c260c 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -39,6 +39,104 @@ use std::{ }; use supervisor_client::supervisor::ProcessConfig; +// QEMU 9.2 hw/pci-host/q35.c: Q35_PCI_HOST_HOLE64_SIZE_DEFAULT. +const Q35_DEFAULT_PCI_HOLE64_SIZE: u64 = 1 << 35; + +fn pci64_bar_bytes(resource: &str) -> Result { + const IORESOURCE_MEM: u64 = 0x200; + const IORESOURCE_MEM_64: u64 = 0x100000; + let mut total = 0u64; + let mut rows = resource.lines(); + for _ in 0..6 { + let row = rows.next().context("missing PCI BAR resource row")?; + let fields = row.split_whitespace().collect::>(); + if fields.len() != 3 { + bail!("invalid PCI BAR resource row: {row}"); + } + let parse = |s: &str| u64::from_str_radix(s.strip_prefix("0x").unwrap_or(s), 16); + let start = parse(fields[0]).context("invalid PCI BAR start")?; + let end = parse(fields[1]).context("invalid PCI BAR end")?; + let flags = parse(fields[2]).context("invalid PCI BAR flags")?; + if start == 0 && end == 0 { + continue; + } + if flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) != (IORESOURCE_MEM | IORESOURCE_MEM_64) { + continue; + } + let size = end + .checked_sub(start) + .and_then(|size| size.checked_add(1)) + .context("invalid PCI BAR range")?; + total = total.checked_add(size).context("PCI BAR size overflow")?; + } + Ok(total) +} + +fn check_pci64_capacity(required: u64, configured: u64) -> Result<()> { + let effective = if configured == 0 { + Q35_DEFAULT_PCI_HOLE64_SIZE + } else { + configured + }; + // This necessary-size check does not model firmware alignment gaps or layout. + if required > effective { + bail!("passthrough 64-bit MMIO BARs require {required} bytes, but effective PCI64 size is {effective} bytes (qemu_pci_hole64_size={configured}); increase qemu_pci_hole64_size"); + } + Ok(()) +} + +pub(crate) fn check_passthrough_pci64(gpus: &GpuConfig, configured: u64) -> Result<()> { + let mut required = 0u64; + for device in gpus.gpus.iter().chain(&gpus.bridges) { + let path = Path::new("/sys/bus/pci/devices") + .join(&device.slot) + .join("resource"); + let bytes = (|| -> Result { pci64_bar_bytes(&fs::read_to_string(&path)?) })() + .with_context(|| { + format!( + "failed to read PCI BAR resources for {} at {}", + device.slot, + path.display() + ) + })?; + required = required + .checked_add(bytes) + .context("passthrough PCI BAR size overflow")?; + } + check_pci64_capacity(required, configured) +} + +#[cfg(test)] +mod pci64_tests { + use super::{check_passthrough_pci64, check_pci64_capacity, pci64_bar_bytes, GpuConfig}; + + #[test] + fn checks_gpu_bar_capacity() { + // usw5 B300 (10de:3182), captured 2026-10-30; extra row tests exclusion. + let resource = "0x0000238000000000 0x0000238003ffffff 0x000000000014220c\n\ + 0 0 0\n\ + 0x0000228000000000 0x000022ffffffffff 0x000000000014220c\n\ + 0 0 0\n\ + 0x0000238044000000 0x0000238045ffffff 0x000000000014220c\n\ + 0 0 0\n\ + 0 ffffffffffffffff 00100200\n"; + let bytes = pci64_bar_bytes(resource).unwrap(); + assert_eq!(bytes, (512u64 << 30) + (96 << 20)); + assert_eq!(bytes * 8, (4u64 << 40) + (768 << 20)); + assert!(check_pci64_capacity(bytes * 8, 4u64 << 40).is_err()); + assert!(check_pci64_capacity(bytes * 8, 8u64 << 40).is_ok()); + let b200 = resource.replace("0x000022ffffffffff", "0x000022bfffffffff"); + let b200_bytes = pci64_bar_bytes(&b200).unwrap(); + assert_eq!(b200_bytes, (256u64 << 30) + (96 << 20)); + assert!(check_pci64_capacity(b200_bytes * 8, 4u64 << 40).is_ok()); + assert!(check_pci64_capacity(32u64 << 30, 0).is_ok()); + assert!(check_pci64_capacity((32u64 << 30) + 1, 0).is_err()); + assert!(check_passthrough_pci64(&GpuConfig::default(), 0).is_ok()); + assert!(pci64_bar_bytes("0 0 0\n").is_err()); + assert!(pci64_bar_bytes(&resource.replace("0x000022ffffffffff", "bad!")).is_err()); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct AmdSevSnpLaunchParams { cbitpos: u32, diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index 4aa36741b..34b4f3515 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -92,6 +92,8 @@ pub async fn run_one_shot( // Create manifest using shared logic let manifest = create_manifest_from_vm_config(vm_config.clone(), &config.cvm)?; + let gpus = manifest.gpus.as_ref().cloned().unwrap_or_default(); + crate::app::qemu::check_passthrough_pci64(&gpus, config.cvm.qemu_pci_hole64_size)?; // Load image let image_path = config.image.path.join(&manifest.image); @@ -269,9 +271,6 @@ Compose file content (first 200 chars): .set_started(false) .context("Failed to create vm-state.json")?; - // Get GPU config from the manifest (already processed) - let gpus = manifest.gpus.as_ref().cloned().unwrap_or_default(); - // Build VM config and generate QEMU command let vm_builder_config = VmConfig {