From 760c18bc6856bd419f1b530cb1ab230ee20c7bf8 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Tue, 12 May 2026 17:33:55 +0100 Subject: [PATCH 01/28] virtio: Change VirtioDevice::reset() return type to bool The reset() trait method returns Option<(Arc, Vec)> for no apparent reason and the values are never used by any caller. Change the return value to a bool to signify success or failure. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/device.rs | 7 +- src/vmm/src/devices/virtio/transport/mmio.rs | 10 +-- .../devices/virtio/transport/pci/device.rs | 65 +++++++++---------- 3 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/vmm/src/devices/virtio/device.rs b/src/vmm/src/devices/virtio/device.rs index 31c70496c04..13d8b35b36d 100644 --- a/src/vmm/src/devices/virtio/device.rs +++ b/src/vmm/src/devices/virtio/device.rs @@ -194,10 +194,9 @@ pub trait VirtioDevice: AsAny + MutEventSubscriber + Send { /// Checks if the resources of this device are activated. fn is_activated(&self) -> bool; - /// Optionally deactivates this device and returns ownership of the guest memory map, interrupt - /// event, and queue events. - fn reset(&mut self) -> Option<(Arc, Vec)> { - None + /// Reset the device. Returns true on success, false otherwise. + fn reset(&mut self) -> bool { + false } /// Mark pages used by queues as dirty. diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index 7488e1b3a4f..c4165623e02 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -187,12 +187,8 @@ impl MmioTransport { let mut locked_device = self.device.lock().expect("Poisoned lock"); if locked_device.is_activated() { let mut device_status = self.device_status; - let reset_result = locked_device.reset(); - match reset_result { - Some((_interrupt_evt, mut _queue_evts)) => {} - None => { - device_status |= FAILED; - } + if !locked_device.reset() { + device_status |= FAILED; } self.device_status = device_status; } @@ -614,7 +610,7 @@ pub(crate) mod tests { let interrupt = Arc::new(IrqTrigger::new()); let mut dummy = DummyDevice::new(); // Validate reset is no-op. - assert!(dummy.reset().is_none()); + assert!(!dummy.reset()); let mut d = MmioTransport::new(m, interrupt, Arc::new(Mutex::new(dummy)), false); // We just make sure here that the implementation of a mmio device behaves as we expect, diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index 050cc9d287f..dafb882916e 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -974,41 +974,36 @@ impl PciDevice for VirtioPciDevice { // Device has been reset by the driver if self.device_activated.load(Ordering::SeqCst) && self.is_driver_init() { let mut device = self.device.lock().unwrap(); - let reset_result = device.reset(); - match reset_result { - Some(_) => { - // Upon reset the device returns its interrupt EventFD - self.virtio_interrupt = None; - self.device_activated.store(false, Ordering::SeqCst); - - // Reset queue readiness (changes queue_enable), queue sizes - // and selected_queue as per spec for reset - self.virtio_device() - .lock() - .unwrap() - .queues_mut() - .iter_mut() - .for_each(Queue::reset); - self.common_config.queue_select = 0; - } - None => { - error!("Attempt to reset device when not implemented in underlying device"); - // The virtio spec does not specify what to do if reset fails. - // - // Our MMIO transport sets FAILED in this case, but we must NOT do that for PCI. - // During shutdown, the Linux kernel issues a reset to each virtio device. The - // virtio PCI driver then polls device_status until it reads back 0, unlike the - // virtio MMIO driver which simply writes 0 and returns. Setting FAILED would - // cause the poll to spin forever, breaking reboot command and Ctrl-Alt-Del. - // - PCI: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_pci_modern.c#L546-L565 - // - MMIO: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_mmio.c#L251-L258 - // - // Since device_status was already set to INIT by set_device_status(), we don't - // need to set it again here. However, the backend device is still active since - // reset() is unimplemented. The combination of device_activated == true and - // device_status == INIT will cause set_device_status() to block any - // re-initialization attempts. - } + if device.reset() { + self.virtio_interrupt = None; + self.device_activated.store(false, Ordering::SeqCst); + + // Reset queue readiness (changes queue_enable), queue sizes + // and selected_queue as per spec for reset + self.virtio_device() + .lock() + .unwrap() + .queues_mut() + .iter_mut() + .for_each(Queue::reset); + self.common_config.queue_select = 0; + } else { + error!("Attempt to reset device when not implemented in underlying device"); + // The virtio spec does not specify what to do if reset fails. + // + // Our MMIO transport sets FAILED in this case, but we must NOT do that for PCI. + // During shutdown, the Linux kernel issues a reset to each virtio device. The + // virtio PCI driver then polls device_status until it reads back 0, unlike the + // virtio MMIO driver which simply writes 0 and returns. Setting FAILED would + // cause the poll to spin forever, breaking reboot command and Ctrl-Alt-Del. + // - PCI: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_pci_modern.c#L546-L565 + // - MMIO: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_mmio.c#L251-L258 + // + // Since device_status was already set to INIT by set_device_status(), we don't + // need to set it again here. However, the backend device is still active since + // reset() is unimplemented. The combination of device_activated == true and + // device_status == INIT will cause set_device_status() to block any + // re-initialization attempts. } } None From 2e2eb2dc8dc16a4d78356c213228917bf7e61229 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Wed, 13 May 2026 17:50:21 +0100 Subject: [PATCH 02/28] virtio: mmio: Simplify MMIO transport reset logic Simplify the MMIO transport reset logic by always performing the device reset regardless of activation state. This requires updating the affected unit tests and implementing reset() for the DummyDevice. The unit test assumed that reset() is never supported (i.e. always returns false), but that is going to change soon so we want to make sure that resetting moves the device to the deactivated state. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/transport/mmio.rs | 36 +++++++++----------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index c4165623e02..a6eb01e039b 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -165,7 +165,6 @@ impl MmioTransport { /// of the driver initialization sequence specified in 3.1. The driver MUST NOT clear /// a device status bit. If the driver sets the FAILED bit, the driver MUST later reset /// the device before attempting to re-initialize. - #[allow(unused_assignments)] fn set_device_status(&mut self, status: u32) { use device_status::*; @@ -183,22 +182,13 @@ impl MmioTransport { // TODO: notify backend driver to stop the device self.device_status |= FAILED; } else if status == INIT { - { - let mut locked_device = self.device.lock().expect("Poisoned lock"); - if locked_device.is_activated() { - let mut device_status = self.device_status; - if !locked_device.reset() { - device_status |= FAILED; - } - self.device_status = device_status; + if self.device_status != INIT { + if self.device.lock().expect("Poisoned lock").reset() { + self.reset(); + } else { + self.device_status |= FAILED; } } - - // If the backend device driver doesn't support reset, - // just leave the device marked as FAILED. - if self.device_status & FAILED == 0 { - self.reset(); - } } else if VALID_TRANSITIONS .iter() .any(|&(from, to)| self.device_status == from && status == to) @@ -596,6 +586,12 @@ pub(crate) mod tests { fn is_activated(&self) -> bool { self.device_activated } + + fn reset(&mut self) -> bool { + self.device_activated = false; + self.acked_features = 0; + true + } } fn set_device_status(d: &mut MmioTransport, status: u32) { @@ -609,8 +605,7 @@ pub(crate) mod tests { let m = single_region_mem(0x1000); let interrupt = Arc::new(IrqTrigger::new()); let mut dummy = DummyDevice::new(); - // Validate reset is no-op. - assert!(!dummy.reset()); + assert!(dummy.reset()); let mut d = MmioTransport::new(m, interrupt, Arc::new(Mutex::new(dummy)), false); // We just make sure here that the implementation of a mmio device behaves as we expect, @@ -1140,11 +1135,12 @@ pub(crate) mod tests { assert_eq!(d.device_status, 0x8f); assert!(d.locked_device().is_activated()); - // Nothing happens when backend driver doesn't support reset + // Resetting the device should deactivate it write_le_u32(&mut buf[..], 0x0); d.write(0x0, 0x70, &buf[..]); - assert_eq!(d.device_status, 0x8f); - assert!(d.locked_device().is_activated()); + assert_eq!(d.device_status, device_status::INIT); + assert!(!d.locked_device().is_activated()); + assert_eq!(d.locked_device().acked_features(), 0); } #[test] From f417588d9ad6c61510c238cdeac26f2a3b60f4ed Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 15:55:49 +0100 Subject: [PATCH 03/28] virtio: pci: Fix deadlock in device reset path The reset success path locked self.device, then tried to lock it again via self.virtio_device().lock() to reset queues. This was a deadlock that was never triggered because no device previously implemented reset() (all returned false). Use the already-held guard instead. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/transport/pci/device.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index dafb882916e..c8de01f40b6 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -980,12 +980,7 @@ impl PciDevice for VirtioPciDevice { // Reset queue readiness (changes queue_enable), queue sizes // and selected_queue as per spec for reset - self.virtio_device() - .lock() - .unwrap() - .queues_mut() - .iter_mut() - .for_each(Queue::reset); + device.queues_mut().iter_mut().for_each(Queue::reset); self.common_config.queue_select = 0; } else { error!("Attempt to reset device when not implemented in underlying device"); From f524b0eabeb592a328444bf133b25a06b6f7ebeb Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 16:05:49 +0100 Subject: [PATCH 04/28] virtio: pci: Reset feature select registers on device reset Reset device_feature_select and driver_feature_select to 0 on device reset, matching what the MMIO transport does with features_select and acked_features_select. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/transport/pci/device.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index c8de01f40b6..095a1efdc19 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -982,6 +982,8 @@ impl PciDevice for VirtioPciDevice { // and selected_queue as per spec for reset device.queues_mut().iter_mut().for_each(Queue::reset); self.common_config.queue_select = 0; + self.common_config.device_feature_select = 0; + self.common_config.driver_feature_select = 0; } else { error!("Attempt to reset device when not implemented in underlying device"); // The virtio spec does not specify what to do if reset fails. From 0c4f6331a4596361a978714a75eb6c2151b80ed5 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 15:56:03 +0100 Subject: [PATCH 05/28] virtio: pci: Drop MSI-X configuration on device reset Add a reset_msix() method that clears the MSI-X configuration, unregisters irqfds and removes the GSI routes from the routing table. Add the eventfd2 system call to the vCPU seccomp filter to allow KvmVm::create_msix_group() create new eventfds for interrupts from within a vCPU thread (where the reset happens). Signed-off-by: Ilias Stamatis --- .../seccomp/aarch64-unknown-linux-musl.json | 4 ++ .../seccomp/x86_64-unknown-linux-musl.json | 4 ++ .../devices/virtio/transport/pci/device.rs | 69 ++++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/resources/seccomp/aarch64-unknown-linux-musl.json b/resources/seccomp/aarch64-unknown-linux-musl.json index 1e0047266e6..f6561c3f2ed 100644 --- a/resources/seccomp/aarch64-unknown-linux-musl.json +++ b/resources/seccomp/aarch64-unknown-linux-musl.json @@ -1029,6 +1029,10 @@ { "syscall": "close" }, + { + "syscall": "eventfd2", + "comment": "Needed for recreating the MSI-X vector eventfds when a PCI device is reset by the guest (MsixVectorGroup recreated from the vCPU thread)" + }, { "syscall": "fstat", "comment": "Used for reading the local timezone from /etc/localtime" diff --git a/resources/seccomp/x86_64-unknown-linux-musl.json b/resources/seccomp/x86_64-unknown-linux-musl.json index ea4d49e98b5..e1326800798 100644 --- a/resources/seccomp/x86_64-unknown-linux-musl.json +++ b/resources/seccomp/x86_64-unknown-linux-musl.json @@ -1041,6 +1041,10 @@ { "syscall": "close" }, + { + "syscall": "eventfd2", + "comment": "Needed for recreating the MSI-X vector eventfds when a PCI device is reset by the guest (MsixVectorGroup recreated from the vCPU thread)" + }, { "syscall": "fstat", "comment": "Used for reading the local timezone from /etc/localtime" diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index 095a1efdc19..fc993a29535 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -652,6 +652,68 @@ impl VirtioPciDevice { self.set_notification_ioevents(vm, false) } + /// Tear down the MSI-X configuration. Used on device reset. + fn reset_msix(&mut self) { + let (num_vectors, vm) = { + let msix_config = self.msix_config.lock().expect("Poisoned lock"); + ( + msix_config.vectors.num_vectors(), + msix_config.vectors.vm.clone(), + ) + }; + + // Build a fresh MSI-X vector group and configuration. + let msix_vectors = match KvmVm::create_msix_group(vm.clone(), num_vectors) { + Ok(vectors) => Arc::new(vectors), + Err(err) => { + error!("Failed to recreate MSI-X vector group on reset: {err:?}"); + return; + } + }; + + self.common_config + .msix_config + .store(VIRTQ_MSI_NO_VECTOR, Ordering::Release); + for vector in self + .common_config + .msix_queues + .lock() + .expect("Poisoned lock") + .iter_mut() + { + *vector = VIRTQ_MSI_NO_VECTOR; + } + + let msix_config = Arc::new(Mutex::new(MsixConfig::new(msix_vectors.clone(), self.sbdf))); + let interrupt = Arc::new(VirtioInterruptMsix::new( + msix_config.clone(), + self.common_config.msix_config.clone(), + self.common_config.msix_queues.clone(), + msix_vectors, + )); + + // Dropping the previous virtio_interrupt and msix_config releases the + // last references to the old MsixVectorGroup, whose Drop unregisters + // the irqfds, removes the GSI routes from the routing table and frees + // the GSIs. + self.virtio_interrupt = Some(interrupt); + self.msix_config = msix_config; + + // Flush the updated routing table + if let Err(err) = vm.set_gsi_routes() { + error!("Failed to update GSI routes after MSI-X reset: {err:?}"); + } + + // Clear the MSI-X Enable and Function Mask bits in the guest-visible + // PCI capability so config-space reads stay consistent with the + // freshly-reset MsixConfig. + let reg_idx = self.msix_config_cap_offset / 4; + let msg_ctl = (self.configuration.read_reg(reg_idx) >> 16) as u16; + let msg_ctl = msg_ctl & !((1 << 15) | (1 << 14)); // clear Enable + Function Mask + self.configuration + .write_config_register(reg_idx, 2, &msg_ctl.to_le_bytes()); + } + pub fn state(&self) -> VirtioPciDeviceState { VirtioPciDeviceState { sbdf: self.sbdf, @@ -973,9 +1035,8 @@ impl PciDevice for VirtioPciDevice { // Device has been reset by the driver if self.device_activated.load(Ordering::SeqCst) && self.is_driver_init() { - let mut device = self.device.lock().unwrap(); - if device.reset() { - self.virtio_interrupt = None; + let reset_succeeded = self.device.lock().unwrap().reset(); + if reset_succeeded { self.device_activated.store(false, Ordering::SeqCst); // Reset queue readiness (changes queue_enable), queue sizes @@ -984,6 +1045,8 @@ impl PciDevice for VirtioPciDevice { self.common_config.queue_select = 0; self.common_config.device_feature_select = 0; self.common_config.driver_feature_select = 0; + + self.reset_msix(); } else { error!("Attempt to reset device when not implemented in underlying device"); // The virtio spec does not specify what to do if reset fails. From 9248ad0001ec5ded2dd77029427bcca93e76f6fe Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Wed, 13 May 2026 17:52:13 +0100 Subject: [PATCH 06/28] virtio: mmio: Add unit test for device reset failure Add test_bus_device_reset_failure to verify that when device.reset() returns false, the FAILED bit is set in device_status. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/transport/mmio.rs | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index a6eb01e039b..4db2a356da8 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -492,6 +492,7 @@ pub(crate) mod tests { device_activated: bool, config_bytes: [u8; 0xeff], activate_should_error: bool, + reset_should_fail: bool, } impl DummyDevice { @@ -508,6 +509,7 @@ pub(crate) mod tests { device_activated: false, config_bytes: [0; 0xeff], activate_should_error: false, + reset_should_fail: false, } } @@ -588,6 +590,9 @@ pub(crate) mod tests { } fn reset(&mut self) -> bool { + if self.reset_should_fail { + return false; + } self.device_activated = false; self.acked_features = 0; true @@ -1143,6 +1148,26 @@ pub(crate) mod tests { assert_eq!(d.locked_device().acked_features(), 0); } + #[test] + fn test_bus_device_reset_failure() { + let m = single_region_mem(0x1000); + let interrupt = Arc::new(IrqTrigger::new()); + let device = DummyDevice { + reset_should_fail: true, + ..DummyDevice::new() + }; + let mut d = MmioTransport::new(m, interrupt, Arc::new(Mutex::new(device)), false); + + activate_device(&mut d); + assert!(d.locked_device().is_activated()); + + // A backend that doesn't support reset must set FAILED. + let mut buf = [0; 4]; + write_le_u32(&mut buf[..], 0x0); + d.write(0x0, 0x70, &buf[..]); + assert_ne!(d.device_status & device_status::FAILED, 0); + } + #[test] fn test_get_avail_features() { let dummy_dev = DummyDevice::new(); From ef74639ff8b2a81b36d69a4d96d88016122f8182 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Fri, 15 May 2026 16:14:07 +0100 Subject: [PATCH 07/28] virtio: Add deactivate() method to VirtioDevice trait Add a deactivate() method that sets the device state to Inactive. This will be used by the generic reset implementation in a subsequent patch. Signed-off-by: Ilias Stamatis --- src/vmm/src/device_manager/mmio.rs | 2 ++ src/vmm/src/devices/virtio/balloon/device.rs | 4 ++++ src/vmm/src/devices/virtio/block/device.rs | 7 +++++++ src/vmm/src/devices/virtio/block/vhost_user/device.rs | 4 ++++ src/vmm/src/devices/virtio/block/virtio/device.rs | 4 ++++ src/vmm/src/devices/virtio/device.rs | 7 +++++++ src/vmm/src/devices/virtio/mem/device.rs | 4 ++++ src/vmm/src/devices/virtio/net/device.rs | 4 ++++ src/vmm/src/devices/virtio/pmem/device.rs | 4 ++++ src/vmm/src/devices/virtio/rng/device.rs | 4 ++++ src/vmm/src/devices/virtio/transport/mmio.rs | 4 ++++ src/vmm/src/devices/virtio/vsock/device.rs | 4 ++++ 12 files changed, 52 insertions(+) diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs index 78b7bbd8c44..c6a7c6bb309 100644 --- a/src/vmm/src/device_manager/mmio.rs +++ b/src/vmm/src/device_manager/mmio.rs @@ -592,6 +592,8 @@ pub(crate) mod tests { fn is_activated(&self) -> bool { false } + + fn deactivate(&mut self) {} } #[test] diff --git a/src/vmm/src/devices/virtio/balloon/device.rs b/src/vmm/src/devices/virtio/balloon/device.rs index 7c68f2bd40e..f98aa7c8593 100644 --- a/src/vmm/src/devices/virtio/balloon/device.rs +++ b/src/vmm/src/devices/virtio/balloon/device.rs @@ -976,6 +976,10 @@ impl VirtioDevice for Balloon { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + fn kick(&mut self) { if self.is_activated() { if self.free_page_hinting() { diff --git a/src/vmm/src/devices/virtio/block/device.rs b/src/vmm/src/devices/virtio/block/device.rs index 74e5c7ecda7..a2fdec80bbe 100644 --- a/src/vmm/src/devices/virtio/block/device.rs +++ b/src/vmm/src/devices/virtio/block/device.rs @@ -207,6 +207,13 @@ impl VirtioDevice for Block { } } + fn deactivate(&mut self) { + match self { + Self::Virtio(b) => b.deactivate(), + Self::VhostUser(b) => b.deactivate(), + } + } + fn prepare_save(&mut self) { match self { Self::Virtio(b) => b.prepare_save(), diff --git a/src/vmm/src/devices/virtio/block/vhost_user/device.rs b/src/vmm/src/devices/virtio/block/vhost_user/device.rs index 5f16d965954..0b3868bfc1d 100644 --- a/src/vmm/src/devices/virtio/block/vhost_user/device.rs +++ b/src/vmm/src/devices/virtio/block/vhost_user/device.rs @@ -375,6 +375,10 @@ where fn is_activated(&self) -> bool { self.device_state.is_activated() } + + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } } #[cfg(test)] diff --git a/src/vmm/src/devices/virtio/block/virtio/device.rs b/src/vmm/src/devices/virtio/block/virtio/device.rs index afdfe6ef812..3b310712b16 100644 --- a/src/vmm/src/devices/virtio/block/virtio/device.rs +++ b/src/vmm/src/devices/virtio/block/virtio/device.rs @@ -670,6 +670,10 @@ impl VirtioDevice for VirtioBlock { fn is_activated(&self) -> bool { self.device_state.is_activated() } + + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } } impl Drop for VirtioBlock { diff --git a/src/vmm/src/devices/virtio/device.rs b/src/vmm/src/devices/virtio/device.rs index 13d8b35b36d..454e9837424 100644 --- a/src/vmm/src/devices/virtio/device.rs +++ b/src/vmm/src/devices/virtio/device.rs @@ -194,6 +194,9 @@ pub trait VirtioDevice: AsAny + MutEventSubscriber + Send { /// Checks if the resources of this device are activated. fn is_activated(&self) -> bool; + /// Set the device state to Inactive + fn deactivate(&mut self); + /// Reset the device. Returns true on success, false otherwise. fn reset(&mut self) -> bool { false @@ -326,6 +329,10 @@ pub(crate) mod tests { fn is_activated(&self) -> bool { todo!() } + + fn deactivate(&mut self) { + todo!() + } } #[test] diff --git a/src/vmm/src/devices/virtio/mem/device.rs b/src/vmm/src/devices/virtio/mem/device.rs index fe02479adef..41690596cf5 100644 --- a/src/vmm/src/devices/virtio/mem/device.rs +++ b/src/vmm/src/devices/virtio/mem/device.rs @@ -649,6 +649,10 @@ impl VirtioDevice for VirtioMem { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + fn activate( &mut self, mem: GuestMemoryMmap, diff --git a/src/vmm/src/devices/virtio/net/device.rs b/src/vmm/src/devices/virtio/net/device.rs index 2fb6bfaf3ea..36de4047057 100644 --- a/src/vmm/src/devices/virtio/net/device.rs +++ b/src/vmm/src/devices/virtio/net/device.rs @@ -1072,6 +1072,10 @@ impl VirtioDevice for Net { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + /// Prepare saving state fn prepare_save(&mut self) { // We shouldn't be messing with the queue if the device is not activated. diff --git a/src/vmm/src/devices/virtio/pmem/device.rs b/src/vmm/src/devices/virtio/pmem/device.rs index c81bb75d19f..81acfc3dc9f 100644 --- a/src/vmm/src/devices/virtio/pmem/device.rs +++ b/src/vmm/src/devices/virtio/pmem/device.rs @@ -579,6 +579,10 @@ impl VirtioDevice for Pmem { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + fn kick(&mut self) { if self.is_activated() { info!("kick pmem {}.", self.config.id); diff --git a/src/vmm/src/devices/virtio/rng/device.rs b/src/vmm/src/devices/virtio/rng/device.rs index cc98fb8b592..a662e2de8ca 100644 --- a/src/vmm/src/devices/virtio/rng/device.rs +++ b/src/vmm/src/devices/virtio/rng/device.rs @@ -309,6 +309,10 @@ impl VirtioDevice for Entropy { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + fn activate( &mut self, mem: GuestMemoryMmap, diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index 4db2a356da8..1c6b53336d9 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -589,6 +589,10 @@ pub(crate) mod tests { self.device_activated } + fn deactivate(&mut self) { + self.device_activated = false; + } + fn reset(&mut self) -> bool { if self.reset_should_fail { return false; diff --git a/src/vmm/src/devices/virtio/vsock/device.rs b/src/vmm/src/devices/virtio/vsock/device.rs index ac022ee02cf..a4d3429ca34 100644 --- a/src/vmm/src/devices/virtio/vsock/device.rs +++ b/src/vmm/src/devices/virtio/vsock/device.rs @@ -396,6 +396,10 @@ where self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + fn kick(&mut self) { if self.is_activated() { self.pending_event_ack = true; From 391f1ee43b273a7129e9fe212e57c2a9a4f189c2 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 25 Jun 2026 14:42:37 +0100 Subject: [PATCH 08/28] virtio: Notify queues after device activation There is a race condition where a guest may add buffers to a queue and ring the notification doorbell as soon as it sets DRIVER_OK, before Firecracker has finished activating the device. If the notification eventfd is signalled while the device is still in the inactive state, the device's event handler discards it as spurious. At the moment the handler doesn't consume the event, so the handler will re-run again and again until the device is finally activated. A subsequent commit will drain the eventfds when a spurious notification arrives. That means that the notification will be lost: the buffers the guest made available sit in the queue and are never processed. Notify the queue eventfds once activation completes so the device processes any buffers that were made available during the activation window. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/transport/mmio.rs | 6 ++++++ .../devices/virtio/transport/pci/device.rs | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index 1c6b53336d9..f622120534d 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -209,6 +209,12 @@ impl MmioTransport { let _ = self.interrupt.trigger(VirtioInterruptType::Config); error!("Failed to activate virtio device: {}", err) + } else { + // A queue notification may have arrived after the guest set DRIVER_OK but + // before the device finished activating, in which case it was discarded as + // spurious. Re-notify the queues so any buffers the guest already made + // available get processed. + locked_device.notify_queue_events(); } } } diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index fc993a29535..1df0c895572 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -1015,13 +1015,18 @@ impl PciDevice for VirtioPciDevice { if self.needs_activation() { debug!("Activating device"); let interrupt = Arc::clone(self.virtio_interrupt.as_ref().unwrap()); - match self - .virtio_device() - .lock() - .unwrap() - .activate(self.memory.clone(), interrupt.clone()) - { - Ok(()) => self.device_activated.store(true, Ordering::SeqCst), + let device = self.virtio_device(); + let mut locked_device = device.lock().unwrap(); + match locked_device.activate(self.memory.clone(), interrupt.clone()) { + Ok(()) => { + self.device_activated.store(true, Ordering::SeqCst); + + // A queue notification may have arrived after the guest set DRIVER_OK but + // before the device finished activating, in which case it was discarded as + // spurious. Re-notify the queues so any buffers the guest already made + // available get processed. + locked_device.notify_queue_events(); + } Err(err) => { self.common_config.driver_status |= DEVICE_NEEDS_RESET; error!("Error activating device: {err:?}"); From 6312d0c79f31512c25d8060a1cb11a86e19436c2 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Tue, 23 Jun 2026 17:20:09 +0100 Subject: [PATCH 09/28] virtio: Consume eventfds while the device is deactivated The notification eventfds stay registered with the event manager and with KVM across a device reset. As a result a notification that arrives while the device is deactivated - for example a guest ringing a doorbell during the reset/re-init window - would be reported by the event manager over and over, spinning the event loop until the device is activated again. Drain the queue (and other) events in the not-activated branch of each device's event handler so the pending state is cleared and the spurious notification is discarded. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/balloon/event_handler.rs | 1 + .../src/devices/virtio/block/virtio/event_handler.rs | 12 ++++++++++++ src/vmm/src/devices/virtio/device.rs | 9 +++++++++ src/vmm/src/devices/virtio/mem/event_handler.rs | 1 + src/vmm/src/devices/virtio/net/event_handler.rs | 10 ++++++++++ src/vmm/src/devices/virtio/pmem/event_handler.rs | 7 +++++++ src/vmm/src/devices/virtio/rng/device.rs | 2 +- src/vmm/src/devices/virtio/rng/event_handler.rs | 7 +++++++ src/vmm/src/devices/virtio/vsock/event_handler.rs | 1 + 9 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/devices/virtio/balloon/event_handler.rs b/src/vmm/src/devices/virtio/balloon/event_handler.rs index a89cfe8477d..54bb8d1fc2e 100644 --- a/src/vmm/src/devices/virtio/balloon/event_handler.rs +++ b/src/vmm/src/devices/virtio/balloon/event_handler.rs @@ -143,6 +143,7 @@ impl MutEventSubscriber for Balloon { "Balloon: The device is not yet activated. Spurious event received: {:?}", source ); + self.drain_queue_events(); } } diff --git a/src/vmm/src/devices/virtio/block/virtio/event_handler.rs b/src/vmm/src/devices/virtio/block/virtio/event_handler.rs index 9f02862f814..3db09166978 100644 --- a/src/vmm/src/devices/virtio/block/virtio/event_handler.rs +++ b/src/vmm/src/devices/virtio/block/virtio/event_handler.rs @@ -95,6 +95,18 @@ impl MutEventSubscriber for VirtioBlock { "Block: The device is not yet activated. Spurious event received: {:?}", source ); + match source { + Self::PROCESS_QUEUE => self.drain_queue_events(), + Self::PROCESS_RATE_LIMITER => { + self.rate_limiter.event_handler(); + } + Self::PROCESS_ASYNC_COMPLETION => { + if let FileEngine::Async(ref engine) = self.disk.file_engine { + engine.completion_evt().read(); + } + } + _ => (), + } } } diff --git a/src/vmm/src/devices/virtio/device.rs b/src/vmm/src/devices/virtio/device.rs index 454e9837424..4bb2816ff99 100644 --- a/src/vmm/src/devices/virtio/device.rs +++ b/src/vmm/src/devices/virtio/device.rs @@ -226,6 +226,15 @@ pub trait VirtioDevice: AsAny + MutEventSubscriber + Send { } } + /// Drain all queue notification eventfds, discarding any pending + /// notifications. This is used if a notification arrives while a device + /// is being reset and before it's activated again. + fn drain_queue_events(&self) { + for event in self.queue_events() { + event.read(); + } + } + /// Kick the device, as if it had received external events. fn kick(&mut self) { if self.is_activated() { diff --git a/src/vmm/src/devices/virtio/mem/event_handler.rs b/src/vmm/src/devices/virtio/mem/event_handler.rs index ae81ef12a64..8b3a71845c1 100644 --- a/src/vmm/src/devices/virtio/mem/event_handler.rs +++ b/src/vmm/src/devices/virtio/mem/event_handler.rs @@ -76,6 +76,7 @@ impl MutEventSubscriber for VirtioMem { if !self.is_activated() { warn!("virtio-mem: The device is not activated yet. Spurious event received: {source}"); + self.drain_queue_events(); return; } diff --git a/src/vmm/src/devices/virtio/net/event_handler.rs b/src/vmm/src/devices/virtio/net/event_handler.rs index 9d8c09a45f2..3fe80fef197 100644 --- a/src/vmm/src/devices/virtio/net/event_handler.rs +++ b/src/vmm/src/devices/virtio/net/event_handler.rs @@ -114,6 +114,16 @@ impl MutEventSubscriber for Net { "Net: The device is not yet activated. Spurious event received: {:?}", source ); + match source { + Self::PROCESS_VIRTQ_RX | Self::PROCESS_VIRTQ_TX => self.drain_queue_events(), + Self::PROCESS_RX_RATE_LIMITER => { + self.rx_rate_limiter.event_handler(); + } + Self::PROCESS_TX_RATE_LIMITER => { + self.tx_rate_limiter.event_handler(); + } + _ => (), + } } } diff --git a/src/vmm/src/devices/virtio/pmem/event_handler.rs b/src/vmm/src/devices/virtio/pmem/event_handler.rs index 2324c509a69..6c0d239927b 100644 --- a/src/vmm/src/devices/virtio/pmem/event_handler.rs +++ b/src/vmm/src/devices/virtio/pmem/event_handler.rs @@ -78,6 +78,13 @@ impl MutEventSubscriber for Pmem { if !self.is_activated() { warn!("pmem: The device is not activated yet. Spurious event received from {source}"); + match source { + Self::PROCESS_PMEM_QUEUE => self.drain_queue_events(), + Self::PROCESS_RATE_LIMITER => { + self.rate_limiter.event_handler(); + } + _ => (), + } return; } diff --git a/src/vmm/src/devices/virtio/rng/device.rs b/src/vmm/src/devices/virtio/rng/device.rs index a662e2de8ca..acc23660789 100644 --- a/src/vmm/src/devices/virtio/rng/device.rs +++ b/src/vmm/src/devices/virtio/rng/device.rs @@ -59,7 +59,7 @@ pub struct Entropy { queue_events: Vec, // Device specific fields - rate_limiter: RateLimiter, + pub(crate) rate_limiter: RateLimiter, buffer: IoVecBufferMut, } diff --git a/src/vmm/src/devices/virtio/rng/event_handler.rs b/src/vmm/src/devices/virtio/rng/event_handler.rs index dffda5d8845..fc95f8d9dde 100644 --- a/src/vmm/src/devices/virtio/rng/event_handler.rs +++ b/src/vmm/src/devices/virtio/rng/event_handler.rs @@ -83,6 +83,13 @@ impl MutEventSubscriber for Entropy { if !self.is_activated() { warn!("entropy: The device is not activated yet. Spurious event received: {source}"); + match source { + Self::PROCESS_ENTROPY_QUEUE => self.drain_queue_events(), + Self::PROCESS_RATE_LIMITER => { + self.rate_limiter.event_handler(); + } + _ => (), + } return; } diff --git a/src/vmm/src/devices/virtio/vsock/event_handler.rs b/src/vmm/src/devices/virtio/vsock/event_handler.rs index 2a1a556a708..5da6fd70d24 100644 --- a/src/vmm/src/devices/virtio/vsock/event_handler.rs +++ b/src/vmm/src/devices/virtio/vsock/event_handler.rs @@ -221,6 +221,7 @@ where "Vsock: The device is not yet activated. Spurious event received: {:?}", source ); + self.drain_queue_events(); } } From 7cfaf2fab6f5c5d33816933d98f34b0e8e1b640e Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Fri, 15 May 2026 16:16:22 +0100 Subject: [PATCH 10/28] virtio: Move queue reset from transport to VirtioDevice::reset() Move the queue reset logic from the MMIO and PCI transport code into the default reset() implementation in the VirtioDevice trait. This is generic virtio state that should be reset for all devices, regardless of transport. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/device.rs | 3 +++ src/vmm/src/devices/virtio/queue.rs | 13 ------------- src/vmm/src/devices/virtio/transport/mmio.rs | 3 --- src/vmm/src/devices/virtio/transport/pci/device.rs | 3 --- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/src/vmm/src/devices/virtio/device.rs b/src/vmm/src/devices/virtio/device.rs index 4bb2816ff99..dbdc2560a9c 100644 --- a/src/vmm/src/devices/virtio/device.rs +++ b/src/vmm/src/devices/virtio/device.rs @@ -199,6 +199,9 @@ pub trait VirtioDevice: AsAny + MutEventSubscriber + Send { /// Reset the device. Returns true on success, false otherwise. fn reset(&mut self) -> bool { + for queue in self.queues_mut() { + *queue = Queue::new(queue.max_size); + } false } diff --git a/src/vmm/src/devices/virtio/queue.rs b/src/vmm/src/devices/virtio/queue.rs index b4ff6514ed3..4105570220b 100644 --- a/src/vmm/src/devices/virtio/queue.rs +++ b/src/vmm/src/devices/virtio/queue.rs @@ -683,19 +683,6 @@ impl Queue { new - used_event - Wrapping(1) < new - old } - - /// Resets the Virtio Queue - pub(crate) fn reset(&mut self) { - self.ready = false; - self.size = self.max_size; - self.desc_table_address = GuestAddress(0); - self.avail_ring_address = GuestAddress(0); - self.used_ring_address = GuestAddress(0); - self.next_avail = Wrapping(0); - self.next_used = Wrapping(0); - self.num_added = Wrapping(0); - self.uses_notif_suppression = false; - } } #[cfg(kani)] diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index f622120534d..2650511a89f 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -153,9 +153,6 @@ impl MmioTransport { // . Keep interrupt_evt and queue_evts as is. There may be pending notifications in those // eventfds, but nothing will happen other than supurious wakeups. // . Do not reset config_generation and keep it monotonically increasing - for queue in self.locked_device().queues_mut() { - *queue = Queue::new(queue.max_size); - } } /// Update device status according to the state machine defined by VirtIO Spec 1.0. diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index 1df0c895572..fdc899a3309 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -1044,9 +1044,6 @@ impl PciDevice for VirtioPciDevice { if reset_succeeded { self.device_activated.store(false, Ordering::SeqCst); - // Reset queue readiness (changes queue_enable), queue sizes - // and selected_queue as per spec for reset - device.queues_mut().iter_mut().for_each(Queue::reset); self.common_config.queue_select = 0; self.common_config.device_feature_select = 0; self.common_config.driver_feature_select = 0; From d5d2cf4499ef3aff74c8fc7eb114262aaf449a60 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Fri, 15 May 2026 16:17:53 +0100 Subject: [PATCH 11/28] virtio: Complete reset() and introduce _reset() Complete the reset() implementation by deactivating the device and setting acked_features to 0. This must happen for all virtio backends. Introduce a _reset() method that is called at the beginning of reset() and can be overridden by virtio backends with backend specific reset code. Make all backends return false for now since none of them supports reset yet. Signed-off-by: Ilias Stamatis --- src/vmm/src/device_manager/mmio.rs | 4 ++++ src/vmm/src/devices/virtio/balloon/device.rs | 4 ++++ src/vmm/src/devices/virtio/block/device.rs | 7 +++++++ .../devices/virtio/block/vhost_user/device.rs | 4 ++++ .../src/devices/virtio/block/virtio/device.rs | 4 ++++ src/vmm/src/devices/virtio/device.rs | 16 +++++++++++++++- src/vmm/src/devices/virtio/mem/device.rs | 4 ++++ src/vmm/src/devices/virtio/net/device.rs | 4 ++++ src/vmm/src/devices/virtio/pmem/device.rs | 4 ++++ src/vmm/src/devices/virtio/rng/device.rs | 4 ++++ src/vmm/src/devices/virtio/transport/mmio.rs | 9 ++------- src/vmm/src/devices/virtio/vsock/device.rs | 4 ++++ 12 files changed, 60 insertions(+), 8 deletions(-) diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs index c6a7c6bb309..fca77ad8014 100644 --- a/src/vmm/src/device_manager/mmio.rs +++ b/src/vmm/src/device_manager/mmio.rs @@ -594,6 +594,10 @@ pub(crate) mod tests { } fn deactivate(&mut self) {} + + fn _reset(&mut self) -> bool { + false + } } #[test] diff --git a/src/vmm/src/devices/virtio/balloon/device.rs b/src/vmm/src/devices/virtio/balloon/device.rs index f98aa7c8593..00ac9ecbba4 100644 --- a/src/vmm/src/devices/virtio/balloon/device.rs +++ b/src/vmm/src/devices/virtio/balloon/device.rs @@ -980,6 +980,10 @@ impl VirtioDevice for Balloon { self.device_state = DeviceState::Inactive; } + fn _reset(&mut self) -> bool { + false + } + fn kick(&mut self) { if self.is_activated() { if self.free_page_hinting() { diff --git a/src/vmm/src/devices/virtio/block/device.rs b/src/vmm/src/devices/virtio/block/device.rs index a2fdec80bbe..50a8084448e 100644 --- a/src/vmm/src/devices/virtio/block/device.rs +++ b/src/vmm/src/devices/virtio/block/device.rs @@ -214,6 +214,13 @@ impl VirtioDevice for Block { } } + fn _reset(&mut self) -> bool { + match self { + Self::Virtio(b) => b._reset(), + Self::VhostUser(b) => b._reset(), + } + } + fn prepare_save(&mut self) { match self { Self::Virtio(b) => b.prepare_save(), diff --git a/src/vmm/src/devices/virtio/block/vhost_user/device.rs b/src/vmm/src/devices/virtio/block/vhost_user/device.rs index 0b3868bfc1d..61c776034e9 100644 --- a/src/vmm/src/devices/virtio/block/vhost_user/device.rs +++ b/src/vmm/src/devices/virtio/block/vhost_user/device.rs @@ -379,6 +379,10 @@ where fn deactivate(&mut self) { self.device_state = DeviceState::Inactive; } + + fn _reset(&mut self) -> bool { + false + } } #[cfg(test)] diff --git a/src/vmm/src/devices/virtio/block/virtio/device.rs b/src/vmm/src/devices/virtio/block/virtio/device.rs index 3b310712b16..c6bba98ed1a 100644 --- a/src/vmm/src/devices/virtio/block/virtio/device.rs +++ b/src/vmm/src/devices/virtio/block/virtio/device.rs @@ -674,6 +674,10 @@ impl VirtioDevice for VirtioBlock { fn deactivate(&mut self) { self.device_state = DeviceState::Inactive; } + + fn _reset(&mut self) -> bool { + false + } } impl Drop for VirtioBlock { diff --git a/src/vmm/src/devices/virtio/device.rs b/src/vmm/src/devices/virtio/device.rs index dbdc2560a9c..49bcdc33ffd 100644 --- a/src/vmm/src/devices/virtio/device.rs +++ b/src/vmm/src/devices/virtio/device.rs @@ -198,13 +198,23 @@ pub trait VirtioDevice: AsAny + MutEventSubscriber + Send { fn deactivate(&mut self); /// Reset the device. Returns true on success, false otherwise. + /// It must not be overridden. fn reset(&mut self) -> bool { + if !self._reset() { + return false; + } + self.deactivate(); + self.set_acked_features(0); for queue in self.queues_mut() { *queue = Queue::new(queue.max_size); } - false + true } + /// Backend-specific reset logic. Returns true on success, false if the + /// backend does not support reset. + fn _reset(&mut self) -> bool; + /// Mark pages used by queues as dirty. fn mark_queue_memory_dirty(&mut self, mem: &GuestMemoryMmap) -> Result<(), QueueError> { for queue in self.queues_mut() { @@ -345,6 +355,10 @@ pub(crate) mod tests { fn deactivate(&mut self) { todo!() } + + fn _reset(&mut self) -> bool { + todo!() + } } #[test] diff --git a/src/vmm/src/devices/virtio/mem/device.rs b/src/vmm/src/devices/virtio/mem/device.rs index 41690596cf5..324e4261c08 100644 --- a/src/vmm/src/devices/virtio/mem/device.rs +++ b/src/vmm/src/devices/virtio/mem/device.rs @@ -653,6 +653,10 @@ impl VirtioDevice for VirtioMem { self.device_state = DeviceState::Inactive; } + fn _reset(&mut self) -> bool { + false + } + fn activate( &mut self, mem: GuestMemoryMmap, diff --git a/src/vmm/src/devices/virtio/net/device.rs b/src/vmm/src/devices/virtio/net/device.rs index 36de4047057..c8d679d268f 100644 --- a/src/vmm/src/devices/virtio/net/device.rs +++ b/src/vmm/src/devices/virtio/net/device.rs @@ -1076,6 +1076,10 @@ impl VirtioDevice for Net { self.device_state = DeviceState::Inactive; } + fn _reset(&mut self) -> bool { + false + } + /// Prepare saving state fn prepare_save(&mut self) { // We shouldn't be messing with the queue if the device is not activated. diff --git a/src/vmm/src/devices/virtio/pmem/device.rs b/src/vmm/src/devices/virtio/pmem/device.rs index 81acfc3dc9f..6b67d7ab71c 100644 --- a/src/vmm/src/devices/virtio/pmem/device.rs +++ b/src/vmm/src/devices/virtio/pmem/device.rs @@ -583,6 +583,10 @@ impl VirtioDevice for Pmem { self.device_state = DeviceState::Inactive; } + fn _reset(&mut self) -> bool { + false + } + fn kick(&mut self) { if self.is_activated() { info!("kick pmem {}.", self.config.id); diff --git a/src/vmm/src/devices/virtio/rng/device.rs b/src/vmm/src/devices/virtio/rng/device.rs index acc23660789..1897ee97957 100644 --- a/src/vmm/src/devices/virtio/rng/device.rs +++ b/src/vmm/src/devices/virtio/rng/device.rs @@ -313,6 +313,10 @@ impl VirtioDevice for Entropy { self.device_state = DeviceState::Inactive; } + fn _reset(&mut self) -> bool { + false + } + fn activate( &mut self, mem: GuestMemoryMmap, diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index 2650511a89f..a2eae50a5d5 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -596,13 +596,8 @@ pub(crate) mod tests { self.device_activated = false; } - fn reset(&mut self) -> bool { - if self.reset_should_fail { - return false; - } - self.device_activated = false; - self.acked_features = 0; - true + fn _reset(&mut self) -> bool { + !self.reset_should_fail } } diff --git a/src/vmm/src/devices/virtio/vsock/device.rs b/src/vmm/src/devices/virtio/vsock/device.rs index a4d3429ca34..ad1f1186616 100644 --- a/src/vmm/src/devices/virtio/vsock/device.rs +++ b/src/vmm/src/devices/virtio/vsock/device.rs @@ -400,6 +400,10 @@ where self.device_state = DeviceState::Inactive; } + fn _reset(&mut self) -> bool { + false + } + fn kick(&mut self) { if self.is_activated() { self.pending_event_ack = true; From 48fc4406f7ad5acd02cd812ee0ab364ceaef431d Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Wed, 13 May 2026 17:28:05 +0100 Subject: [PATCH 12/28] virtio: net: Add a RxBuffers::clear() method Add a clear() method to RxBuffers that resets all fields to their initial state. This will be used by the virtio-net reset implementation. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/net/device.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/vmm/src/devices/virtio/net/device.rs b/src/vmm/src/devices/virtio/net/device.rs index c8d679d268f..cce9d2ffbc4 100644 --- a/src/vmm/src/devices/virtio/net/device.rs +++ b/src/vmm/src/devices/virtio/net/device.rs @@ -135,6 +135,15 @@ impl RxBuffers { }) } + /// Reset the RX buffers to their initial state. + fn clear(&mut self) { + self.iovec.clear(); + self.parsed_descriptors.clear(); + self.used_descriptors = 0; + self.used_bytes = 0; + self.min_buffer_size = 0; + } + /// Add a new `DescriptorChain` that we received from the RX queue in the buffer. /// /// SAFETY: The `DescriptorChain` cannot be referencing the same memory location as any other From 96313eb24d404e58c8384d0753d5abac166ff188 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Wed, 13 May 2026 17:28:11 +0100 Subject: [PATCH 13/28] virtio: net: Implement device reset Implement the _reset() method for the virtio-net device, resetting the net specific state in-place. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/net/device.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/devices/virtio/net/device.rs b/src/vmm/src/devices/virtio/net/device.rs index cce9d2ffbc4..15831d06850 100644 --- a/src/vmm/src/devices/virtio/net/device.rs +++ b/src/vmm/src/devices/virtio/net/device.rs @@ -1086,7 +1086,9 @@ impl VirtioDevice for Net { } fn _reset(&mut self) -> bool { - false + self.rx_buffer.clear(); + self.tx_buffer.clear(); + true } /// Prepare saving state @@ -2606,4 +2608,16 @@ pub mod tests { assert!(queues[RX_INDEX].uses_notif_suppression); assert!(queues[TX_INDEX].uses_notif_suppression); } + + #[test] + fn test_reset() { + let mem = single_region_mem(2 * MAX_BUFFER_SIZE); + let mut th = TestHelper::get_default(&mem); + th.activate_net(); + + assert!(th.net().is_activated()); + assert!(th.net().reset()); + assert!(!th.net().is_activated()); + assert_eq!(th.net().acked_features(), 0); + } } From 36668035e7590182e8253b5d18386199818f8df5 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Wed, 13 May 2026 17:28:19 +0100 Subject: [PATCH 14/28] tests: Add integration test for virtio-net device reset Add a test that verifies virtio-net device reset works end-to-end by unbinding and rebinding the guest driver and checking the device remains functional and MMDS is still reachable. Suggested-by: Adam Jensen Signed-off-by: Ilias Stamatis --- .../integration_tests/functional/test_net.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/integration_tests/functional/test_net.py b/tests/integration_tests/functional/test_net.py index c3e6c3fbe8f..9dade56d156 100644 --- a/tests/integration_tests/functional/test_net.py +++ b/tests/integration_tests/functional/test_net.py @@ -11,6 +11,13 @@ import host_tools.network as net_tools from framework import utils from framework.artifacts import GUEST_KERNEL_DEFAULT, pin_guest_kernel +from framework.utils import ( + configure_mmds, + generate_mmds_get_request, + generate_mmds_session_token, +) + +MMDS_IPV4_ADDR = "169.254.169.254" # The iperf version to run this tests with IPERF_BINARY = "iperf3" @@ -171,3 +178,63 @@ def test_tap_mtu_advertised_to_guest(uvm): f"{iface_name} (guest: {guest_if}): VIRTIO_NET_F_MTU (bit {VIRTIO_NET_F_MTU_BIT})" f" not set in negotiated features: {features!r}" ) + + +def test_device_reset(uvm): + """ + Test that virtio-net device reset works. + + MMDS is configured on the reset interface and exercised before and after + the reset. + """ + vm = uvm + vm.spawn() + vm.basic_config() + vm.add_net_iface() # eth0 - used for SSH + iface2 = vm.add_net_iface() # eth1 - will be reset + configure_mmds(vm, iface_ids=["eth1"], version="V2") + vm.api.mmds.put(**{"key": "value"}) + vm.start() + + guest_ip2 = iface2.guest_ip + host_ip2 = iface2.host_ip + virtio_dev = vm.ssh.check_output( + "basename $(readlink /sys/class/net/eth1/device)" + ).stdout.strip() + + def configure_eth1(): + vm.ssh.check_output( + f"ip addr flush dev eth1 && " + f"ip addr add {guest_ip2}/30 dev eth1 && " + f"ip link set eth1 up && " + f"ip route replace {MMDS_IPV4_ADDR} dev eth1" + ) + + def check_mmds(): + token = generate_mmds_session_token(vm.ssh, MMDS_IPV4_ADDR, token_ttl=60) + cmd = ( + generate_mmds_get_request(MMDS_IPV4_ADDR, token=token, app_json=False) + + "key" + ) + _, stdout, _ = vm.ssh.check_output(cmd) + assert stdout == "value" + + configure_eth1() + vm.ssh.check_output(f"ping -c 1 {host_ip2}") + check_mmds() + + # Reset eth1 by unbinding and rebinding its virtio driver. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_net/unbind" + ) + + # Verify that ping fails after unbind. + ret = vm.ssh.run(f"ping -c 1 -W 1 {host_ip2}") + assert ret.returncode != 0 + + # Re-bind and check again + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_net/bind") + + configure_eth1() + vm.ssh.check_output(f"ping -c 1 {host_ip2}") + check_mmds() From d57cc1febfdd4714833472dc9d9dc20269d429f5 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 15:25:51 +0100 Subject: [PATCH 15/28] virtio: block: Implement device reset Implement the _reset() method for the virtio-block device, resetting the device-specific state in-place. For the async io_uring engine drain the ring so that any in-flight operations complete. Adjust the seccomp filter such that vCPU threads can call io_uring_enter. This only adds support for the Virtio backend for now and not VhostUser. Signed-off-by: Ilias Stamatis --- resources/seccomp/aarch64-unknown-linux-musl.json | 4 ++++ resources/seccomp/x86_64-unknown-linux-musl.json | 4 ++++ src/vmm/src/devices/virtio/block/virtio/device.rs | 7 ++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/resources/seccomp/aarch64-unknown-linux-musl.json b/resources/seccomp/aarch64-unknown-linux-musl.json index f6561c3f2ed..fe2c7ea09f2 100644 --- a/resources/seccomp/aarch64-unknown-linux-musl.json +++ b/resources/seccomp/aarch64-unknown-linux-musl.json @@ -1033,6 +1033,10 @@ "syscall": "eventfd2", "comment": "Needed for recreating the MSI-X vector eventfds when a PCI device is reset by the guest (MsixVectorGroup recreated from the vCPU thread)" }, + { + "syscall": "io_uring_enter", + "comment": "Needed to drain the async block IO engine's io_uring when the device is reset by the guest (from the vCPU thread)" + }, { "syscall": "fstat", "comment": "Used for reading the local timezone from /etc/localtime" diff --git a/resources/seccomp/x86_64-unknown-linux-musl.json b/resources/seccomp/x86_64-unknown-linux-musl.json index e1326800798..4f8f7187491 100644 --- a/resources/seccomp/x86_64-unknown-linux-musl.json +++ b/resources/seccomp/x86_64-unknown-linux-musl.json @@ -1045,6 +1045,10 @@ "syscall": "eventfd2", "comment": "Needed for recreating the MSI-X vector eventfds when a PCI device is reset by the guest (MsixVectorGroup recreated from the vCPU thread)" }, + { + "syscall": "io_uring_enter", + "comment": "Needed to drain the async block IO engine's io_uring when the device is reset by the guest (from the vCPU thread)" + }, { "syscall": "fstat", "comment": "Used for reading the local timezone from /etc/localtime" diff --git a/src/vmm/src/devices/virtio/block/virtio/device.rs b/src/vmm/src/devices/virtio/block/virtio/device.rs index c6bba98ed1a..eba1b3e0f70 100644 --- a/src/vmm/src/devices/virtio/block/virtio/device.rs +++ b/src/vmm/src/devices/virtio/block/virtio/device.rs @@ -676,7 +676,12 @@ impl VirtioDevice for VirtioBlock { } fn _reset(&mut self) -> bool { - false + if let Err(err) = self.disk.file_engine.drain(true) { + error!("Failed to reset block IO engine: {:?}", err); + return false; + } + self.is_io_engine_throttled = false; + true } } From eee0b22431440154bd7deb9f1b84245a27ff25ec Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 15:25:58 +0100 Subject: [PATCH 16/28] tests: Add integration test for virtio-block device reset Add a test that verifies virtio-block device reset works end-to-end by unbinding and rebinding the guest driver for a scratch block device and checking the device remains functional. Signed-off-by: Ilias Stamatis --- .../functional/test_drive_virtio.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/integration_tests/functional/test_drive_virtio.py b/tests/integration_tests/functional/test_drive_virtio.py index 6830dad3bd1..bc792ffba9e 100644 --- a/tests/integration_tests/functional/test_drive_virtio.py +++ b/tests/integration_tests/functional/test_drive_virtio.py @@ -386,3 +386,38 @@ def _check_mount(ssh_connection, dev_path): assert stderr == "" _, _, stderr = ssh_connection.run("umount /tmp", timeout=30.0) assert stderr == "" + + +def test_device_reset(uvm, io_engine): + """ + Test that virtio-block device reset works. + """ + vm = uvm + vm.spawn() + vm.basic_config() + vm.add_net_iface() + + fs = drive_tools.FilesystemFile(os.path.join(vm.fsfiles, "scratch"), size=2) + vm.add_drive("scratch", fs.path, io_engine=io_engine) + vm.start() + + # Verify the scratch drive is accessible. + vm.ssh.check_output("mount /dev/vdb /tmp && umount /tmp") + + # Find the virtio device backing vdb and unbind it. + virtio_dev = vm.ssh.check_output( + "basename $(readlink /sys/block/vdb/device)" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_blk/unbind" + ) + + # Verify the drive is gone. + ret = vm.ssh.run("ls /dev/vdb") + assert ret.returncode != 0 + + # Rebind and verify the drive is back. + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_blk/bind") + vm.ssh.check_output("ls /dev/vdb") + vm.ssh.check_output("mount /dev/vdb /tmp && umount /tmp") From 5317f5970734d6cadb72aa70123f61e3a9db34a8 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:15:00 +0100 Subject: [PATCH 17/28] virtio: pmem: Implement device reset Pmem does not have any backend specific state that needs resetting so implement the _reset() method by simply returning true. The rest of the state is handled by the generic reset(). Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/pmem/device.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vmm/src/devices/virtio/pmem/device.rs b/src/vmm/src/devices/virtio/pmem/device.rs index 6b67d7ab71c..783b59a22f2 100644 --- a/src/vmm/src/devices/virtio/pmem/device.rs +++ b/src/vmm/src/devices/virtio/pmem/device.rs @@ -584,7 +584,7 @@ impl VirtioDevice for Pmem { } fn _reset(&mut self) -> bool { - false + true } fn kick(&mut self) { From 0113a092300d094be10ecd2cc748a0d94069fd95 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:22:07 +0100 Subject: [PATCH 18/28] tests: Add integration test for virtio-pmem device reset Add a test that verifies virtio-pmem device reset works end-to-end by unbinding and rebinding the guest driver and checking the device remains functional. Signed-off-by: Ilias Stamatis --- .../integration_tests/functional/test_pmem.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/integration_tests/functional/test_pmem.py b/tests/integration_tests/functional/test_pmem.py index 51cd9548c5f..cf109ac17a3 100644 --- a/tests/integration_tests/functional/test_pmem.py +++ b/tests/integration_tests/functional/test_pmem.py @@ -7,6 +7,7 @@ import re import pytest +from tenacity import Retrying, stop_after_delay, wait_fixed import host_tools.drive as drive_tools from framework import utils @@ -228,3 +229,47 @@ def test_pmem_dax_memory_saving( assert ( pmem_rss_usage < block_rss_usage ), f"{block_cache_usage} <= {pmem_cache_usage}" + + +def test_device_reset(uvm): + """ + Test that virtio-pmem device reset works. + """ + vm = uvm + vm.spawn() + vm.basic_config(add_root_device=True) + vm.add_net_iface() + + fs = drive_tools.FilesystemFile(os.path.join(vm.fsfiles, "scratch"), size=2) + vm.add_pmem("pmem_scratch", fs.path, False, False) + vm.start() + + # Verify the pmem device is accessible. + vm.ssh.check_output("ls /dev/pmem0") + + virtio_dev = vm.ssh.check_output( + "basename $(realpath /sys/block/pmem0/device/../../..)" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_pmem/unbind" + ) + + # Verify the device is gone. + ret = vm.ssh.run("ls /dev/pmem0") + assert ret.returncode != 0 + + # Rebind and verify the device is functional. + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_pmem/bind") + # The NVDIMM subsystem creates the device node asynchronously after driver + # probe, so we need to wait for it. + for attempt in Retrying( + wait=wait_fixed(0.1), stop=stop_after_delay(5.0), reraise=True + ): + with attempt: + vm.ssh.check_output("ls /dev/pmem0") + vm.ssh.check_output("mount /dev/pmem0 /tmp") + vm.ssh.check_output("echo reset_test > /tmp/testfile") + ret = vm.ssh.check_output("cat /tmp/testfile") + assert "reset_test" in ret.stdout + vm.ssh.check_output("umount /tmp") From 290a5dce02de9a817f54ede38ee48651e7531a44 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:37:26 +0100 Subject: [PATCH 19/28] virtio: balloon: Implement device reset Implement the _reset() method for the virtio-balloon device, resetting the balloon specific state in-place. Do not deflate the balloon on reset. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/balloon/device.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/devices/virtio/balloon/device.rs b/src/vmm/src/devices/virtio/balloon/device.rs index 00ac9ecbba4..80588a166c7 100644 --- a/src/vmm/src/devices/virtio/balloon/device.rs +++ b/src/vmm/src/devices/virtio/balloon/device.rs @@ -981,7 +981,13 @@ impl VirtioDevice for Balloon { } fn _reset(&mut self) -> bool { - false + self.config_space.actual_pages = 0; + self.config_space.free_page_hint_cmd_id = FREE_PAGE_HINT_STOP; + self.stats_timer.arm(Duration::ZERO, None); + self.stats_desc_index = None; + self.latest_stats = BalloonStats::default(); + self.hinting_state = Default::default(); + true } fn kick(&mut self) { From f9ea2f48a7c8f5e5fa790db41e40eb5993de5f5c Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:44:33 +0100 Subject: [PATCH 20/28] tests: Add integration test for virtio-balloon device reset Add a test that verifies virtio-balloon device reset works end-to-end by unbinding and rebinding the guest driver and checking the device remains functional. Signed-off-by: Ilias Stamatis --- .../functional/test_balloon.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/integration_tests/functional/test_balloon.py b/tests/integration_tests/functional/test_balloon.py index de4d0f0d5bb..7201dd9d6a4 100644 --- a/tests/integration_tests/functional/test_balloon.py +++ b/tests/integration_tests/functional/test_balloon.py @@ -622,3 +622,60 @@ def test_memory_scrub(uvm, method): microvm.ssh.check_output("/usr/local/bin/readmem {} {}".format(60, 1)) check_guest_dmesg_for_stalls(microvm.ssh) + + +def test_device_reset(uvm): + """ + Test that virtio-balloon device reset works. + """ + vm = uvm + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.api.balloon.put(amount_mib=0, deflate_on_oom=True, stats_polling_interval_s=0) + vm.start() + + meminfo = MeminfoGuest(vm) + free_initial = meminfo.get().mem_free.kib() + + # Inflate the balloon by 64 MiB and confirm guest free memory drops. + vm.api.balloon.patch(amount_mib=64) + get_stable_rss_mem(vm) + free_inflated = meminfo.get().mem_free.kib() + # Inflating 64 MiB should reclaim at least 85% of that from guest free + # memory. The 15% slack accounts for kernel accounting overhead. + inflated_drop = 64 * 1024 * 85 // 100 + assert free_inflated <= free_initial - inflated_drop + + # Find the virtio balloon device. + virtio_dev = vm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/virtio_balloon/virtio* | xargs -n1 basename" + ).stdout.strip() + + # Reset the device by unbinding the driver. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_balloon/unbind" + ) + + # Verify the balloon is gone. + ret = vm.ssh.run("ls /sys/bus/virtio/drivers/virtio_balloon/virtio*") + assert ret.returncode != 0 + + # Rebind and make sure the device node is back. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_balloon/bind" + ) + vm.ssh.check_output("ls /sys/bus/virtio/drivers/virtio_balloon/virtio*") + + # The inflation target is preserved across reset, assert that the driver + # re-inflates the balloon. + get_stable_rss_mem(vm) + free_after_reset = meminfo.get().mem_free.kib() + assert free_after_reset <= free_initial - inflated_drop + + # Deflate to make sure the device is functional in both directions + vm.api.balloon.patch(amount_mib=0) + get_stable_rss_mem(vm) + free_deflated = meminfo.get().mem_free.kib() + DEFLATE_SLACK_KIB = 8 * 1024 # arbitrary 8 MiB tolerance for measurement noise + assert free_deflated >= free_initial - DEFLATE_SLACK_KIB From 06181045af23ae2da0bf06d746108ca3a89d3de3 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:37:33 +0100 Subject: [PATCH 21/28] virtio: entropy: Implement device reset The entropy device does not have any backend specific state that needs resetting so implement the _reset() method by simply returning true. The test_failed_reset_blocks_reinitialization() test used an entropy device assuming it doesn't support reset. Since it now does, adjust the test to check that the device is first deactivated after a reset and then the driver can perform the initialization sequence again. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/rng/device.rs | 2 +- .../devices/virtio/transport/pci/device.rs | 45 +++++-------------- 2 files changed, 13 insertions(+), 34 deletions(-) diff --git a/src/vmm/src/devices/virtio/rng/device.rs b/src/vmm/src/devices/virtio/rng/device.rs index 1897ee97957..2481a8462d1 100644 --- a/src/vmm/src/devices/virtio/rng/device.rs +++ b/src/vmm/src/devices/virtio/rng/device.rs @@ -314,7 +314,7 @@ impl VirtioDevice for Entropy { } fn _reset(&mut self) -> bool { - false + true } fn activate( diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index fdc899a3309..da65ac90755 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -1761,7 +1761,7 @@ mod tests { } #[test] - fn test_failed_reset_blocks_reinitialization() { + fn test_reset_and_reinitialization() { let mut vmm = create_vmm_with_virtio_pci_device(); let device = get_virtio_device(&vmm); let mut locked = device.lock().unwrap(); @@ -1777,41 +1777,20 @@ mod tests { assert!(locked.device_activated.load(Ordering::SeqCst)); // Write 0 to device_status to request a reset. - // Entropy's reset() returns None (unimplemented), so the reset fails. write_driver_status(&mut locked, 0); assert_eq!(read_driver_status(&mut locked), 0); - // device_activated stays true because the backend was not actually reset. - assert!(locked.device_activated.load(Ordering::SeqCst)); + // Device should be deactivated after successful reset. + assert!(!locked.device_activated.load(Ordering::SeqCst)); - // Attempt to re-initialize should be rejected because device_activated is - // still true while driver_status is INIT. + // Re-initialization should succeed after a successful reset. write_driver_status(&mut locked, ACKNOWLEDGE); - assert_eq!(read_driver_status(&mut locked), 0); - - // Save state and restore into a new device -- the combination of - // device_activated == true and driver_status == INIT is preserved in the - // snapshot, so the blocking behavior survives restore. - let saved_state = locked.state(); - drop(locked); - - // Fully drop the original device before constructing the restored copy: the restored - // copy reuses the same MSI-X GSIs, so the two `MsixVectorGroup` instances must never - // exist concurrently. This is how it will happen in real scenario anyway. - let kvm_vm = vmm.vm.as_kvm().unwrap().clone(); - let saved_allocator = kvm_vm.resource_allocator().save(); - drop(device); - drop(vmm); - // Restore allocator state through `ResourceAllocator::restore`, matching the VM restore - // path. This resets the MSI GSI allocator before the restored device replays its saved GSI - // allocations. - *kvm_vm.resource_allocator() = ResourceAllocator::restore((), &saved_allocator).unwrap(); - - let new_entropy = Arc::new(Mutex::new(Entropy::new(RateLimiter::default()).unwrap())); - let restored = - VirtioPciDevice::new_from_state("rng".to_string(), &kvm_vm, new_entropy, saved_state) - .unwrap(); - - assert!(restored.device_activated.load(Ordering::SeqCst)); - assert_eq!(restored.common_config.driver_status, 0); + assert_eq!(read_driver_status(&mut locked), ACKNOWLEDGE); + write_driver_status(&mut locked, ACKNOWLEDGE | DRIVER); + let features = read_device_features(&mut locked); + write_driver_features(&mut locked, features); + write_driver_status(&mut locked, ACKNOWLEDGE | DRIVER | FEATURES_OK); + setup_queues(&mut locked); + write_driver_status(&mut locked, ACKNOWLEDGE | DRIVER | FEATURES_OK | DRIVER_OK); + assert!(locked.device_activated.load(Ordering::SeqCst)); } } From be64d191b8a6e175f4063099c32c30af5b2f3f8c Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:44:40 +0100 Subject: [PATCH 22/28] tests: Add integration test for virtio-rng device reset Add a test that verifies virtio-rng device reset works end-to-end by unbinding and rebinding the guest driver and checking the device remains functional. Signed-off-by: Ilias Stamatis --- .../integration_tests/functional/test_rng.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration_tests/functional/test_rng.py b/tests/integration_tests/functional/test_rng.py index e7706947b65..648f545f18b 100644 --- a/tests/integration_tests/functional/test_rng.py +++ b/tests/integration_tests/functional/test_rng.py @@ -232,3 +232,35 @@ def test_rng_bw_rate_limiter(uvm_any): # Check the rate limiter using a request size equal to the size # of the token bucket. _check_entropy_rate_limited(vm.ssh, size, expected_kbps) + + +def test_device_reset(uvm): + """ + Test that virtio-rng device reset works. + """ + vm = uvm + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.api.entropy.put() + vm.start() + + # Verify the rng device works. + vm.ssh.check_output("dd if=/dev/hwrng of=/dev/null bs=32 count=1") + + # Find the virtio rng device. + virtio_dev = vm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/virtio_rng/virtio* | xargs -n1 basename" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_rng/unbind" + ) + + # Verify the rng device is gone. + ret = vm.ssh.run("ls /sys/bus/virtio/drivers/virtio_rng/virtio*") + assert ret.returncode != 0 + + # Rebind and verify it works again. + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_rng/bind") + vm.ssh.check_output("dd if=/dev/hwrng of=/dev/null bs=32 count=1") From b856877e0c623472ad5d147b7f9d3f50df562b70 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 2 Jul 2026 17:01:10 +0100 Subject: [PATCH 23/28] virtio: vsock: Register the host socket listener on activation Register the muxer's host socket epoll listener when the device is activated rather than when the muxer is constructed, by adding an activate() method to the VsockBackend trait and calling it from Vsock::activate(). This ties the listener's lifetime to device activation instead of muxer construction. This lays the groundwork for device reset, which needs the listener set to be re-established on each activation. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/mod.rs | 2 ++ src/vmm/src/devices/virtio/vsock/device.rs | 5 +++++ src/vmm/src/devices/virtio/vsock/mod.rs | 5 ++++- src/vmm/src/devices/virtio/vsock/test_utils.rs | 6 +++++- src/vmm/src/devices/virtio/vsock/unix/muxer.rs | 15 ++++++++++----- 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/vmm/src/devices/virtio/mod.rs b/src/vmm/src/devices/virtio/mod.rs index b17db5b02ac..ec95368701c 100644 --- a/src/vmm/src/devices/virtio/mod.rs +++ b/src/vmm/src/devices/virtio/mod.rs @@ -67,6 +67,8 @@ pub enum ActivateError { QueueMemoryError(QueueError), /// The driver didn't acknowledge a required feature: {0} RequiredFeatureNotAcked(&'static str), + /// Vsock backend: {0} + VsockBackend(vsock::VsockError), } /// Trait that helps in upcasting an object to Any diff --git a/src/vmm/src/devices/virtio/vsock/device.rs b/src/vmm/src/devices/virtio/vsock/device.rs index ad1f1186616..04b6e787223 100644 --- a/src/vmm/src/devices/virtio/vsock/device.rs +++ b/src/vmm/src/devices/virtio/vsock/device.rs @@ -382,6 +382,11 @@ where } } + self.backend.activate().map_err(|err| { + METRICS.activate_fails.inc(); + ActivateError::VsockBackend(err) + })?; + if self.activate_evt.write(1).is_err() { METRICS.activate_fails.inc(); return Err(ActivateError::EventFd); diff --git a/src/vmm/src/devices/virtio/vsock/mod.rs b/src/vmm/src/devices/virtio/vsock/mod.rs index c8160f33381..eb47919e17d 100644 --- a/src/vmm/src/devices/virtio/vsock/mod.rs +++ b/src/vmm/src/devices/virtio/vsock/mod.rs @@ -183,4 +183,7 @@ pub trait VsockChannel { /// The vsock backend, which is basically an epoll-event-driven vsock channel. /// Currently, the only implementation we have is `crate::devices::virtio::unix::muxer::VsockMuxer`, /// which translates guest-side vsock connections to host-side Unix domain socket connections. -pub trait VsockBackend: VsockChannel + VsockEpollListener + Send {} +pub trait VsockBackend: VsockChannel + VsockEpollListener + Send { + /// Activate the backend, adding its listeners to the poll set. + fn activate(&mut self) -> Result<(), VsockError>; +} diff --git a/src/vmm/src/devices/virtio/vsock/test_utils.rs b/src/vmm/src/devices/virtio/vsock/test_utils.rs index 4001f2b940d..196a7580378 100644 --- a/src/vmm/src/devices/virtio/vsock/test_utils.rs +++ b/src/vmm/src/devices/virtio/vsock/test_utils.rs @@ -102,7 +102,11 @@ impl VsockEpollListener for TestBackend { self.evset = Some(evset); } } -impl VsockBackend for TestBackend {} +impl VsockBackend for TestBackend { + fn activate(&mut self) -> Result<(), VsockError> { + Ok(()) + } +} #[derive(Debug)] pub struct TestContext { diff --git a/src/vmm/src/devices/virtio/vsock/unix/muxer.rs b/src/vmm/src/devices/virtio/vsock/unix/muxer.rs index 526bdb708d5..cb3678c0361 100644 --- a/src/vmm/src/devices/virtio/vsock/unix/muxer.rs +++ b/src/vmm/src/devices/virtio/vsock/unix/muxer.rs @@ -301,7 +301,13 @@ impl VsockEpollListener for VsockMuxer { } } -impl VsockBackend for VsockMuxer {} +impl VsockBackend for VsockMuxer { + fn activate(&mut self) -> Result<(), VsockError> { + // Listen on the host-initiated socket, for incoming connections. + self.add_listener(self.host_sock.as_raw_fd(), EpollListener::HostSock) + .map_err(VsockError::VsockUdsBackend) + } +} impl VsockMuxer { /// Muxer constructor. @@ -312,7 +318,7 @@ impl VsockMuxer { .and_then(|sock| sock.set_nonblocking(true).map(|_| sock)) .map_err(VsockUnixBackendError::UnixBind)?; - let mut muxer = Self { + let muxer = Self { cid, host_sock, host_sock_path, @@ -325,8 +331,6 @@ impl VsockMuxer { local_port_set: HashSet::with_capacity(defs::MAX_CONNECTIONS), }; - // Listen on the host initiated socket, for incoming connections. - muxer.add_listener(muxer.host_sock.as_raw_fd(), EpollListener::HostSock)?; Ok(muxer) } @@ -851,7 +855,8 @@ mod tests { ) .unwrap(); - let muxer = VsockMuxer::new(PEER_CID, get_file(name)).unwrap(); + let mut muxer = VsockMuxer::new(PEER_CID, get_file(name)).unwrap(); + muxer.activate().unwrap(); Self { _vsock_test_ctx: vsock_test_ctx, rx_pkt, From a20e5129f066ff8ccbdf502bdfa20fae30ab906b Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:37:41 +0100 Subject: [PATCH 24/28] virtio: vsock: Implement device reset Implement the _reset() method for the virtio-vsock device, resetting the vsock specific state in-place. Add epoll_ctl to the vCPU seccomp filter, since it's needed to remove the vsock muxer's connection and listener epoll listeners when the device is reset. Signed-off-by: Ilias Stamatis --- .../seccomp/aarch64-unknown-linux-musl.json | 4 ++++ .../seccomp/x86_64-unknown-linux-musl.json | 4 ++++ src/vmm/src/devices/virtio/vsock/device.rs | 6 +++++- src/vmm/src/devices/virtio/vsock/mod.rs | 4 ++++ src/vmm/src/devices/virtio/vsock/packet.rs | 12 ++++++++++++ src/vmm/src/devices/virtio/vsock/test_utils.rs | 2 ++ src/vmm/src/devices/virtio/vsock/unix/muxer.rs | 18 ++++++++++++++++++ 7 files changed, 49 insertions(+), 1 deletion(-) diff --git a/resources/seccomp/aarch64-unknown-linux-musl.json b/resources/seccomp/aarch64-unknown-linux-musl.json index fe2c7ea09f2..5a58fd2ccef 100644 --- a/resources/seccomp/aarch64-unknown-linux-musl.json +++ b/resources/seccomp/aarch64-unknown-linux-musl.json @@ -1037,6 +1037,10 @@ "syscall": "io_uring_enter", "comment": "Needed to drain the async block IO engine's io_uring when the device is reset by the guest (from the vCPU thread)" }, + { + "syscall": "epoll_ctl", + "comment": "Needed to remove the vsock muxer's connection and listener epoll listeners when the device is reset by the guest (from the vCPU thread)" + }, { "syscall": "fstat", "comment": "Used for reading the local timezone from /etc/localtime" diff --git a/resources/seccomp/x86_64-unknown-linux-musl.json b/resources/seccomp/x86_64-unknown-linux-musl.json index 4f8f7187491..35fae0db21b 100644 --- a/resources/seccomp/x86_64-unknown-linux-musl.json +++ b/resources/seccomp/x86_64-unknown-linux-musl.json @@ -1049,6 +1049,10 @@ "syscall": "io_uring_enter", "comment": "Needed to drain the async block IO engine's io_uring when the device is reset by the guest (from the vCPU thread)" }, + { + "syscall": "epoll_ctl", + "comment": "Needed to remove the vsock muxer's connection and listener epoll listeners when the device is reset by the guest (from the vCPU thread)" + }, { "syscall": "fstat", "comment": "Used for reading the local timezone from /etc/localtime" diff --git a/src/vmm/src/devices/virtio/vsock/device.rs b/src/vmm/src/devices/virtio/vsock/device.rs index 04b6e787223..a9706d4b491 100644 --- a/src/vmm/src/devices/virtio/vsock/device.rs +++ b/src/vmm/src/devices/virtio/vsock/device.rs @@ -406,7 +406,11 @@ where } fn _reset(&mut self) -> bool { - false + self.backend.reset(); + self.rx_packet.clear(); + self.tx_packet.clear(); + self.pending_event_ack = false; + true } fn kick(&mut self) { diff --git a/src/vmm/src/devices/virtio/vsock/mod.rs b/src/vmm/src/devices/virtio/vsock/mod.rs index eb47919e17d..d20da8d660d 100644 --- a/src/vmm/src/devices/virtio/vsock/mod.rs +++ b/src/vmm/src/devices/virtio/vsock/mod.rs @@ -186,4 +186,8 @@ pub trait VsockChannel { pub trait VsockBackend: VsockChannel + VsockEpollListener + Send { /// Activate the backend, adding its listeners to the poll set. fn activate(&mut self) -> Result<(), VsockError>; + + /// Reset the backend, dropping all active connections and removing its listeners + /// from the poll set. + fn reset(&mut self); } diff --git a/src/vmm/src/devices/virtio/vsock/packet.rs b/src/vmm/src/devices/virtio/vsock/packet.rs index 7253f41ce76..baac307e773 100644 --- a/src/vmm/src/devices/virtio/vsock/packet.rs +++ b/src/vmm/src/devices/virtio/vsock/packet.rs @@ -197,6 +197,12 @@ pub struct VsockPacketTx { } impl VsockPacketTx { + /// Clear the packet buffer. + pub fn clear(&mut self) { + self.hdr = Default::default(); + self.buffer.clear(); + } + /// Create the packet wrapper from a TX virtq chain head. /// /// ## Errors @@ -290,6 +296,12 @@ pub struct VsockPacketRx { } impl VsockPacketRx { + /// Clear the packet buffer. + pub fn clear(&mut self) { + self.hdr = Default::default(); + self.buffer.clear(); + } + /// Creates new VsockPacketRx. pub fn new() -> Result { let buffer = IoVecBufferMut::new().map_err(VsockError::IovDeque)?; diff --git a/src/vmm/src/devices/virtio/vsock/test_utils.rs b/src/vmm/src/devices/virtio/vsock/test_utils.rs index 196a7580378..8877690337a 100644 --- a/src/vmm/src/devices/virtio/vsock/test_utils.rs +++ b/src/vmm/src/devices/virtio/vsock/test_utils.rs @@ -106,6 +106,8 @@ impl VsockBackend for TestBackend { fn activate(&mut self) -> Result<(), VsockError> { Ok(()) } + + fn reset(&mut self) {} } #[derive(Debug)] diff --git a/src/vmm/src/devices/virtio/vsock/unix/muxer.rs b/src/vmm/src/devices/virtio/vsock/unix/muxer.rs index cb3678c0361..d7d0eb89d72 100644 --- a/src/vmm/src/devices/virtio/vsock/unix/muxer.rs +++ b/src/vmm/src/devices/virtio/vsock/unix/muxer.rs @@ -307,6 +307,24 @@ impl VsockBackend for VsockMuxer { self.add_listener(self.host_sock.as_raw_fd(), EpollListener::HostSock) .map_err(VsockError::VsockUdsBackend) } + + fn reset(&mut self) { + // Remove all connections and their epoll listeners. + let keys: Vec = self.conn_map.keys().copied().collect(); + for key in keys { + self.remove_connection(key); + } + + let fds: Vec = self.listener_map.keys().copied().collect(); + for fd in fds { + self.remove_listener(fd); + } + + self.rxq = MuxerRxQ::new(); + self.killq = MuxerKillQ::new(); + self.local_port_set.clear(); + self.local_port_last = (1u32 << 30) - 1; + } } impl VsockMuxer { From 88470bc372b86f0e75f6df2aee36802e27ec6e25 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:44:48 +0100 Subject: [PATCH 25/28] tests: Add integration test for virtio-vsock device reset Add a test that verifies virtio-vsock device reset works end-to-end by unbinding and rebinding the guest driver and checking the device remains functional. Signed-off-by: Ilias Stamatis --- .../functional/test_vsock.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/integration_tests/functional/test_vsock.py b/tests/integration_tests/functional/test_vsock.py index fc5afffb74c..ad264ff433e 100644 --- a/tests/integration_tests/functional/test_vsock.py +++ b/tests/integration_tests/functional/test_vsock.py @@ -538,3 +538,47 @@ def test_snapshot_restore_with_inflight_vsock_tx( new_vm.path, make_host_port_path(VSOCK_UDS_PATH, ECHO_SERVER_PORT) ) check_guest_connections(new_vm, path, vm_blob_path, blob_hash) + + +def test_device_reset(uvm): + """ + Test that virtio-vsock device reset works. + """ + vm = uvm + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.api.vsock.put(guest_cid=3, vsock_id="vsock0", uds_path="/" + VSOCK_UDS_PATH) + vm.start() + + uds_path = start_guest_echo_server(vm) + blob_path, blob_hash = make_blob(vm.path) + check_host_connections(uds_path, blob_path, blob_hash) + + # Keep a connection open across the reset. + held_conn = vsock_connect_to_guest(uds_path, ECHO_SERVER_PORT) + + # Find the virtio vsock device. + virtio_dev = vm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/virtio*" + " | xargs -n1 basename" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/unbind" + ) + + held_conn.close() + + # Verify the vsock device is gone. + ret = vm.ssh.run("ls /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/virtio*") + assert ret.returncode != 0 + + # Rebind and verify the data path works by starting a guest echo server + # and sending data from the host through the vsock. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/bind" + ) + uds_path = start_guest_echo_server(vm) + blob_path, blob_hash = make_blob(vm.path) + check_host_connections(uds_path, blob_path, blob_hash) From 1baf3f52d1fb78904c143d51abe201568eb3fccc Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Thu, 14 May 2026 17:37:48 +0100 Subject: [PATCH 26/28] virtio: mem: Implement device reset The virtio-mem device does not have any backend specific state that needs resetting so implement the _reset() method by simply returning true. Plugged memory regions remain intact. Signed-off-by: Ilias Stamatis --- src/vmm/src/devices/virtio/mem/device.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/devices/virtio/mem/device.rs b/src/vmm/src/devices/virtio/mem/device.rs index 324e4261c08..ac01f0246e8 100644 --- a/src/vmm/src/devices/virtio/mem/device.rs +++ b/src/vmm/src/devices/virtio/mem/device.rs @@ -654,7 +654,14 @@ impl VirtioDevice for VirtioMem { } fn _reset(&mut self) -> bool { - false + // Virtio spec, section 5.15.5.2: + // The device MUST NOT change the state of memory blocks during device + // reset. The device MUST NOT modify memory or memory properties of + // plugged memory blocks during device reset. + // + // Note: the Linux virtio-mem driver does not support rebinding when + // memory is plugged + true } fn activate( From eecc9243610272349b9bb80577f881076e1b4369 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Fri, 15 May 2026 15:38:14 +0100 Subject: [PATCH 27/28] tests: Add integration test for virtio-mem device reset Add a test that verifies virtio-mem device reset works end-to-end by unbinding and rebinding the guest driver and checking the device remains functional. Signed-off-by: Ilias Stamatis --- .../performance/test_hotplug_memory.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration_tests/performance/test_hotplug_memory.py b/tests/integration_tests/performance/test_hotplug_memory.py index 84b5e09f674..e481057f355 100644 --- a/tests/integration_tests/performance/test_hotplug_memory.py +++ b/tests/integration_tests/performance/test_hotplug_memory.py @@ -514,3 +514,29 @@ def test_memory_hotplug_latency( timed_memory_hotplug(uvm, 0, metrics, "hotunplug", "unplug_agg") timed_memory_hotplug(uvm, hotplug_size, metrics, "hotplug_2nd", "plug_agg") uvm.kill() + + +def test_device_reset(uvm): + """ + Test that virtio-mem device reset works. + + Note: the Linux virtio-mem driver does not support rebinding when memory is + plugged (the resource region can't be re-registered), so we reset without + any plugged memory and verify the device is functional afterwards. + """ + config = {"total_size_mib": 1024, "slot_size_mib": 128, "block_size_mib": 2} + uvm = uvm_booted_memhp(uvm, None, None, False, config, None, None, None) + + # Reset the device via driver unbind/bind. + virtio_dev = uvm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/virtio_mem/virtio* | xargs -n1 basename" + ).stdout.strip() + + uvm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_mem/unbind" + ) + uvm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_mem/bind") + + # Verify the device is functional after reset by hotplugging memory. + # check_hotplug() asserts that guest mem_total reflects the new size. + check_hotplug(uvm, 256) From fa12003f30aa8863c416a0c002127f177d6abb15 Mon Sep 17 00:00:00 2001 From: Ilias Stamatis Date: Fri, 15 May 2026 17:30:00 +0100 Subject: [PATCH 28/28] CHANGELOG: Mention virtio device reset support https://github.com/firecracker-microvm/firecracker/pull/5891 Signed-off-by: Ilias Stamatis --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5977f24e39f..34140cbad71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to ### Added +- [#5891](https://github.com/firecracker-microvm/firecracker/pull/5891): Added + support for virtio device reset. - [#5983](https://github.com/firecracker-microvm/firecracker/pull/5983): Add two optional metrics fields, set via `PUT /metrics` or a config file: `emit_id` emits the microVM instance id under a top-level `id` field, and `properties`